From 70b9bb72932a2cff4b2f3ae6a80b20ce67fb5124 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Tue, 2 Mar 2021 09:43:12 -0500
Subject: [PATCH 01/59] runtime service

---
 qiskit/providers/ibmq/accountprovider.py      |  11 +
 qiskit/providers/ibmq/api/clients/__init__.py |   1 +
 qiskit/providers/ibmq/api/clients/runtime.py  | 120 +++++++++
 qiskit/providers/ibmq/api/rest/runtime.py     | 230 ++++++++++++++++++
 qiskit/providers/ibmq/runtime/__init__.py     |   0
 .../ibmq/runtime/ibm_runtime_service.py       |  75 ++++++
 qiskit/providers/ibmq/runtime/runtime_job.py  |  76 ++++++
 .../providers/ibmq/runtime/runtime_program.py | 107 ++++++++
 qiskit/providers/ibmq/utils/json_encoder.py   |  11 +
 test/ibmq/test_runtime.py                     |  51 ++++
 10 files changed, 682 insertions(+)
 create mode 100644 qiskit/providers/ibmq/api/clients/runtime.py
 create mode 100644 qiskit/providers/ibmq/api/rest/runtime.py
 create mode 100644 qiskit/providers/ibmq/runtime/__init__.py
 create mode 100644 qiskit/providers/ibmq/runtime/ibm_runtime_service.py
 create mode 100644 qiskit/providers/ibmq/runtime/runtime_job.py
 create mode 100644 qiskit/providers/ibmq/runtime/runtime_program.py
 create mode 100644 test/ibmq/test_runtime.py

diff --git a/qiskit/providers/ibmq/accountprovider.py b/qiskit/providers/ibmq/accountprovider.py
index b68a0464f..d552f4116 100644
--- a/qiskit/providers/ibmq/accountprovider.py
+++ b/qiskit/providers/ibmq/accountprovider.py
@@ -28,6 +28,7 @@
 from .utils.json_decoder import decode_backend_configuration
 from .random.ibmqrandomservice import IBMQRandomService
 from .experiment.experimentservice import ExperimentService
+from .runtime.ibm_runtime_service import IBMRuntimeService
 from .exceptions import IBMQNotAuthorizedError, IBMQInputValueError
 
 logger = logging.getLogger(__name__)
@@ -113,6 +114,7 @@ def __init__(self, credentials: Credentials, access_token: str) -> None:
             if credentials.extractor_url else None
         self._experiment = ExperimentService(self, access_token) \
             if credentials.experiment_url else None
+        self._runtime = IBMRuntimeService(self, access_token)
 
         self._services = {'backend': self._backend,
                           'random': self._random,
@@ -259,6 +261,15 @@ def random(self) -> IBMQRandomService:
         else:
             raise IBMQNotAuthorizedError("You are not authorized to use the random number service.")
 
+    @property
+    def runtime(self) -> IBMRuntimeService:
+        """Return the runtime service.
+
+        Returns:
+            The runtime service instance.
+        """
+        return self._runtime
+
     def __eq__(
             self,
             other: Any
diff --git a/qiskit/providers/ibmq/api/clients/__init__.py b/qiskit/providers/ibmq/api/clients/__init__.py
index 0bf1ad7cd..4e74efd1d 100644
--- a/qiskit/providers/ibmq/api/clients/__init__.py
+++ b/qiskit/providers/ibmq/api/clients/__init__.py
@@ -17,3 +17,4 @@
 from .auth import AuthClient
 from .version import VersionClient
 from .websocket import WebsocketClient
+from .runtime import RuntimeClient
diff --git a/qiskit/providers/ibmq/api/clients/runtime.py b/qiskit/providers/ibmq/api/clients/runtime.py
new file mode 100644
index 000000000..e79a83f47
--- /dev/null
+++ b/qiskit/providers/ibmq/api/clients/runtime.py
@@ -0,0 +1,120 @@
+# This code is part of Qiskit.
+#
+# (C) Copyright IBM 2020.
+#
+# This code is licensed under the Apache License, Version 2.0. You may
+# obtain a copy of this license in the LICENSE.txt file in the root directory
+# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Any modifications or derivative works of this code must retain this
+# copyright notice, and modified files need to carry a notice indicating
+# that they have been altered from the originals.
+
+"""Client for accessing Random Number Generator (RNG) services."""
+
+import logging
+from typing import List, Dict, Any
+
+from qiskit.providers.ibmq.credentials import Credentials
+from qiskit.providers.ibmq.api.session import RetrySession
+
+from ..rest.runtime import Runtime
+
+logger = logging.getLogger(__name__)
+
+
+class RuntimeClient:
+    """Client for accessing runtime service."""
+
+    def __init__(
+            self,
+            access_token: str,
+            credentials: Credentials,
+    ) -> None:
+        """RandomClient constructor.
+
+        Args:
+            access_token: IBM Quantum Experience access token.
+            credentials: Account credentials.
+        """
+        url = 'https://api.atabey-01bb92749871351d190eb4b7dadc92fb-0000.us-east.containers.appdomain.cloud'
+        self._session = RetrySession(url, access_token,
+                                     **credentials.connection_parameters())
+        self.api = Runtime(self._session)
+
+    def list_programs(self) -> List[Dict]:
+        """Return a list of runtime programs.
+
+        Returns:
+            A list of quantum programs.
+        """
+        return self.api.list_programs()
+
+    def program_get(self, program_id: str) -> Dict:
+        """Return a specific program.
+
+        Args:
+            program_id: Program ID.
+
+        Returns:
+            Program information.
+        """
+        return self.api.program(program_id).get()
+
+    def program_get_data(self, program_id: str) -> Dict:
+        """Return a specific program and its data.
+
+        Args:
+            program_id: Program ID.
+
+        Returns:
+            Program information, including data.
+        """
+        return self.api.program(program_id).get_data()
+
+    def program_run(
+            self,
+            program_id: str,
+            credentials: Credentials,
+            backend_name: str,
+            params: Dict
+    ) -> Dict:
+        """Run the specified program.
+
+        Args:
+            program_id: Program ID.
+            credentials: Credentials used to run the program.
+            backend_name: Name of the backend to run the program.
+            params: Parameters to use.
+
+        Returns:
+            JSON response.
+        """
+        return self.api.program(program_id).run(
+            hub=credentials.hub, group=credentials.group, project=credentials.project,
+            backend_name=backend_name, params=params)
+
+    def program_delete(self, program_id: str):
+        """Delete the specified program.
+
+        Args:
+            program_id: Program ID.
+
+        Returns:
+            JSON response.
+        """
+        return self.api.program(program_id).delete()
+
+    def program_job_get(self, program_id, job_id):
+        return self.api.program_job(program_id, job_id).get()
+
+    def program_job_results(self, program_id, job_id: str) -> Dict:
+        """Get the results of a program job.
+
+        Args:
+            job_id: Program job ID.
+
+        Returns:
+            JSON response.
+        """
+        return self.api.program_job(program_id, job_id).results()
diff --git a/qiskit/providers/ibmq/api/rest/runtime.py b/qiskit/providers/ibmq/api/rest/runtime.py
new file mode 100644
index 000000000..6a41a071a
--- /dev/null
+++ b/qiskit/providers/ibmq/api/rest/runtime.py
@@ -0,0 +1,230 @@
+# This code is part of Qiskit.
+#
+# (C) Copyright IBM 2020.
+#
+# This code is licensed under the Apache License, Version 2.0. You may
+# obtain a copy of this license in the LICENSE.txt file in the root directory
+# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Any modifications or derivative works of this code must retain this
+# copyright notice, and modified files need to carry a notice indicating
+# that they have been altered from the originals.
+
+"""Random REST adapter."""
+
+import logging
+from typing import Dict, List, Any
+import json
+import subprocess
+
+from qiskit.providers.ibmq.utils import json_encoder
+from .base import RestAdapterBase
+from ..session import RetrySession
+
+logger = logging.getLogger(__name__)
+process = None
+
+
+class Runtime(RestAdapterBase):
+    """Rest adapter for RNG related endpoints."""
+
+    URL_MAP = {
+        'self': '/programs'
+    }
+
+    def program(self, program_id: str) -> 'Program':
+        """Return an adapter for the program.
+
+        Args:
+            program_id: ID of the program.
+
+        Returns:
+            The program adapter.
+        """
+        return Program(self.session, program_id)
+
+    def program_job(self, program_id, job_id):
+        return ProgramJob(self.session, program_id, job_id)
+
+    def list_programs(self) -> List[Dict]:
+        """Return a list of runtime programs.
+
+        Returns:
+            JSON response.
+        """
+        url = self.get_url('self')
+        # return self.session.get(url).json()
+        # temporary code
+        with open('qka_doc.json', 'r') as file:
+            data = json.load(file)
+        return [data]
+
+    def create_program(self, name: str, data: bytes) -> Dict:
+        """Upload a new program.
+
+        Args:
+            name: Name of the program.
+            data: Program data.
+
+        Returns:
+            JSON response.
+        """
+        url = self.get_url('self')
+        data = {'name': name,
+                'program': (name, data)}  # type: ignore[dict-item]
+        return self.session.post(url, files=data).json()
+
+
+class Program(RestAdapterBase):
+    """Rest adapter for program related endpoints."""
+
+    URL_MAP = {
+        'self': '',
+        'data': '/data',
+        'run': '/jobs'
+    }
+
+    def __init__(self, session: RetrySession, program_id: str, url_prefix: str = '') -> None:
+        """Job constructor.
+
+        Args:
+            session: Session to be used in the adapter.
+            program_id: ID of the runtime program.
+            url_prefix: Prefix to use in the URL.
+        """
+        super().__init__(session, '{}/programs/{}'.format(url_prefix, program_id))
+
+    def get(self) -> Dict[str, Any]:
+        """Return program information.
+
+        Returns:
+            JSON response.
+        """
+        url = self.get_url('self')
+        return self.session.get(url).json()
+
+    def get_data(self) -> Dict[str, Any]:
+        """Return program information, including data.
+
+        Returns:
+            JSON response.
+        """
+        url = self.get_url('data')
+        return self.session.get(url).json()
+
+    def run(self, hub: str, group: str, project: str, backend_name: str, params: Dict) -> Dict:
+        """Execute the program.
+
+        Args:
+            hub: Hub to be used.
+            group: Group to be used.
+            project: Project to be used.
+            backend_name: Name of the backend.
+            params: Program parameters.
+
+        Returns:
+            JSON response.
+        """
+        url = self.get_url('run')
+        payload = {
+            'hub': hub,
+            'group': group,
+            'project': project,
+            'backend': backend_name,
+            'params': params
+        }
+        # data = json.dumps(payload, cls=json_encoder.IQXJsonEncoder)
+        data = json.dumps(params, cls=json_encoder.NumpyEncoder)
+        global process
+        process = subprocess.Popen(["/Users/jessieyu/.pyenv/versions/provider-dev-3.8.1/bin/python3",
+                                    "qka_program.py", data],
+                                   stdout=subprocess.PIPE, stderr=subprocess.PIPE,
+                                   universal_newlines=True)
+        import uuid
+        return {'id': uuid.uuid4().hex}
+        # return self.session.post(url, data=data).json()
+
+    def delete(self) -> Dict:
+        """Delete this program.
+
+        Returns:
+            JSON response.
+        """
+        url = self.get_url('self')
+        return self.session.delete(url).json()
+
+
+class ProgramJob(RestAdapterBase):
+    """Rest adapter for program job related endpoints."""
+
+    URL_MAP = {
+        'self': '',
+        'results': 'results'
+    }
+
+    def __init__(
+            self,
+            session: RetrySession,
+            program_id: str,
+            job_id: str,
+            url_prefix: str = ''
+    ) -> None:
+        """ProgramJob constructor.
+
+        Args:
+            session: Session to be used in the adapter.
+            program_id: ID of the runtime program.
+            job_id: ID of the program job.
+            url_prefix: Prefix to use in the URL.
+        """
+        super().__init__(session, '{}/programs/{}/jobs/{}'.format(
+            url_prefix, program_id, job_id))
+
+    def get(self) -> Dict:
+        """Return program job information.
+
+        Returns:
+            JSON response.
+        """
+        output = {}
+        global process
+        if process is not None:
+            rc = process.poll()
+            if rc is None:
+                output['status'] = 'RUNNING'
+            elif rc < 0:
+                output['status'] = 'ERROR'
+            else:
+                output['status'] = 'DONE'
+        return output
+        # return self.session.get(self.get_url('self')).json()
+
+    def delete(self) -> Dict:
+        """Delete program job.
+
+        Returns:
+            JSON response.
+        """
+        return self.session.delete(self.get_url('self')).json()
+
+    def results(self) -> Dict:
+        """Return program job results.
+
+        Returns:
+            JSON response.
+        """
+        global process
+        if process is not None:
+            outs, errs = process.communicate()
+            outs = outs.split('\n')
+            for line in outs:
+                try:
+                    parsed = json.loads(line)
+                    if 'results' in parsed:
+                        return parsed['results']
+                except:
+                    print(line)
+            print(f">>>>> errs: {errs}")
+
+        return {}
+        # return self.session.get(self.get_url('results')).json()
diff --git a/qiskit/providers/ibmq/runtime/__init__.py b/qiskit/providers/ibmq/runtime/__init__.py
new file mode 100644
index 000000000..e69de29bb
diff --git a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
new file mode 100644
index 000000000..8e3ce203d
--- /dev/null
+++ b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
@@ -0,0 +1,75 @@
+# This code is part of Qiskit.
+#
+# (C) Copyright IBM 2021.
+#
+# This code is licensed under the Apache License, Version 2.0. You may
+# obtain a copy of this license in the LICENSE.txt file in the root directory
+# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Any modifications or derivative works of this code must retain this
+# copyright notice, and modified files need to carry a notice indicating
+# that they have been altered from the originals.
+
+"""IBM Quantum Experience runtime service."""
+
+import logging
+from typing import Dict
+
+from qiskit.providers.ibmq import accountprovider  # pylint: disable=unused-import
+
+from .runtime_job import RuntimeJob
+from .runtime_program import RuntimeProgram
+from ..ibmqbackend import IBMQBackend
+from ..api.clients.runtime import RuntimeClient
+
+logger = logging.getLogger(__name__)
+
+
+class IBMRuntimeService:
+    """Backend namespace for an IBM Quantum Experience account provider.
+    """
+
+    def __init__(self, provider: 'accountprovider.AccountProvider', access_token: str) -> None:
+        """IBMQRandomService constructor.
+
+        Args:
+            provider: IBM Quantum Experience account provider.
+            access_token: IBM Quantum Experience access token.
+        """
+        self._provider = provider
+        self._api_client = RuntimeClient(access_token, provider.credentials)
+        self._programs = {}
+
+    def programs(self):
+        response = self._api_client.list_programs()
+        for prog_dict in response:
+            kwargs = {}
+            if 'cost' in prog_dict:
+                kwargs['cost'] = prog_dict['cost']
+            if 'data' in prog_dict:
+                kwargs['data'] = prog_dict['data']
+            program = RuntimeProgram(program_name=prog_dict['name'],
+                                     program_id=prog_dict['id'],
+                                     description=prog_dict['description'],
+                                     parameters=prog_dict['parameters'],
+                                     return_values=prog_dict['return_values'],
+                                     **kwargs)
+            self._programs[program.name] = program
+        for prog in self._programs.values():
+            prog.pprint()
+
+    def program(self, program_name: str):
+        if program_name in self._programs:
+            self._programs[program_name].pprint()
+        else:
+            program = RuntimeProgram(**self._api_client.program_get(program_name))
+            self._programs[program.name] = program
+            program.pprint()
+
+    def run(self, program_name: str, backend: IBMQBackend, params: Dict) -> RuntimeJob:
+        response = self._api_client.program_run(program_id=program_name,
+                                                credentials=self._provider.credentials,
+                                                backend_name=backend.name(),
+                                                params=params)
+        job = RuntimeJob(backend=backend, api_client=self._api_client, job_id=response['id'])
+        return job
diff --git a/qiskit/providers/ibmq/runtime/runtime_job.py b/qiskit/providers/ibmq/runtime/runtime_job.py
new file mode 100644
index 000000000..5dae76cf4
--- /dev/null
+++ b/qiskit/providers/ibmq/runtime/runtime_job.py
@@ -0,0 +1,76 @@
+# This code is part of Qiskit.
+#
+# (C) Copyright IBM 2021.
+#
+# This code is licensed under the Apache License, Version 2.0. You may
+# obtain a copy of this license in the LICENSE.txt file in the root directory
+# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Any modifications or derivative works of this code must retain this
+# copyright notice, and modified files need to carry a notice indicating
+# that they have been altered from the originals.
+
+"""IBM Quantum Experience Runtime job."""
+
+from typing import Any, Optional
+
+from qiskit.providers.job import JobV1 as Job
+from qiskit.providers.jobstatus import JobStatus
+
+from ..api.clients import RuntimeClient
+
+
+class RuntimeJob(Job):
+
+    def __init__(
+            self,
+            backend: 'ibmqbackend.IBMQBackend',
+            api_client: RuntimeClient,
+            job_id: str,
+    ) -> None:
+        """RuntimeJob constructor.
+
+        Args:
+            backend: The backend instance used to run this job.
+            api_client: Object for connecting to the server.
+            job_id: Job ID.
+        """
+        super().__init__(backend, job_id)
+        self._api_client = api_client
+
+    def submit(self):
+        """Unsupported method.
+
+        Note:
+            This method is not supported, please use
+            :meth:`~qiskit.providers.ibmq.ibmqbackend.IBMQBackend.run`
+            to submit a job.
+
+        Raises:
+            NotImplementedError: Upon invocation.
+        """
+        raise NotImplementedError("submit() is not supported. Please use "
+                                  "IBMRuntimeService.run() to submit a runtime job.")
+
+    def result(
+            self,
+            timeout: Optional[float] = None
+    ) -> Any:
+        """Return the results of the job."""
+        self.wait_for_final_state(timeout=timeout)
+        return self._api_client.program_job_results(program_id='123', job_id=self.job_id())
+
+    def cancel(self):
+        """Attempt to cancel the job."""
+        raise NotImplementedError
+
+    def status(self) -> JobStatus:
+        """Return the status of the job."""
+        response = self._api_client.program_job_get(program_id='123', job_id=self.job_id())
+        status = response['status']
+        if status == 'RUNNING':
+            return JobStatus.RUNNING
+        elif status == 'DONE':
+            return JobStatus.DONE
+        else:
+            return JobStatus.ERROR
diff --git a/qiskit/providers/ibmq/runtime/runtime_program.py b/qiskit/providers/ibmq/runtime/runtime_program.py
new file mode 100644
index 000000000..4d272edee
--- /dev/null
+++ b/qiskit/providers/ibmq/runtime/runtime_program.py
@@ -0,0 +1,107 @@
+# This code is part of Qiskit.
+#
+# (C) Copyright IBM 2021.
+#
+# This code is licensed under the Apache License, Version 2.0. You may
+# obtain a copy of this license in the LICENSE.txt file in the root directory
+# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Any modifications or derivative works of this code must retain this
+# copyright notice, and modified files need to carry a notice indicating
+# that they have been altered from the originals.
+
+import logging
+from datetime import datetime
+from typing import Dict, Optional, List, Union
+
+logger = logging.getLogger(__name__)
+
+
+class RuntimeProgram:
+    """Class representing a program definition."""
+
+    def __init__(
+            self,
+            program_name: str,
+            program_id: str,
+            description: str,
+            parameters: List,
+            return_values: List,
+            cost: float = 0,
+            data: Optional[bytes] = None
+    ) -> None:
+        """
+
+        Args:
+            program_name:
+            program_id:
+            description:
+            cost:
+        """
+        self.name = program_name
+        self._id = program_id
+        self._description = description
+        self._cost = cost
+        self._data = data
+        self._parameters = []
+        self._return_values = []
+        for param in parameters:
+            self._parameters.append(
+                ProgramParameter(name=param['name'],
+                                 description=param['description'],
+                                 param_type=param['type']))
+        for ret in return_values:
+            self._return_values.append(ProgramReturn(name=ret['name'],
+                                                     description=ret['description'],
+                                                     return_type=ret['type']))
+
+    def pprint(self):
+        formatted = [f"Runtime Program {self.name}:",
+                     f"  Description: {self._description}",
+                     f"  Parameters:"]
+
+        if self._parameters:
+            for param in self._parameters:
+                formatted.append(" "*4 + param.name + ":")
+                formatted.append(" "*6 + "description: " + param.description)
+                formatted.append(" "*6 + "type: " + param.type)
+        else:
+            formatted.append(" "*4 + "none")
+
+        formatted.append("  Returns:")
+        if self._return_values:
+            for ret in self._return_values:
+                formatted.append(" "*4 + ret.name + ":")
+                formatted.append(" "*6 + "description: " + ret.description)
+                formatted.append(" "*6 + "type: " + ret.type)
+        else:
+            formatted.append(" "*4 + "none")
+        print('\n'.join(formatted))
+
+
+class ProgramParameter:
+
+    def __init__(self, name: str, description: str, param_type: str):
+        """
+
+        Args:
+            description:
+            param_type:
+        """
+        self.name = name
+        self.description = description
+        self.type = param_type
+
+
+class ProgramReturn:
+
+    def __init__(self, name: str, description: str, return_type: str):
+        """
+
+        Args:
+            description:
+            return_type:
+        """
+        self.name = name
+        self.description = description
+        self.type = return_type
diff --git a/qiskit/providers/ibmq/utils/json_encoder.py b/qiskit/providers/ibmq/utils/json_encoder.py
index bb926dd29..71f396ab1 100644
--- a/qiskit/providers/ibmq/utils/json_encoder.py
+++ b/qiskit/providers/ibmq/utils/json_encoder.py
@@ -37,3 +37,14 @@ def default(self, o: Any) -> Any:
                 val = complex(o)
                 return val.real, val.imag
         return json.JSONEncoder.default(self, o)
+
+
+class NumpyEncoder(json.JSONEncoder):
+    """JSON Encoder for Numpy arrays and complex numbers."""
+
+    def default(self, obj: Any) -> Any:
+        if hasattr(obj, 'tolist'):
+            return {'type': 'array', 'value': obj.tolist()}
+        if isinstance(obj, complex):
+            return {'type': 'complex', 'value': [obj.real, obj.imag]}
+        return super().default(obj)
diff --git a/test/ibmq/test_runtime.py b/test/ibmq/test_runtime.py
new file mode 100644
index 000000000..167b258dd
--- /dev/null
+++ b/test/ibmq/test_runtime.py
@@ -0,0 +1,51 @@
+# This code is part of Qiskit.
+#
+# (C) Copyright IBM 2021.
+#
+# This code is licensed under the Apache License, Version 2.0. You may
+# obtain a copy of this license in the LICENSE.txt file in the root directory
+# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Any modifications or derivative works of this code must retain this
+# copyright notice, and modified files need to carry a notice indicating
+# that they have been altered from the originals.
+
+"""Tests for random number services."""
+
+import time
+import uuid
+from unittest import skipIf
+from concurrent.futures import ThreadPoolExecutor
+
+import numpy as np
+from qiskit.providers.jobstatus import JobStatus
+from qiskit.providers.ibmq.exceptions import IBMQError
+
+from ..ibmqtestcase import IBMQTestCase
+from ..decorators import requires_provider
+
+
+class TestRuntime(IBMQTestCase):
+
+    @classmethod
+    @requires_provider
+    def setUpClass(cls, provider):
+        """Initial class level setup."""
+        # pylint: disable=arguments-differ
+        super().setUpClass()
+        cls.provider = provider
+
+    def test_list_programs(self):
+        """Test listing programs."""
+        self.provider.runtime.programs()
+
+    def test_run_program(self):
+        """Test running program."""
+        params = {'param1': 'foo'}
+        backend = self.provider.backend.ibmq_qasm_simulator
+        job = self.provider.runtime.run("QKA", backend=backend, params=params)
+        self.assertTrue(job.job_id())
+        self.assertIsInstance(job.status(), JobStatus)
+        job.wait_for_final_state()
+        self.assertEqual(job.status(), JobStatus.DONE)
+        self.assertTrue(job.result())

From c574e0f89ce23364a2beee810920aea7146c37e9 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Tue, 2 Mar 2021 16:09:01 -0500
Subject: [PATCH 02/59] add directory

---
 qiskit/providers/ibmq/api/clients/runtime.py |  2 +-
 qiskit/providers/ibmq/api/rest/runtime.py    | 10 ++++++----
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/qiskit/providers/ibmq/api/clients/runtime.py b/qiskit/providers/ibmq/api/clients/runtime.py
index e79a83f47..4c3362121 100644
--- a/qiskit/providers/ibmq/api/clients/runtime.py
+++ b/qiskit/providers/ibmq/api/clients/runtime.py
@@ -37,7 +37,7 @@ def __init__(
             access_token: IBM Quantum Experience access token.
             credentials: Account credentials.
         """
-        url = 'https://api.atabey-01bb92749871351d190eb4b7dadc92fb-0000.us-east.containers.appdomain.cloud'
+        url = ''
         self._session = RetrySession(url, access_token,
                                      **credentials.connection_parameters())
         self.api = Runtime(self._session)
diff --git a/qiskit/providers/ibmq/api/rest/runtime.py b/qiskit/providers/ibmq/api/rest/runtime.py
index 6a41a071a..339a9d6da 100644
--- a/qiskit/providers/ibmq/api/rest/runtime.py
+++ b/qiskit/providers/ibmq/api/rest/runtime.py
@@ -16,6 +16,7 @@
 from typing import Dict, List, Any
 import json
 import subprocess
+import os
 
 from qiskit.providers.ibmq.utils import json_encoder
 from .base import RestAdapterBase
@@ -55,7 +56,7 @@ def list_programs(self) -> List[Dict]:
         url = self.get_url('self')
         # return self.session.get(url).json()
         # temporary code
-        with open('qka_doc.json', 'r') as file:
+        with open('runtime/qka_doc.json', 'r') as file:
             data = json.load(file)
         return [data]
 
@@ -134,10 +135,12 @@ def run(self, hub: str, group: str, project: str, backend_name: str, params: Dic
             'params': params
         }
         # data = json.dumps(payload, cls=json_encoder.IQXJsonEncoder)
+        # temporary code
+        python_bin = os.getenv('PYTHON_EXEC', 'python3')
         data = json.dumps(params, cls=json_encoder.NumpyEncoder)
         global process
-        process = subprocess.Popen(["/Users/jessieyu/.pyenv/versions/provider-dev-3.8.1/bin/python3",
-                                    "qka_program.py", data],
+        process = subprocess.Popen([python_bin,
+                                    "runtime/qka_program.py", data],
                                    stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                                    universal_newlines=True)
         import uuid
@@ -224,7 +227,6 @@ def results(self) -> Dict:
                         return parsed['results']
                 except:
                     print(line)
-            print(f">>>>> errs: {errs}")
 
         return {}
         # return self.session.get(self.get_url('results')).json()

From ff71d41f173a7588f4ef4332bdd78be620b0de4b Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Tue, 2 Mar 2021 17:08:17 -0500
Subject: [PATCH 03/59] use env var for file names

---
 qiskit/providers/ibmq/api/rest/runtime.py     |  7 ++--
 qiskit/providers/ibmq/runtime/__init__.py     | 42 +++++++++++++++++++
 .../ibmq/runtime/ibm_runtime_service.py       | 11 +++--
 setup.py                                      |  3 +-
 4 files changed, 53 insertions(+), 10 deletions(-)

diff --git a/qiskit/providers/ibmq/api/rest/runtime.py b/qiskit/providers/ibmq/api/rest/runtime.py
index 339a9d6da..cb2bda9b7 100644
--- a/qiskit/providers/ibmq/api/rest/runtime.py
+++ b/qiskit/providers/ibmq/api/rest/runtime.py
@@ -56,7 +56,8 @@ def list_programs(self) -> List[Dict]:
         url = self.get_url('self')
         # return self.session.get(url).json()
         # temporary code
-        with open('runtime/qka_doc.json', 'r') as file:
+        doc_file = os.getenv('NTC_DOC_FILE', '../runtime/qka_doc.json')
+        with open(doc_file, 'r') as file:
             data = json.load(file)
         return [data]
 
@@ -137,10 +138,10 @@ def run(self, hub: str, group: str, project: str, backend_name: str, params: Dic
         # data = json.dumps(payload, cls=json_encoder.IQXJsonEncoder)
         # temporary code
         python_bin = os.getenv('PYTHON_EXEC', 'python3')
+        program_file = os.getenv('NTC_PROGRAM_FILE', '../runtime/qka_program.py')
         data = json.dumps(params, cls=json_encoder.NumpyEncoder)
         global process
-        process = subprocess.Popen([python_bin,
-                                    "runtime/qka_program.py", data],
+        process = subprocess.Popen([python_bin, program_file, data],
                                    stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                                    universal_newlines=True)
         import uuid
diff --git a/qiskit/providers/ibmq/runtime/__init__.py b/qiskit/providers/ibmq/runtime/__init__.py
index e69de29bb..609e1e1fb 100644
--- a/qiskit/providers/ibmq/runtime/__init__.py
+++ b/qiskit/providers/ibmq/runtime/__init__.py
@@ -0,0 +1,42 @@
+# This code is part of Qiskit.
+#
+# (C) Copyright IBM 2021.
+#
+# This code is licensed under the Apache License, Version 2.0. You may
+# obtain a copy of this license in the LICENSE.txt file in the root directory
+# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Any modifications or derivative works of this code must retain this
+# copyright notice, and modified files need to carry a notice indicating
+# that they have been altered from the originals.
+
+"""
+============================================================
+Runtime Service (:mod:`qiskit.providers.ibmq.runtime`)
+============================================================
+
+.. currentmodule:: qiskit.providers.ibmq.runtime
+
+Modules related to IBM Quantum Experience runtime service.
+
+.. caution::
+
+  This package is currently provided in beta form and heavy modifications to
+  both functionality and API are likely to occur.
+
+.. note::
+
+  The runtime service is not available to all accounts.
+
+Classes
+==========================
+.. autosummary::
+   :toctree: ../stubs/
+
+   IBMRuntimeService
+   RuntimeJob
+
+"""
+
+from .ibm_runtime_service import IBMRuntimeService
+from .runtime_job import RuntimeJob
diff --git a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
index 8e3ce203d..8894b7808 100644
--- a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
+++ b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
@@ -10,7 +10,7 @@
 # copyright notice, and modified files need to carry a notice indicating
 # that they have been altered from the originals.
 
-"""IBM Quantum Experience runtime service."""
+"""IBM Quantum runtime service."""
 
 import logging
 from typing import Dict
@@ -26,15 +26,14 @@
 
 
 class IBMRuntimeService:
-    """Backend namespace for an IBM Quantum Experience account provider.
-    """
+    """IBM Quantum runtime service."""
 
     def __init__(self, provider: 'accountprovider.AccountProvider', access_token: str) -> None:
-        """IBMQRandomService constructor.
+        """IBMRuntimeService constructor.
 
         Args:
-            provider: IBM Quantum Experience account provider.
-            access_token: IBM Quantum Experience access token.
+            provider: IBM Quantum account provider.
+            access_token: IBM Quantum access token.
         """
         self._provider = provider
         self._api_client = RuntimeClient(access_token, provider.credentials)
diff --git a/setup.py b/setup.py
index a69d84ae6..13b84d5f5 100644
--- a/setup.py
+++ b/setup.py
@@ -81,7 +81,8 @@
               'qiskit.providers.ibmq.jupyter',
               'qiskit.providers.ibmq.jupyter.dashboard',
               'qiskit.providers.ibmq.random',
-              'qiskit.providers.ibmq.experiment'],
+              'qiskit.providers.ibmq.experiment',
+              'qiskit.providers.ibmq.runtime'],
     install_requires=REQUIREMENTS,
     include_package_data=True,
     python_requires=">=3.6",

From 2c71ce8c38df314ef314e7979f848321c9b96a68 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Tue, 2 Mar 2021 18:59:43 -0500
Subject: [PATCH 04/59] add interim queue

---
 qiskit/providers/ibmq/api/clients/runtime.py  |  5 ++-
 qiskit/providers/ibmq/api/rest/runtime.py     | 38 ++++++++++++++++---
 .../ibmq/runtime/ibm_runtime_service.py       | 29 ++++++++++++--
 3 files changed, 62 insertions(+), 10 deletions(-)

diff --git a/qiskit/providers/ibmq/api/clients/runtime.py b/qiskit/providers/ibmq/api/clients/runtime.py
index 4c3362121..6931b1781 100644
--- a/qiskit/providers/ibmq/api/clients/runtime.py
+++ b/qiskit/providers/ibmq/api/clients/runtime.py
@@ -77,7 +77,8 @@ def program_run(
             program_id: str,
             credentials: Credentials,
             backend_name: str,
-            params: Dict
+            params: Dict,
+            interim_queue
     ) -> Dict:
         """Run the specified program.
 
@@ -92,7 +93,7 @@ def program_run(
         """
         return self.api.program(program_id).run(
             hub=credentials.hub, group=credentials.group, project=credentials.project,
-            backend_name=backend_name, params=params)
+            backend_name=backend_name, params=params, interim_queue=interim_queue)
 
     def program_delete(self, program_id: str):
         """Delete the specified program.
diff --git a/qiskit/providers/ibmq/api/rest/runtime.py b/qiskit/providers/ibmq/api/rest/runtime.py
index cb2bda9b7..8dcff4900 100644
--- a/qiskit/providers/ibmq/api/rest/runtime.py
+++ b/qiskit/providers/ibmq/api/rest/runtime.py
@@ -13,10 +13,13 @@
 """Random REST adapter."""
 
 import logging
-from typing import Dict, List, Any
+from typing import Dict, List, Any, Optional
 import json
 import subprocess
 import os
+import queue
+from concurrent import futures
+import uuid
 
 from qiskit.providers.ibmq.utils import json_encoder
 from .base import RestAdapterBase
@@ -56,7 +59,7 @@ def list_programs(self) -> List[Dict]:
         url = self.get_url('self')
         # return self.session.get(url).json()
         # temporary code
-        doc_file = os.getenv('NTC_DOC_FILE', '../runtime/qka_doc.json')
+        doc_file = os.getenv('NTC_DOC_FILE', 'runtime/qka_doc.json')
         with open(doc_file, 'r') as file:
             data = json.load(file)
         return [data]
@@ -86,6 +89,8 @@ class Program(RestAdapterBase):
         'run': '/jobs'
     }
 
+    _executor = futures.ThreadPoolExecutor()
+
     def __init__(self, session: RetrySession, program_id: str, url_prefix: str = '') -> None:
         """Job constructor.
 
@@ -114,7 +119,15 @@ def get_data(self) -> Dict[str, Any]:
         url = self.get_url('data')
         return self.session.get(url).json()
 
-    def run(self, hub: str, group: str, project: str, backend_name: str, params: Dict) -> Dict:
+    def run(
+            self,
+            hub: str,
+            group: str,
+            project: str,
+            backend_name: str,
+            params: Dict,
+            interim_queue: Optional[queue.Queue] = None
+    ) -> Dict:
         """Execute the program.
 
         Args:
@@ -138,16 +151,31 @@ def run(self, hub: str, group: str, project: str, backend_name: str, params: Dic
         # data = json.dumps(payload, cls=json_encoder.IQXJsonEncoder)
         # temporary code
         python_bin = os.getenv('PYTHON_EXEC', 'python3')
-        program_file = os.getenv('NTC_PROGRAM_FILE', '../runtime/qka_program.py')
+        program_file = os.getenv('NTC_PROGRAM_FILE', 'runtime/qka_program.py')
         data = json.dumps(params, cls=json_encoder.NumpyEncoder)
         global process
         process = subprocess.Popen([python_bin, program_file, data],
                                    stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                                    universal_newlines=True)
-        import uuid
+        if interim_queue:
+            self._executor.submit(self._interim_result, interim_queue)
+
         return {'id': uuid.uuid4().hex}
         # return self.session.post(url, data=data).json()
 
+    def _interim_result(self, interim_queue: queue.Queue):
+        global process
+        if process is None:
+            return
+        while True:
+            nextline = process.stdout.readline()
+            if nextline == '' and process.poll() is not None:
+                break
+            parsed = json.loads(nextline)
+            if any(word in parsed for word in ['post', 'results']):
+                interim_queue.put_nowait(parsed)
+        interim_queue.put_nowait('poison_pill')
+
     def delete(self) -> Dict:
         """Delete this program.
 
diff --git a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
index 8894b7808..52b47799d 100644
--- a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
+++ b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
@@ -13,7 +13,9 @@
 """IBM Quantum runtime service."""
 
 import logging
-from typing import Dict
+from typing import Dict, Callable
+import queue
+from concurrent import futures
 
 from qiskit.providers.ibmq import accountprovider  # pylint: disable=unused-import
 
@@ -28,6 +30,8 @@
 class IBMRuntimeService:
     """IBM Quantum runtime service."""
 
+    _executor = futures.ThreadPoolExecutor()
+
     def __init__(self, provider: 'accountprovider.AccountProvider', access_token: str) -> None:
         """IBMRuntimeService constructor.
 
@@ -65,10 +69,29 @@ def program(self, program_name: str):
             self._programs[program.name] = program
             program.pprint()
 
-    def run(self, program_name: str, backend: IBMQBackend, params: Dict) -> RuntimeJob:
+    def run(
+            self,
+            program_name: str,
+            backend: IBMQBackend,
+            params: Dict,
+            callback: Callable
+    ) -> RuntimeJob:
+        interim_queue = None
+        if callback:
+            interim_queue = queue.Queue()
+            self._executor.submit(self._get_interim_result, interim_queue, callback)
         response = self._api_client.program_run(program_id=program_name,
                                                 credentials=self._provider.credentials,
                                                 backend_name=backend.name(),
-                                                params=params)
+                                                params=params, interim_queue=interim_queue)
         job = RuntimeJob(backend=backend, api_client=self._api_client, job_id=response['id'])
         return job
+
+    def _get_interim_result(self, result_queue: queue.Queue, user_callback):
+        try:
+            interim_result = result_queue.get(block=True, timeout=5)
+            if interim_result == 'poison_pill':
+                return
+            user_callback(interim_result)
+        except queue.Empty:
+            pass

From f2c801ae039994c391c10f797732130adb4c9e6f Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Wed, 3 Mar 2021 10:55:04 -0500
Subject: [PATCH 05/59] interim result

---
 qiskit/providers/ibmq/api/clients/runtime.py  |  5 ++-
 qiskit/providers/ibmq/api/rest/runtime.py     | 37 ++++++++++++++-----
 .../ibmq/runtime/ibm_runtime_service.py       | 21 +++--------
 qiskit/providers/ibmq/runtime/runtime_job.py  | 34 ++++++++++++++++-
 test/ibmq/test_runtime.py                     | 17 +++++----
 5 files changed, 76 insertions(+), 38 deletions(-)

diff --git a/qiskit/providers/ibmq/api/clients/runtime.py b/qiskit/providers/ibmq/api/clients/runtime.py
index 6931b1781..dea6f8a14 100644
--- a/qiskit/providers/ibmq/api/clients/runtime.py
+++ b/qiskit/providers/ibmq/api/clients/runtime.py
@@ -13,7 +13,8 @@
 """Client for accessing Random Number Generator (RNG) services."""
 
 import logging
-from typing import List, Dict, Any
+from typing import List, Dict, Optional
+import queue
 
 from qiskit.providers.ibmq.credentials import Credentials
 from qiskit.providers.ibmq.api.session import RetrySession
@@ -78,7 +79,7 @@ def program_run(
             credentials: Credentials,
             backend_name: str,
             params: Dict,
-            interim_queue
+            interim_queue: Optional[queue.Queue] = None
     ) -> Dict:
         """Run the specified program.
 
diff --git a/qiskit/providers/ibmq/api/rest/runtime.py b/qiskit/providers/ibmq/api/rest/runtime.py
index 8dcff4900..86abe6b82 100644
--- a/qiskit/providers/ibmq/api/rest/runtime.py
+++ b/qiskit/providers/ibmq/api/rest/runtime.py
@@ -20,6 +20,7 @@
 import queue
 from concurrent import futures
 import uuid
+import numpy as np
 
 from qiskit.providers.ibmq.utils import json_encoder
 from .base import RestAdapterBase
@@ -158,22 +159,22 @@ def run(
                                    stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                                    universal_newlines=True)
         if interim_queue:
-            self._executor.submit(self._interim_result, interim_queue)
+            self._executor.submit(self._interim_result, interim_queue, process)
 
         return {'id': uuid.uuid4().hex}
         # return self.session.post(url, data=data).json()
 
-    def _interim_result(self, interim_queue: queue.Queue):
-        global process
-        if process is None:
-            return
+    def _interim_result(self, interim_queue: queue.Queue, pgm_process):
         while True:
-            nextline = process.stdout.readline()
-            if nextline == '' and process.poll() is not None:
+            nextline = pgm_process.stdout.readline()
+            if nextline == '' and pgm_process.poll() is not None:
                 break
-            parsed = json.loads(nextline)
-            if any(word in parsed for word in ['post', 'results']):
-                interim_queue.put_nowait(parsed)
+            try:
+                parsed = json.loads(nextline, cls=NumpyDecoder)
+                if any(text in parsed for text in ['post', 'results']):
+                    interim_queue.put_nowait(parsed)
+            except:
+                print(nextline)
         interim_queue.put_nowait('poison_pill')
 
     def delete(self) -> Dict:
@@ -259,3 +260,19 @@ def results(self) -> Dict:
 
         return {}
         # return self.session.get(self.get_url('results')).json()
+
+
+class NumpyDecoder(json.JSONDecoder):
+    """JSON Decoder for Numpy arrays and complex numbers."""
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(object_hook=self.object_hook, *args, **kwargs)
+
+    def object_hook(self, obj):
+        if 'type' in obj:
+            if obj['type'] == 'complex':
+                val = obj['value']
+                return val[0] + 1j * val[1]
+            if obj['type'] == 'array':
+                return np.array(obj['value'])
+        return obj
diff --git a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
index 52b47799d..923c5b49a 100644
--- a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
+++ b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
@@ -13,7 +13,7 @@
 """IBM Quantum runtime service."""
 
 import logging
-from typing import Dict, Callable
+from typing import Dict, Callable, Optional
 import queue
 from concurrent import futures
 
@@ -74,24 +74,13 @@ def run(
             program_name: str,
             backend: IBMQBackend,
             params: Dict,
-            callback: Callable
+            callback: Optional[Callable] = None
     ) -> RuntimeJob:
-        interim_queue = None
-        if callback:
-            interim_queue = queue.Queue()
-            self._executor.submit(self._get_interim_result, interim_queue, callback)
+        interim_queue = queue.Queue() if callback else None
         response = self._api_client.program_run(program_id=program_name,
                                                 credentials=self._provider.credentials,
                                                 backend_name=backend.name(),
                                                 params=params, interim_queue=interim_queue)
-        job = RuntimeJob(backend=backend, api_client=self._api_client, job_id=response['id'])
+        job = RuntimeJob(backend=backend, api_client=self._api_client, job_id=response['id'],
+                         interim_queue=interim_queue, user_callback=callback)
         return job
-
-    def _get_interim_result(self, result_queue: queue.Queue, user_callback):
-        try:
-            interim_result = result_queue.get(block=True, timeout=5)
-            if interim_result == 'poison_pill':
-                return
-            user_callback(interim_result)
-        except queue.Empty:
-            pass
diff --git a/qiskit/providers/ibmq/runtime/runtime_job.py b/qiskit/providers/ibmq/runtime/runtime_job.py
index 5dae76cf4..09173a19d 100644
--- a/qiskit/providers/ibmq/runtime/runtime_job.py
+++ b/qiskit/providers/ibmq/runtime/runtime_job.py
@@ -12,7 +12,9 @@
 
 """IBM Quantum Experience Runtime job."""
 
-from typing import Any, Optional
+from typing import Any, Optional, Callable
+import queue
+from concurrent import futures
 
 from qiskit.providers.job import JobV1 as Job
 from qiskit.providers.jobstatus import JobStatus
@@ -22,11 +24,15 @@
 
 class RuntimeJob(Job):
 
+    _executor = futures.ThreadPoolExecutor()
+
     def __init__(
             self,
             backend: 'ibmqbackend.IBMQBackend',
             api_client: RuntimeClient,
             job_id: str,
+            interim_queue: Optional[queue.Queue] = None,
+            user_callback: Optional[Callable] = None
     ) -> None:
         """RuntimeJob constructor.
 
@@ -37,6 +43,24 @@ def __init__(
         """
         super().__init__(backend, job_id)
         self._api_client = api_client
+        self._result = None
+
+        self._user_callback = user_callback
+        self._interim_queue = interim_queue
+
+    def _interim_results(self):
+        while True:
+            try:
+                interim_result = self._interim_queue.get(block=True, timeout=5)
+                if interim_result == 'poison_pill':
+                    return
+                if 'post' in interim_result:
+                    self._user_callback(interim_result['post'])
+                elif 'results' in interim_result:
+                    self._result = interim_result['results']
+                    return
+            except queue.Empty:
+                pass
 
     def submit(self):
         """Unsupported method.
@@ -57,8 +81,14 @@ def result(
             timeout: Optional[float] = None
     ) -> Any:
         """Return the results of the job."""
+        if self._user_callback:
+            future = self._executor.submit(self._interim_results)
+            futures.wait([future])
         self.wait_for_final_state(timeout=timeout)
-        return self._api_client.program_job_results(program_id='123', job_id=self.job_id())
+        if not self._result:
+            self._result = self._api_client.program_job_results(
+                program_id='123', job_id=self.job_id())
+        return self._result
 
     def cancel(self):
         """Attempt to cancel the job."""
diff --git a/test/ibmq/test_runtime.py b/test/ibmq/test_runtime.py
index 167b258dd..7b93d5cc6 100644
--- a/test/ibmq/test_runtime.py
+++ b/test/ibmq/test_runtime.py
@@ -10,16 +10,9 @@
 # copyright notice, and modified files need to carry a notice indicating
 # that they have been altered from the originals.
 
-"""Tests for random number services."""
+"""Tests for runtime service."""
 
-import time
-import uuid
-from unittest import skipIf
-from concurrent.futures import ThreadPoolExecutor
-
-import numpy as np
 from qiskit.providers.jobstatus import JobStatus
-from qiskit.providers.ibmq.exceptions import IBMQError
 
 from ..ibmqtestcase import IBMQTestCase
 from ..decorators import requires_provider
@@ -49,3 +42,11 @@ def test_run_program(self):
         job.wait_for_final_state()
         self.assertEqual(job.status(), JobStatus.DONE)
         self.assertTrue(job.result())
+
+    def test_interim_results(self):
+        def _callback(interim_result):
+            print(f"interim result {interim_result}")
+        params = {'param1': 'foo'}
+        backend = self.provider.backend.ibmq_qasm_simulator
+        job = self.provider.runtime.run("QKA", backend=backend, params=params, callback=_callback)
+        job.result()

From d153155586f5339f0d8438ca27045af9f72f7984 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Wed, 17 Mar 2021 14:30:44 -0400
Subject: [PATCH 06/59] add runtime options

---
 .../ibmq/runtime/ibm_runtime_service.py       | 22 ++++++++++++++++---
 1 file changed, 19 insertions(+), 3 deletions(-)

diff --git a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
index 923c5b49a..8f21adf4b 100644
--- a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
+++ b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
@@ -18,10 +18,10 @@
 from concurrent import futures
 
 from qiskit.providers.ibmq import accountprovider  # pylint: disable=unused-import
+from qiskit import QiskitError
 
 from .runtime_job import RuntimeJob
 from .runtime_program import RuntimeProgram
-from ..ibmqbackend import IBMQBackend
 from ..api.clients.runtime import RuntimeClient
 
 logger = logging.getLogger(__name__)
@@ -72,15 +72,31 @@ def program(self, program_name: str):
     def run(
             self,
             program_name: str,
-            backend: IBMQBackend,
+            options: Dict,
             params: Dict,
             callback: Optional[Callable] = None
     ) -> RuntimeJob:
+        """Execute the runtime program.
+
+        Args:
+            program_name: Name of the program.
+            options: Runtime options. Currently the only available option is
+                ``backend_name``, which is required.
+            params: Program parameters.
+            callback: Callback function to be invoked for any interim results.
+
+        Returns:
+            A ``RuntimeJob`` instance representing the execution.
+        """
+        if 'backend_name' not in options:
+            raise QiskitError('"backend_name" is required field in "options"')
+        backend_name = options['backend_name']
         interim_queue = queue.Queue() if callback else None
         response = self._api_client.program_run(program_id=program_name,
                                                 credentials=self._provider.credentials,
-                                                backend_name=backend.name(),
+                                                backend_name=backend_name,
                                                 params=params, interim_queue=interim_queue)
+        backend = self._provider.get_backend(backend_name)
         job = RuntimeJob(backend=backend, api_client=self._api_client, job_id=response['id'],
                          interim_queue=interim_queue, user_callback=callback)
         return job

From 4a659e21e664d01fcf24f08ce498d39e62bead91 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Thu, 18 Mar 2021 16:41:19 -0400
Subject: [PATCH 07/59] add circuits to encoder

---
 qiskit/providers/ibmq/api/clients/runtime.py  |  2 +-
 qiskit/providers/ibmq/api/rest/runtime.py     |  6 +-
 .../ibmq/runtime/ibm_runtime_service.py       |  5 +-
 qiskit/providers/ibmq/runtime/utils.py        | 58 +++++++++++++++++++
 qiskit/providers/ibmq/utils/json_encoder.py   | 11 ----
 5 files changed, 65 insertions(+), 17 deletions(-)
 create mode 100644 qiskit/providers/ibmq/runtime/utils.py

diff --git a/qiskit/providers/ibmq/api/clients/runtime.py b/qiskit/providers/ibmq/api/clients/runtime.py
index dea6f8a14..96598904d 100644
--- a/qiskit/providers/ibmq/api/clients/runtime.py
+++ b/qiskit/providers/ibmq/api/clients/runtime.py
@@ -78,7 +78,7 @@ def program_run(
             program_id: str,
             credentials: Credentials,
             backend_name: str,
-            params: Dict,
+            params: str,
             interim_queue: Optional[queue.Queue] = None
     ) -> Dict:
         """Run the specified program.
diff --git a/qiskit/providers/ibmq/api/rest/runtime.py b/qiskit/providers/ibmq/api/rest/runtime.py
index 86abe6b82..967d36252 100644
--- a/qiskit/providers/ibmq/api/rest/runtime.py
+++ b/qiskit/providers/ibmq/api/rest/runtime.py
@@ -22,7 +22,6 @@
 import uuid
 import numpy as np
 
-from qiskit.providers.ibmq.utils import json_encoder
 from .base import RestAdapterBase
 from ..session import RetrySession
 
@@ -126,7 +125,7 @@ def run(
             group: str,
             project: str,
             backend_name: str,
-            params: Dict,
+            params: str,
             interim_queue: Optional[queue.Queue] = None
     ) -> Dict:
         """Execute the program.
@@ -153,9 +152,8 @@ def run(
         # temporary code
         python_bin = os.getenv('PYTHON_EXEC', 'python3')
         program_file = os.getenv('NTC_PROGRAM_FILE', 'runtime/qka_program.py')
-        data = json.dumps(params, cls=json_encoder.NumpyEncoder)
         global process
-        process = subprocess.Popen([python_bin, program_file, data],
+        process = subprocess.Popen([python_bin, program_file, params],
                                    stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                                    universal_newlines=True)
         if interim_queue:
diff --git a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
index 8f21adf4b..21e02bfd1 100644
--- a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
+++ b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
@@ -16,12 +16,14 @@
 from typing import Dict, Callable, Optional
 import queue
 from concurrent import futures
+import json
 
 from qiskit.providers.ibmq import accountprovider  # pylint: disable=unused-import
 from qiskit import QiskitError
 
 from .runtime_job import RuntimeJob
 from .runtime_program import RuntimeProgram
+from .utils import RuntimeEncoder
 from ..api.clients.runtime import RuntimeClient
 
 logger = logging.getLogger(__name__)
@@ -92,10 +94,11 @@ def run(
             raise QiskitError('"backend_name" is required field in "options"')
         backend_name = options['backend_name']
         interim_queue = queue.Queue() if callback else None
+        params_str = json.dumps(params, cls=RuntimeEncoder)
         response = self._api_client.program_run(program_id=program_name,
                                                 credentials=self._provider.credentials,
                                                 backend_name=backend_name,
-                                                params=params, interim_queue=interim_queue)
+                                                params=params_str, interim_queue=interim_queue)
         backend = self._provider.get_backend(backend_name)
         job = RuntimeJob(backend=backend, api_client=self._api_client, job_id=response['id'],
                          interim_queue=interim_queue, user_callback=callback)
diff --git a/qiskit/providers/ibmq/runtime/utils.py b/qiskit/providers/ibmq/runtime/utils.py
new file mode 100644
index 000000000..fc8b70b54
--- /dev/null
+++ b/qiskit/providers/ibmq/runtime/utils.py
@@ -0,0 +1,58 @@
+# This code is part of Qiskit.
+#
+# (C) Copyright IBM 2020.
+#
+# This code is licensed under the Apache License, Version 2.0. You may
+# obtain a copy of this license in the LICENSE.txt file in the root directory
+# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Any modifications or derivative works of this code must retain this
+# copyright notice, and modified files need to carry a notice indicating
+# that they have been altered from the originals.
+
+# pylint: disable=method-hidden
+
+"""Custom JSON encoders."""
+
+import json
+from typing import Any
+import numpy as np
+
+from qiskit.compiler import assemble
+from qiskit.assembler.disassemble import disassemble
+from qiskit.circuit.quantumcircuit import QuantumCircuit
+from qiskit.qobj import QasmQobj
+
+
+class RuntimeEncoder(json.JSONEncoder):
+    """JSON Encoder for Numpy arrays, complex numbers, and circuits."""
+
+    def default(self, obj: Any) -> Any:
+        if hasattr(obj, 'tolist'):
+            return {'type': 'array', 'value': obj.tolist()}
+        if isinstance(obj, complex):
+            return {'type': 'complex', 'value': [obj.real, obj.imag]}
+        if isinstance(obj, QuantumCircuit):
+            return {'type': 'circuits', 'value': assemble(obj).to_dict()}
+        return super().default(obj)
+
+
+class RuntimeDecoder(json.JSONDecoder):
+    """JSON Decoder for Numpy arrays, complex numbers, and circuits."""
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(object_hook=self.object_hook, *args, **kwargs)
+
+    def object_hook(self, obj):
+        if 'type' in obj:
+            if obj['type'] == 'complex':
+                val = obj['value']
+                return val[0] + 1j * val[1]
+            if obj['type'] == 'array':
+                return np.array(obj['value'])
+            if obj['type'] == 'circuits':
+                circuits, _, _ = disassemble(QasmQobj.from_dict(obj['value']))
+                if len(circuits) == 1:
+                    return circuits[0]
+                return circuits
+        return obj
diff --git a/qiskit/providers/ibmq/utils/json_encoder.py b/qiskit/providers/ibmq/utils/json_encoder.py
index 71f396ab1..bb926dd29 100644
--- a/qiskit/providers/ibmq/utils/json_encoder.py
+++ b/qiskit/providers/ibmq/utils/json_encoder.py
@@ -37,14 +37,3 @@ def default(self, o: Any) -> Any:
                 val = complex(o)
                 return val.real, val.imag
         return json.JSONEncoder.default(self, o)
-
-
-class NumpyEncoder(json.JSONEncoder):
-    """JSON Encoder for Numpy arrays and complex numbers."""
-
-    def default(self, obj: Any) -> Any:
-        if hasattr(obj, 'tolist'):
-            return {'type': 'array', 'value': obj.tolist()}
-        if isinstance(obj, complex):
-            return {'type': 'complex', 'value': [obj.real, obj.imag]}
-        return super().default(obj)

From f760eb7c30e54c750a1d0bd7d9c19546846717e2 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Thu, 18 Mar 2021 18:43:26 -0400
Subject: [PATCH 08/59] move encoder to utils

---
 qiskit/providers/ibmq/api/rest/runtime.py     | 23 ++++---------------
 .../ibmq/runtime/ibm_runtime_service.py       |  2 +-
 .../{runtime/utils.py => utils/runtime.py}    |  5 ++++
 3 files changed, 10 insertions(+), 20 deletions(-)
 rename qiskit/providers/ibmq/{runtime/utils.py => utils/runtime.py} (90%)

diff --git a/qiskit/providers/ibmq/api/rest/runtime.py b/qiskit/providers/ibmq/api/rest/runtime.py
index 967d36252..82ace40b3 100644
--- a/qiskit/providers/ibmq/api/rest/runtime.py
+++ b/qiskit/providers/ibmq/api/rest/runtime.py
@@ -20,7 +20,8 @@
 import queue
 from concurrent import futures
 import uuid
-import numpy as np
+
+from qiskit.providers.ibmq.utils.runtime import RuntimeDecoder
 
 from .base import RestAdapterBase
 from ..session import RetrySession
@@ -168,7 +169,7 @@ def _interim_result(self, interim_queue: queue.Queue, pgm_process):
             if nextline == '' and pgm_process.poll() is not None:
                 break
             try:
-                parsed = json.loads(nextline, cls=NumpyDecoder)
+                parsed = json.loads(nextline, cls=RuntimeDecoder)
                 if any(text in parsed for text in ['post', 'results']):
                     interim_queue.put_nowait(parsed)
             except:
@@ -250,7 +251,7 @@ def results(self) -> Dict:
             outs = outs.split('\n')
             for line in outs:
                 try:
-                    parsed = json.loads(line)
+                    parsed = json.loads(line, cls=RuntimeDecoder)
                     if 'results' in parsed:
                         return parsed['results']
                 except:
@@ -258,19 +259,3 @@ def results(self) -> Dict:
 
         return {}
         # return self.session.get(self.get_url('results')).json()
-
-
-class NumpyDecoder(json.JSONDecoder):
-    """JSON Decoder for Numpy arrays and complex numbers."""
-
-    def __init__(self, *args, **kwargs):
-        super().__init__(object_hook=self.object_hook, *args, **kwargs)
-
-    def object_hook(self, obj):
-        if 'type' in obj:
-            if obj['type'] == 'complex':
-                val = obj['value']
-                return val[0] + 1j * val[1]
-            if obj['type'] == 'array':
-                return np.array(obj['value'])
-        return obj
diff --git a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
index 21e02bfd1..546984f42 100644
--- a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
+++ b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
@@ -23,7 +23,7 @@
 
 from .runtime_job import RuntimeJob
 from .runtime_program import RuntimeProgram
-from .utils import RuntimeEncoder
+from ..utils.runtime import RuntimeEncoder
 from ..api.clients.runtime import RuntimeClient
 
 logger = logging.getLogger(__name__)
diff --git a/qiskit/providers/ibmq/runtime/utils.py b/qiskit/providers/ibmq/utils/runtime.py
similarity index 90%
rename from qiskit/providers/ibmq/runtime/utils.py
rename to qiskit/providers/ibmq/utils/runtime.py
index fc8b70b54..3b2fc1d76 100644
--- a/qiskit/providers/ibmq/runtime/utils.py
+++ b/qiskit/providers/ibmq/utils/runtime.py
@@ -22,6 +22,7 @@
 from qiskit.assembler.disassemble import disassemble
 from qiskit.circuit.quantumcircuit import QuantumCircuit
 from qiskit.qobj import QasmQobj
+from qiskit.result import Result
 
 
 class RuntimeEncoder(json.JSONEncoder):
@@ -34,6 +35,8 @@ def default(self, obj: Any) -> Any:
             return {'type': 'complex', 'value': [obj.real, obj.imag]}
         if isinstance(obj, QuantumCircuit):
             return {'type': 'circuits', 'value': assemble(obj).to_dict()}
+        if isinstance(obj, Result):
+            return {'type': 'result', 'value': obj.to_dict()}
         return super().default(obj)
 
 
@@ -55,4 +58,6 @@ def object_hook(self, obj):
                 if len(circuits) == 1:
                     return circuits[0]
                 return circuits
+            if obj['type'] == 'result':
+                return Result.from_dict(obj['value'])
         return obj

From c30f06ea627412dbacdf1e6fce243feb371d8dd6 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Thu, 18 Mar 2021 20:48:53 -0400
Subject: [PATCH 09/59] reformat doc

---
 qiskit/providers/ibmq/api/rest/runtime.py            | 2 +-
 qiskit/providers/ibmq/runtime/ibm_runtime_service.py | 1 +
 qiskit/providers/ibmq/runtime/runtime_program.py     | 6 +++---
 3 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/qiskit/providers/ibmq/api/rest/runtime.py b/qiskit/providers/ibmq/api/rest/runtime.py
index 82ace40b3..beb47b5f8 100644
--- a/qiskit/providers/ibmq/api/rest/runtime.py
+++ b/qiskit/providers/ibmq/api/rest/runtime.py
@@ -63,7 +63,7 @@ def list_programs(self) -> List[Dict]:
         doc_file = os.getenv('NTC_DOC_FILE', 'runtime/qka_doc.json')
         with open(doc_file, 'r') as file:
             data = json.load(file)
-        return [data]
+        return data
 
     def create_program(self, name: str, data: bytes) -> Dict:
         """Upload a new program.
diff --git a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
index 546984f42..9b8835149 100644
--- a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
+++ b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
@@ -61,6 +61,7 @@ def programs(self):
                                      **kwargs)
             self._programs[program.name] = program
         for prog in self._programs.values():
+            print("="*50)
             prog.pprint()
 
     def program(self, program_name: str):
diff --git a/qiskit/providers/ibmq/runtime/runtime_program.py b/qiskit/providers/ibmq/runtime/runtime_program.py
index 4d272edee..362e5fa96 100644
--- a/qiskit/providers/ibmq/runtime/runtime_program.py
+++ b/qiskit/providers/ibmq/runtime/runtime_program.py
@@ -56,13 +56,13 @@ def __init__(
                                                      return_type=ret['type']))
 
     def pprint(self):
-        formatted = [f"Runtime Program {self.name}:",
+        formatted = [f'"{self.name}":',
                      f"  Description: {self._description}",
                      f"  Parameters:"]
 
         if self._parameters:
             for param in self._parameters:
-                formatted.append(" "*4 + param.name + ":")
+                formatted.append(" "*4 + "- " + param.name + ":")
                 formatted.append(" "*6 + "description: " + param.description)
                 formatted.append(" "*6 + "type: " + param.type)
         else:
@@ -71,7 +71,7 @@ def pprint(self):
         formatted.append("  Returns:")
         if self._return_values:
             for ret in self._return_values:
-                formatted.append(" "*4 + ret.name + ":")
+                formatted.append(" "*4 + "- " + ret.name + ":")
                 formatted.append(" "*6 + "description: " + ret.description)
                 formatted.append(" "*6 + "type: " + ret.type)
         else:

From bfdf8c752c81fd8de46403da82a7b850c90a021f Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Fri, 19 Mar 2021 14:05:27 -0400
Subject: [PATCH 10/59] add whether parameter is required

---
 .../providers/ibmq/runtime/runtime_program.py   | 17 ++++++++++-------
 1 file changed, 10 insertions(+), 7 deletions(-)

diff --git a/qiskit/providers/ibmq/runtime/runtime_program.py b/qiskit/providers/ibmq/runtime/runtime_program.py
index 362e5fa96..8cf7900b4 100644
--- a/qiskit/providers/ibmq/runtime/runtime_program.py
+++ b/qiskit/providers/ibmq/runtime/runtime_program.py
@@ -49,22 +49,24 @@ def __init__(
             self._parameters.append(
                 ProgramParameter(name=param['name'],
                                  description=param['description'],
-                                 param_type=param['type']))
+                                 param_type=param['type'],
+                                 required=param['required']))
         for ret in return_values:
             self._return_values.append(ProgramReturn(name=ret['name'],
                                                      description=ret['description'],
                                                      return_type=ret['type']))
 
     def pprint(self):
-        formatted = [f'"{self.name}":',
+        formatted = [f'{self.name}:',
                      f"  Description: {self._description}",
                      f"  Parameters:"]
 
         if self._parameters:
             for param in self._parameters:
                 formatted.append(" "*4 + "- " + param.name + ":")
-                formatted.append(" "*6 + "description: " + param.description)
-                formatted.append(" "*6 + "type: " + param.type)
+                formatted.append(" "*6 + "Description: " + param.description)
+                formatted.append(" "*6 + "Type: " + param.type)
+                formatted.append(" "*6 + "Required: " + str(param.required))
         else:
             formatted.append(" "*4 + "none")
 
@@ -72,8 +74,8 @@ def pprint(self):
         if self._return_values:
             for ret in self._return_values:
                 formatted.append(" "*4 + "- " + ret.name + ":")
-                formatted.append(" "*6 + "description: " + ret.description)
-                formatted.append(" "*6 + "type: " + ret.type)
+                formatted.append(" "*6 + "Description: " + ret.description)
+                formatted.append(" "*6 + "Type: " + ret.type)
         else:
             formatted.append(" "*4 + "none")
         print('\n'.join(formatted))
@@ -81,7 +83,7 @@ def pprint(self):
 
 class ProgramParameter:
 
-    def __init__(self, name: str, description: str, param_type: str):
+    def __init__(self, name: str, description: str, param_type: str, required: bool):
         """
 
         Args:
@@ -91,6 +93,7 @@ def __init__(self, name: str, description: str, param_type: str):
         self.name = name
         self.description = description
         self.type = param_type
+        self.required = required
 
 
 class ProgramReturn:

From 8b55c3dbbe9fb6d780f0f3d3c1fe9959054218e9 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Tue, 23 Mar 2021 10:30:05 -0400
Subject: [PATCH 11/59] use real url

---
 qiskit/providers/ibmq/api/clients/runtime.py  |  19 +++-
 qiskit/providers/ibmq/api/rest/runtime.py     | 103 +++++-------------
 .../ibmq/runtime/ibm_runtime_service.py       |  41 +++++--
 .../providers/ibmq/runtime/runtime_program.py |  27 +++--
 4 files changed, 88 insertions(+), 102 deletions(-)

diff --git a/qiskit/providers/ibmq/api/clients/runtime.py b/qiskit/providers/ibmq/api/clients/runtime.py
index 96598904d..bdf649aba 100644
--- a/qiskit/providers/ibmq/api/clients/runtime.py
+++ b/qiskit/providers/ibmq/api/clients/runtime.py
@@ -10,10 +10,11 @@
 # copyright notice, and modified files need to carry a notice indicating
 # that they have been altered from the originals.
 
-"""Client for accessing Random Number Generator (RNG) services."""
+"""Client for accessing IBM Quantum runtime service."""
 
+import os
 import logging
-from typing import List, Dict, Optional
+from typing import List, Dict, Optional, Union
 import queue
 
 from qiskit.providers.ibmq.credentials import Credentials
@@ -38,7 +39,7 @@ def __init__(
             access_token: IBM Quantum Experience access token.
             credentials: Account credentials.
         """
-        url = ''
+        url = os.getenv("QE_RUNTIME_URL", "")
         self._session = RetrySession(url, access_token,
                                      **credentials.connection_parameters())
         self.api = Runtime(self._session)
@@ -51,6 +52,13 @@ def list_programs(self) -> List[Dict]:
         """
         return self.api.list_programs()
 
+    def program_create(
+            self,
+            program_name: str,
+            program_data: Union[bytes, str],
+    ):
+        return self.api.create_program(program_name=program_name, program_data=program_data)
+
     def program_get(self, program_id: str) -> Dict:
         """Return a specific program.
 
@@ -78,8 +86,7 @@ def program_run(
             program_id: str,
             credentials: Credentials,
             backend_name: str,
-            params: str,
-            interim_queue: Optional[queue.Queue] = None
+            params: str
     ) -> Dict:
         """Run the specified program.
 
@@ -94,7 +101,7 @@ def program_run(
         """
         return self.api.program(program_id).run(
             hub=credentials.hub, group=credentials.group, project=credentials.project,
-            backend_name=backend_name, params=params, interim_queue=interim_queue)
+            backend_name=backend_name, params=params)
 
     def program_delete(self, program_id: str):
         """Delete the specified program.
diff --git a/qiskit/providers/ibmq/api/rest/runtime.py b/qiskit/providers/ibmq/api/rest/runtime.py
index beb47b5f8..8c95c9473 100644
--- a/qiskit/providers/ibmq/api/rest/runtime.py
+++ b/qiskit/providers/ibmq/api/rest/runtime.py
@@ -13,15 +13,9 @@
 """Random REST adapter."""
 
 import logging
-from typing import Dict, List, Any, Optional
+from typing import Dict, List, Any, Optional, Union
 import json
-import subprocess
-import os
-import queue
 from concurrent import futures
-import uuid
-
-from qiskit.providers.ibmq.utils.runtime import RuntimeDecoder
 
 from .base import RestAdapterBase
 from ..session import RetrySession
@@ -58,27 +52,37 @@ def list_programs(self) -> List[Dict]:
             JSON response.
         """
         url = self.get_url('self')
-        # return self.session.get(url).json()
-        # temporary code
-        doc_file = os.getenv('NTC_DOC_FILE', 'runtime/qka_doc.json')
-        with open(doc_file, 'r') as file:
-            data = json.load(file)
-        return data
-
-    def create_program(self, name: str, data: bytes) -> Dict:
+        return self.session.get(url).json()
+
+    def create_program(
+            self,
+            program_name: str,
+            program_data: Union[bytes, str]
+    ) -> Dict:
         """Upload a new program.
 
         Args:
-            name: Name of the program.
-            data: Program data.
+            program_name: Name of the program.
+            program_data: Program data.
 
         Returns:
             JSON response.
         """
         url = self.get_url('self')
-        data = {'name': name,
-                'program': (name, data)}  # type: ignore[dict-item]
-        return self.session.post(url, files=data).json()
+        if isinstance(program_data, str):
+            with open(program_data, 'rb') as file:
+                data = {'name': (None, program_name),
+                        'program': (program_name, file)}  # type: ignore[dict-item]
+                response = self.session.post(url, files=data).json()
+        else:
+            data = {'name': (None, program_name),
+                    'program': (program_name, program_data)}  # type: ignore[dict-item]
+            response = self.session.post(url, files=data).json()
+        return response
+
+        # data = {'name': program_name,
+        #         'program': (program_name, program_data)}  # type: ignore[dict-item]
+        # return self.session.post(url, files=data).json()
 
 
 class Program(RestAdapterBase):
@@ -127,7 +131,6 @@ def run(
             project: str,
             backend_name: str,
             params: str,
-            interim_queue: Optional[queue.Queue] = None
     ) -> Dict:
         """Execute the program.
 
@@ -147,34 +150,10 @@ def run(
             'group': group,
             'project': project,
             'backend': backend_name,
-            'params': params
+            'params': [params]
         }
-        # data = json.dumps(payload, cls=json_encoder.IQXJsonEncoder)
-        # temporary code
-        python_bin = os.getenv('PYTHON_EXEC', 'python3')
-        program_file = os.getenv('NTC_PROGRAM_FILE', 'runtime/qka_program.py')
-        global process
-        process = subprocess.Popen([python_bin, program_file, params],
-                                   stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-                                   universal_newlines=True)
-        if interim_queue:
-            self._executor.submit(self._interim_result, interim_queue, process)
-
-        return {'id': uuid.uuid4().hex}
-        # return self.session.post(url, data=data).json()
-
-    def _interim_result(self, interim_queue: queue.Queue, pgm_process):
-        while True:
-            nextline = pgm_process.stdout.readline()
-            if nextline == '' and pgm_process.poll() is not None:
-                break
-            try:
-                parsed = json.loads(nextline, cls=RuntimeDecoder)
-                if any(text in parsed for text in ['post', 'results']):
-                    interim_queue.put_nowait(parsed)
-            except:
-                print(nextline)
-        interim_queue.put_nowait('poison_pill')
+        data = json.dumps(payload)
+        return self.session.post(url, data=data).json()
 
     def delete(self) -> Dict:
         """Delete this program.
@@ -218,18 +197,7 @@ def get(self) -> Dict:
         Returns:
             JSON response.
         """
-        output = {}
-        global process
-        if process is not None:
-            rc = process.poll()
-            if rc is None:
-                output['status'] = 'RUNNING'
-            elif rc < 0:
-                output['status'] = 'ERROR'
-            else:
-                output['status'] = 'DONE'
-        return output
-        # return self.session.get(self.get_url('self')).json()
+        return self.session.get(self.get_url('self')).json()
 
     def delete(self) -> Dict:
         """Delete program job.
@@ -245,17 +213,4 @@ def results(self) -> Dict:
         Returns:
             JSON response.
         """
-        global process
-        if process is not None:
-            outs, errs = process.communicate()
-            outs = outs.split('\n')
-            for line in outs:
-                try:
-                    parsed = json.loads(line, cls=RuntimeDecoder)
-                    if 'results' in parsed:
-                        return parsed['results']
-                except:
-                    print(line)
-
-        return {}
-        # return self.session.get(self.get_url('results')).json()
+        return self.session.get(self.get_url('results')).json()
diff --git a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
index 9b8835149..bb14b9eba 100644
--- a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
+++ b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
@@ -13,7 +13,7 @@
 """IBM Quantum runtime service."""
 
 import logging
-from typing import Dict, Callable, Optional
+from typing import Dict, Callable, Optional, Union
 import queue
 from concurrent import futures
 import json
@@ -55,9 +55,9 @@ def programs(self):
                 kwargs['data'] = prog_dict['data']
             program = RuntimeProgram(program_name=prog_dict['name'],
                                      program_id=prog_dict['id'],
-                                     description=prog_dict['description'],
-                                     parameters=prog_dict['parameters'],
-                                     return_values=prog_dict['return_values'],
+                                     description=prog_dict.get('description', ""),
+                                     parameters=prog_dict.get('parameters', None),
+                                     return_values=prog_dict.get('return_values', None),
                                      **kwargs)
             self._programs[program.name] = program
         for prog in self._programs.values():
@@ -74,7 +74,7 @@ def program(self, program_name: str):
 
     def run(
             self,
-            program_name: str,
+            program_id: str,
             options: Dict,
             params: Dict,
             callback: Optional[Callable] = None
@@ -82,7 +82,7 @@ def run(
         """Execute the runtime program.
 
         Args:
-            program_name: Name of the program.
+            program_id: Program ID.
             options: Runtime options. Currently the only available option is
                 ``backend_name``, which is required.
             params: Program parameters.
@@ -94,13 +94,34 @@ def run(
         if 'backend_name' not in options:
             raise QiskitError('"backend_name" is required field in "options"')
         backend_name = options['backend_name']
-        interim_queue = queue.Queue() if callback else None
+        # interim_queue = queue.Queue() if callback else None
         params_str = json.dumps(params, cls=RuntimeEncoder)
-        response = self._api_client.program_run(program_id=program_name,
+        response = self._api_client.program_run(program_id=program_id,
                                                 credentials=self._provider.credentials,
                                                 backend_name=backend_name,
-                                                params=params_str, interim_queue=interim_queue)
+                                                params=params_str)
         backend = self._provider.get_backend(backend_name)
         job = RuntimeJob(backend=backend, api_client=self._api_client, job_id=response['id'],
-                         interim_queue=interim_queue, user_callback=callback)
+                         user_callback=callback)
         return job
+
+    def upload(
+            self,
+            name: str,
+            data: Union[bytes, str],
+    ) -> str:
+        """Upload a runtime program.
+
+        Args:
+            name: Name of the program.
+            data: Name of the program file or program data to upload.
+
+        Returns:
+            Program ID.
+        """
+        response = self._api_client.program_create(name, data)
+        return response['id']
+
+    def job(self, program_id: str, job_id: str):
+        response = self._api_client.program_job_get(program_id, job_id)
+        print(f">>>>>> response is {response}")
diff --git a/qiskit/providers/ibmq/runtime/runtime_program.py b/qiskit/providers/ibmq/runtime/runtime_program.py
index 8cf7900b4..aa78d5c9c 100644
--- a/qiskit/providers/ibmq/runtime/runtime_program.py
+++ b/qiskit/providers/ibmq/runtime/runtime_program.py
@@ -25,8 +25,8 @@ def __init__(
             program_name: str,
             program_id: str,
             description: str,
-            parameters: List,
-            return_values: List,
+            parameters: Optional[List] = None,
+            return_values: Optional[List] = None,
             cost: float = 0,
             data: Optional[bytes] = None
     ) -> None:
@@ -45,19 +45,22 @@ def __init__(
         self._data = data
         self._parameters = []
         self._return_values = []
-        for param in parameters:
-            self._parameters.append(
-                ProgramParameter(name=param['name'],
-                                 description=param['description'],
-                                 param_type=param['type'],
-                                 required=param['required']))
-        for ret in return_values:
-            self._return_values.append(ProgramReturn(name=ret['name'],
-                                                     description=ret['description'],
-                                                     return_type=ret['type']))
+        if parameters is not None:
+            for param in parameters:
+                self._parameters.append(
+                    ProgramParameter(name=param['name'],
+                                     description=param['description'],
+                                     param_type=param['type'],
+                                     required=param['required']))
+        if return_values is not None:
+            for ret in return_values:
+                self._return_values.append(ProgramReturn(name=ret['name'],
+                                                         description=ret['description'],
+                                                         return_type=ret['type']))
 
     def pprint(self):
         formatted = [f'{self.name}:',
+                     f"  ID: {self._id}",
                      f"  Description: {self._description}",
                      f"  Parameters:"]
 

From 5247968ff8c96bd19e47e3a214c5681ef16dd84f Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Tue, 23 Mar 2021 18:52:39 -0400
Subject: [PATCH 12/59] get single program

---
 .../ibmq/runtime/ibm_runtime_service.py        | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
index bb14b9eba..1ef71a3b7 100644
--- a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
+++ b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
@@ -46,6 +46,14 @@ def __init__(self, provider: 'accountprovider.AccountProvider', access_token: st
         self._programs = {}
 
     def programs(self):
+        if not self._programs:
+            self._get_programs()
+
+        for prog in self._programs.values():
+            print("="*50)
+            prog.pprint()
+
+    def _get_programs(self):
         response = self._api_client.list_programs()
         for prog_dict in response:
             kwargs = {}
@@ -60,17 +68,15 @@ def programs(self):
                                      return_values=prog_dict.get('return_values', None),
                                      **kwargs)
             self._programs[program.name] = program
-        for prog in self._programs.values():
-            print("="*50)
-            prog.pprint()
 
     def program(self, program_name: str):
+        if not self._programs:
+            self._get_programs()
+
         if program_name in self._programs:
             self._programs[program_name].pprint()
         else:
-            program = RuntimeProgram(**self._api_client.program_get(program_name))
-            self._programs[program.name] = program
-            program.pprint()
+            raise ValueError(f"Program {program_name} is not found.")
 
     def run(
             self,

From 7d73603a7e160bd8565645a5d930c945faf6da09 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Wed, 24 Mar 2021 20:47:07 -0400
Subject: [PATCH 13/59] always use interim queue

---
 qiskit/providers/ibmq/runtime/runtime_job.py | 9 ++++-----
 1 file changed, 4 insertions(+), 5 deletions(-)

diff --git a/qiskit/providers/ibmq/runtime/runtime_job.py b/qiskit/providers/ibmq/runtime/runtime_job.py
index 09173a19d..3e5b95540 100644
--- a/qiskit/providers/ibmq/runtime/runtime_job.py
+++ b/qiskit/providers/ibmq/runtime/runtime_job.py
@@ -55,7 +55,8 @@ def _interim_results(self):
                 if interim_result == 'poison_pill':
                     return
                 if 'post' in interim_result:
-                    self._user_callback(interim_result['post'])
+                    if self._user_callback:
+                        self._user_callback(interim_result['post'])
                 elif 'results' in interim_result:
                     self._result = interim_result['results']
                     return
@@ -81,10 +82,8 @@ def result(
             timeout: Optional[float] = None
     ) -> Any:
         """Return the results of the job."""
-        if self._user_callback:
-            future = self._executor.submit(self._interim_results)
-            futures.wait([future])
-        self.wait_for_final_state(timeout=timeout)
+        future = self._executor.submit(self._interim_results)
+        futures.wait([future])
         if not self._result:
             self._result = self._api_client.program_job_results(
                 program_id='123', job_id=self.job_id())

From 14afe539fff1a5efa55846a9ce47600e40abec4c Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Thu, 25 Mar 2021 18:11:16 -0400
Subject: [PATCH 14/59] use dill

---
 qiskit/providers/ibmq/utils/runtime.py | 44 ++++++++++++--------------
 1 file changed, 21 insertions(+), 23 deletions(-)

diff --git a/qiskit/providers/ibmq/utils/runtime.py b/qiskit/providers/ibmq/utils/runtime.py
index 3b2fc1d76..267bc79ea 100644
--- a/qiskit/providers/ibmq/utils/runtime.py
+++ b/qiskit/providers/ibmq/utils/runtime.py
@@ -17,47 +17,45 @@
 import json
 from typing import Any
 import numpy as np
+import dill
+import base64
 
-from qiskit.compiler import assemble
-from qiskit.assembler.disassemble import disassemble
-from qiskit.circuit.quantumcircuit import QuantumCircuit
-from qiskit.qobj import QasmQobj
 from qiskit.result import Result
 
 
 class RuntimeEncoder(json.JSONEncoder):
-    """JSON Encoder for Numpy arrays, complex numbers, and circuits."""
+    """JSON Encoder used by runtime service."""
 
     def default(self, obj: Any) -> Any:
         if hasattr(obj, 'tolist'):
-            return {'type': 'array', 'value': obj.tolist()}
+            return {'__type__': 'array', '__value__': obj.tolist()}
         if isinstance(obj, complex):
-            return {'type': 'complex', 'value': [obj.real, obj.imag]}
-        if isinstance(obj, QuantumCircuit):
-            return {'type': 'circuits', 'value': assemble(obj).to_dict()}
+            return {'__type__': 'complex', '__value__': [obj.real, obj.imag]}
         if isinstance(obj, Result):
-            return {'type': 'result', 'value': obj.to_dict()}
+            return {'__type__': 'result', '__value__': obj.to_dict()}
+        if hasattr(obj, '__class__'):
+            encoded = base64.standard_b64encode(dill.dumps(obj))
+            return {'__type__': 'dill', '__value__': encoded.decode('utf-8')}
+
         return super().default(obj)
 
 
 class RuntimeDecoder(json.JSONDecoder):
-    """JSON Decoder for Numpy arrays, complex numbers, and circuits."""
+    """JSON Decoder used by runtime service."""
 
     def __init__(self, *args, **kwargs):
         super().__init__(object_hook=self.object_hook, *args, **kwargs)
 
     def object_hook(self, obj):
-        if 'type' in obj:
-            if obj['type'] == 'complex':
-                val = obj['value']
+        if '__type__' in obj:
+            if obj['__type__'] == 'complex':
+                val = obj['__value__']
                 return val[0] + 1j * val[1]
-            if obj['type'] == 'array':
-                return np.array(obj['value'])
-            if obj['type'] == 'circuits':
-                circuits, _, _ = disassemble(QasmQobj.from_dict(obj['value']))
-                if len(circuits) == 1:
-                    return circuits[0]
-                return circuits
-            if obj['type'] == 'result':
-                return Result.from_dict(obj['value'])
+            if obj['__type__'] == 'array':
+                return np.array(obj['__value__'])
+            if obj['__type__'] == 'result':
+                return Result.from_dict(obj['__value__'])
+            if obj['__type__'] == 'dill':
+                decoded = base64.standard_b64decode(obj['__value__'])
+                return dill.loads(decoded)
         return obj

From a4b24a1625c038ab77855ced66d0f54c9d4adb95 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Tue, 23 Mar 2021 10:30:05 -0400
Subject: [PATCH 15/59] use real url

---
 qiskit/providers/ibmq/runtime/ibm_runtime_service.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
index 1ef71a3b7..d124bc37a 100644
--- a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
+++ b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
@@ -76,7 +76,9 @@ def program(self, program_name: str):
         if program_name in self._programs:
             self._programs[program_name].pprint()
         else:
-            raise ValueError(f"Program {program_name} is not found.")
+            program = RuntimeProgram(**self._api_client.program_get(program_name))
+            self._programs[program.name] = program
+            program.pprint()
 
     def run(
             self,

From 6138f03e5d8a31f99dffb53bdce42f9bc14bd218 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Thu, 25 Mar 2021 21:56:22 -0400
Subject: [PATCH 16/59] fix urls

---
 qiskit/providers/ibmq/api/clients/runtime.py  |  28 +++--
 qiskit/providers/ibmq/api/rest/runtime.py     | 101 +++++++++--------
 qiskit/providers/ibmq/api/session.py          |   1 +
 qiskit/providers/ibmq/runtime/constants.py    |  24 +++++
 .../ibmq/runtime/ibm_runtime_service.py       | 102 +++++++++++-------
 qiskit/providers/ibmq/runtime/runtime_job.py  |  38 +++----
 .../providers/ibmq/runtime/runtime_program.py |  11 +-
 qiskit/providers/ibmq/utils/runtime.py        |  66 ++++++++----
 8 files changed, 226 insertions(+), 145 deletions(-)
 create mode 100644 qiskit/providers/ibmq/runtime/constants.py

diff --git a/qiskit/providers/ibmq/api/clients/runtime.py b/qiskit/providers/ibmq/api/clients/runtime.py
index bdf649aba..42f2d9784 100644
--- a/qiskit/providers/ibmq/api/clients/runtime.py
+++ b/qiskit/providers/ibmq/api/clients/runtime.py
@@ -14,8 +14,7 @@
 
 import os
 import logging
-from typing import List, Dict, Optional, Union
-import queue
+from typing import List, Dict, Union
 
 from qiskit.providers.ibmq.credentials import Credentials
 from qiskit.providers.ibmq.api.session import RetrySession
@@ -39,7 +38,7 @@ def __init__(
             access_token: IBM Quantum Experience access token.
             credentials: Account credentials.
         """
-        url = os.getenv("QE_RUNTIME_URL", "")
+        url = 'https://api-ntc.processing-prod-5dd5718798d097eccc65fac4e78a33ce-0000.us-east.containers.appdomain.cloud'
         self._session = RetrySession(url, access_token,
                                      **credentials.connection_parameters())
         self.api = Runtime(self._session)
@@ -99,25 +98,24 @@ def program_run(
         Returns:
             JSON response.
         """
-        return self.api.program(program_id).run(
-            hub=credentials.hub, group=credentials.group, project=credentials.project,
-            backend_name=backend_name, params=params)
+        return self.api.program_run(program_id=program_id, hub=credentials.hub,
+                                    group=credentials.group, project=credentials.project,
+                                    backend_name=backend_name, params=params)
 
-    def program_delete(self, program_id: str):
+    def program_delete(self, program_id: str) -> None:
         """Delete the specified program.
 
         Args:
             program_id: Program ID.
-
-        Returns:
-            JSON response.
         """
-        return self.api.program(program_id).delete()
+        self.api.program(program_id).delete()
 
-    def program_job_get(self, program_id, job_id):
-        return self.api.program_job(program_id, job_id).get()
+    def program_job_get(self, job_id):
+        response = self.api.program_job(job_id).get()
+        print(f">>>>>> response is {response}")
+        return response
 
-    def program_job_results(self, program_id, job_id: str) -> Dict:
+    def program_job_results(self, job_id: str) -> Dict:
         """Get the results of a program job.
 
         Args:
@@ -126,4 +124,4 @@ def program_job_results(self, program_id, job_id: str) -> Dict:
         Returns:
             JSON response.
         """
-        return self.api.program_job(program_id, job_id).results()
+        return self.api.program_job(job_id).results()
diff --git a/qiskit/providers/ibmq/api/rest/runtime.py b/qiskit/providers/ibmq/api/rest/runtime.py
index 8c95c9473..891e1cc65 100644
--- a/qiskit/providers/ibmq/api/rest/runtime.py
+++ b/qiskit/providers/ibmq/api/rest/runtime.py
@@ -13,7 +13,7 @@
 """Random REST adapter."""
 
 import logging
-from typing import Dict, List, Any, Optional, Union
+from typing import Dict, List, Any, Union
 import json
 from concurrent import futures
 
@@ -28,7 +28,8 @@ class Runtime(RestAdapterBase):
     """Rest adapter for RNG related endpoints."""
 
     URL_MAP = {
-        'self': '/programs'
+        'programs': '/programs',
+        'jobs': '/jobs'
     }
 
     def program(self, program_id: str) -> 'Program':
@@ -42,8 +43,8 @@ def program(self, program_id: str) -> 'Program':
         """
         return Program(self.session, program_id)
 
-    def program_job(self, program_id, job_id):
-        return ProgramJob(self.session, program_id, job_id)
+    def program_job(self, job_id):
+        return ProgramJob(self.session, job_id)
 
     def list_programs(self) -> List[Dict]:
         """Return a list of runtime programs.
@@ -51,7 +52,7 @@ def list_programs(self) -> List[Dict]:
         Returns:
             JSON response.
         """
-        url = self.get_url('self')
+        url = self.get_url('programs')
         return self.session.get(url).json()
 
     def create_program(
@@ -68,7 +69,7 @@ def create_program(
         Returns:
             JSON response.
         """
-        url = self.get_url('self')
+        url = self.get_url('programs')
         if isinstance(program_data, str):
             with open(program_data, 'rb') as file:
                 data = {'name': (None, program_name),
@@ -84,6 +85,40 @@ def create_program(
         #         'program': (program_name, program_data)}  # type: ignore[dict-item]
         # return self.session.post(url, files=data).json()
 
+    def program_run(
+            self,
+            program_id: str,
+            hub: str,
+            group: str,
+            project: str,
+            backend_name: str,
+            params: str,
+    ) -> Dict:
+        """Execute the program.
+
+        Args:
+            program_id: Program ID.
+            hub: Hub to be used.
+            group: Group to be used.
+            project: Project to be used.
+            backend_name: Name of the backend.
+            params: Program parameters.
+
+        Returns:
+            JSON response.
+        """
+        url = self.get_url('jobs')
+        payload = {
+            'programId': program_id,
+            'hub': hub,
+            'group': group,
+            'project': project,
+            'backend': backend_name,
+            'params': [params]
+        }
+        data = json.dumps(payload)
+        return self.session.post(url, data=data).json()
+
 
 class Program(RestAdapterBase):
     """Rest adapter for program related endpoints."""
@@ -124,45 +159,14 @@ def get_data(self) -> Dict[str, Any]:
         url = self.get_url('data')
         return self.session.get(url).json()
 
-    def run(
-            self,
-            hub: str,
-            group: str,
-            project: str,
-            backend_name: str,
-            params: str,
-    ) -> Dict:
-        """Execute the program.
-
-        Args:
-            hub: Hub to be used.
-            group: Group to be used.
-            project: Project to be used.
-            backend_name: Name of the backend.
-            params: Program parameters.
-
-        Returns:
-            JSON response.
-        """
-        url = self.get_url('run')
-        payload = {
-            'hub': hub,
-            'group': group,
-            'project': project,
-            'backend': backend_name,
-            'params': [params]
-        }
-        data = json.dumps(payload)
-        return self.session.post(url, data=data).json()
-
-    def delete(self) -> Dict:
+    def delete(self) -> None:
         """Delete this program.
 
         Returns:
             JSON response.
         """
         url = self.get_url('self')
-        return self.session.delete(url).json()
+        self.session.delete(url)
 
 
 class ProgramJob(RestAdapterBase):
@@ -170,13 +174,12 @@ class ProgramJob(RestAdapterBase):
 
     URL_MAP = {
         'self': '',
-        'results': 'results'
+        'results': '/results'
     }
 
     def __init__(
             self,
             session: RetrySession,
-            program_id: str,
             job_id: str,
             url_prefix: str = ''
     ) -> None:
@@ -184,12 +187,11 @@ def __init__(
 
         Args:
             session: Session to be used in the adapter.
-            program_id: ID of the runtime program.
             job_id: ID of the program job.
             url_prefix: Prefix to use in the URL.
         """
-        super().__init__(session, '{}/programs/{}/jobs/{}'.format(
-            url_prefix, program_id, job_id))
+        super().__init__(session, '{}/jobs/{}'.format(
+            url_prefix, job_id))
 
     def get(self) -> Dict:
         """Return program job information.
@@ -213,4 +215,13 @@ def results(self) -> Dict:
         Returns:
             JSON response.
         """
-        return self.session.get(self.get_url('results')).json()
+        r = self.session.get(self.get_url('results'))
+        try:
+            print(f">>>>> result json {r.json()}")
+        except:
+            print(f">>>>> no result json")
+        try:
+            print(f">>>>>> result text {r.text}")
+        except:
+            print(f">>>>>>> no result text")
+
diff --git a/qiskit/providers/ibmq/api/session.py b/qiskit/providers/ibmq/api/session.py
index 3a30b3b91..c6a90f757 100644
--- a/qiskit/providers/ibmq/api/session.py
+++ b/qiskit/providers/ibmq/api/session.py
@@ -268,6 +268,7 @@ def request(  # type: ignore[override]
 
         try:
             self._log_request_info(url, method, kwargs)
+            print(f">>>>>> final url is {final_url}")
             response = super().request(method, final_url, headers=headers, **kwargs)
             response.raise_for_status()
         except RequestException as ex:
diff --git a/qiskit/providers/ibmq/runtime/constants.py b/qiskit/providers/ibmq/runtime/constants.py
new file mode 100644
index 000000000..0e0c98f0d
--- /dev/null
+++ b/qiskit/providers/ibmq/runtime/constants.py
@@ -0,0 +1,24 @@
+# This code is part of Qiskit.
+#
+# (C) Copyright IBM 2021.
+#
+# This code is licensed under the Apache License, Version 2.0. You may
+# obtain a copy of this license in the LICENSE.txt file in the root directory
+# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Any modifications or derivative works of this code must retain this
+# copyright notice, and modified files need to carry a notice indicating
+# that they have been altered from the originals.
+
+"""Values used by the API for different values."""
+
+import enum
+
+
+class ApiRuntimeJobStatus(enum.Enum):
+    """Possible values used by the API for a runtime job status."""
+
+    PENDING = 'PENDING'
+    RUNNING = 'RUNNING'
+    ERROR = 'ERROR'
+    SUCCEEDED = 'SUCCEEDED'
diff --git a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
index d124bc37a..e4ae73575 100644
--- a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
+++ b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
@@ -14,8 +14,6 @@
 
 import logging
 from typing import Dict, Callable, Optional, Union
-import queue
-from concurrent import futures
 import json
 
 from qiskit.providers.ibmq import accountprovider  # pylint: disable=unused-import
@@ -32,8 +30,6 @@
 class IBMRuntimeService:
     """IBM Quantum runtime service."""
 
-    _executor = futures.ThreadPoolExecutor()
-
     def __init__(self, provider: 'accountprovider.AccountProvider', access_token: str) -> None:
         """IBMRuntimeService constructor.
 
@@ -45,39 +41,50 @@ def __init__(self, provider: 'accountprovider.AccountProvider', access_token: st
         self._api_client = RuntimeClient(access_token, provider.credentials)
         self._programs = {}
 
-    def programs(self):
-        if not self._programs:
-            self._get_programs()
+    def programs(self, refresh: bool = False):
+        """Return available runtime programs.
+
+        Args:
+            refresh: If ``True``, re-query the server for the programs. Otherwise
+                return the cached value.
+
+        Returns:
+
+        """
+        if not self._programs or refresh:
+            response = self._api_client.list_programs()
+            for prog_dict in response:
+                kwargs = {}
+                if 'cost' in prog_dict:
+                    kwargs['cost'] = prog_dict['cost']
+                if 'data' in prog_dict:
+                    kwargs['data'] = prog_dict['data']
+                program = RuntimeProgram(program_name=prog_dict['name'],
+                                         program_id=prog_dict['id'],
+                                         description=prog_dict.get('description', ""),
+                                         parameters=prog_dict.get('parameters', None),
+                                         return_values=prog_dict.get('return_values', None),
+                                         **kwargs)
+                self._programs[program.id] = program
 
         for prog in self._programs.values():
             print("="*50)
             prog.pprint()
 
-    def _get_programs(self):
-        response = self._api_client.list_programs()
-        for prog_dict in response:
-            kwargs = {}
-            if 'cost' in prog_dict:
-                kwargs['cost'] = prog_dict['cost']
-            if 'data' in prog_dict:
-                kwargs['data'] = prog_dict['data']
-            program = RuntimeProgram(program_name=prog_dict['name'],
-                                     program_id=prog_dict['id'],
-                                     description=prog_dict.get('description', ""),
-                                     parameters=prog_dict.get('parameters', None),
-                                     return_values=prog_dict.get('return_values', None),
-                                     **kwargs)
-            self._programs[program.name] = program
-
-    def program(self, program_name: str):
-        if not self._programs:
-            self._get_programs()
-
-        if program_name in self._programs:
-            self._programs[program_name].pprint()
+    def program(self, program_id: str):
+        """Retrieve a runtime program.
+
+        Args:
+            program_id: Program ID.
+
+        Returns:
+            Runtime program.
+        """
+        if program_id in self._programs:
+            self._programs[program_id].pprint()
         else:
-            program = RuntimeProgram(**self._api_client.program_get(program_name))
-            self._programs[program.name] = program
+            program = RuntimeProgram(**self._api_client.program_get(program_id))
+            self._programs[program.id] = program
             program.pprint()
 
     def run(
@@ -101,19 +108,20 @@ def run(
         """
         if 'backend_name' not in options:
             raise QiskitError('"backend_name" is required field in "options"')
+
         backend_name = options['backend_name']
-        # interim_queue = queue.Queue() if callback else None
         params_str = json.dumps(params, cls=RuntimeEncoder)
         response = self._api_client.program_run(program_id=program_id,
                                                 credentials=self._provider.credentials,
                                                 backend_name=backend_name,
                                                 params=params_str)
+
         backend = self._provider.get_backend(backend_name)
-        job = RuntimeJob(backend=backend, api_client=self._api_client, job_id=response['id'],
-                         user_callback=callback)
+        job = RuntimeJob(backend=backend, api_client=self._api_client,
+                         job_id=response['id'], params=params)
         return job
 
-    def upload(
+    def upload_program(
             self,
             name: str,
             data: Union[bytes, str],
@@ -130,6 +138,24 @@ def upload(
         response = self._api_client.program_create(name, data)
         return response['id']
 
-    def job(self, program_id: str, job_id: str):
-        response = self._api_client.program_job_get(program_id, job_id)
-        print(f">>>>>> response is {response}")
+    def delete_program(self, program_id: str):
+        """Delete a runtime program.
+
+        Args:
+            program_id: Program ID.
+        """
+        self._api_client.program_delete(program_id=program_id)
+
+    def job(self, job_id: str):
+        """Retrieve a runtime job.
+
+        Args:
+            job_id: Job ID.
+
+        Returns:
+            Runtime job retrieved.
+        """
+        response = self._api_client.program_job_get(job_id)
+        backend = self._provider.get_backend(response['backend'])
+        return RuntimeJob(backend=backend, api_client=self._api_client, job_id=response['id'],
+                          params=response.get('params', {}))
diff --git a/qiskit/providers/ibmq/runtime/runtime_job.py b/qiskit/providers/ibmq/runtime/runtime_job.py
index 3e5b95540..8a1a1eb49 100644
--- a/qiskit/providers/ibmq/runtime/runtime_job.py
+++ b/qiskit/providers/ibmq/runtime/runtime_job.py
@@ -12,7 +12,7 @@
 
 """IBM Quantum Experience Runtime job."""
 
-from typing import Any, Optional, Callable
+from typing import Any, Optional, Callable, Dict
 import queue
 from concurrent import futures
 
@@ -31,7 +31,7 @@ def __init__(
             backend: 'ibmqbackend.IBMQBackend',
             api_client: RuntimeClient,
             job_id: str,
-            interim_queue: Optional[queue.Queue] = None,
+            params: Dict,
             user_callback: Optional[Callable] = None
     ) -> None:
         """RuntimeJob constructor.
@@ -44,24 +44,9 @@ def __init__(
         super().__init__(backend, job_id)
         self._api_client = api_client
         self._result = None
+        self._params = params
 
         self._user_callback = user_callback
-        self._interim_queue = interim_queue
-
-    def _interim_results(self):
-        while True:
-            try:
-                interim_result = self._interim_queue.get(block=True, timeout=5)
-                if interim_result == 'poison_pill':
-                    return
-                if 'post' in interim_result:
-                    if self._user_callback:
-                        self._user_callback(interim_result['post'])
-                elif 'results' in interim_result:
-                    self._result = interim_result['results']
-                    return
-            except queue.Empty:
-                pass
 
     def submit(self):
         """Unsupported method.
@@ -82,11 +67,8 @@ def result(
             timeout: Optional[float] = None
     ) -> Any:
         """Return the results of the job."""
-        future = self._executor.submit(self._interim_results)
-        futures.wait([future])
         if not self._result:
-            self._result = self._api_client.program_job_results(
-                program_id='123', job_id=self.job_id())
+            self._result = self._api_client.program_job_results(job_id=self.job_id())
         return self._result
 
     def cancel(self):
@@ -95,11 +77,17 @@ def cancel(self):
 
     def status(self) -> JobStatus:
         """Return the status of the job."""
-        response = self._api_client.program_job_get(program_id='123', job_id=self.job_id())
-        status = response['status']
+        response = self._api_client.program_job_get(job_id=self.job_id())
+        status = response['status'].upper()
         if status == 'RUNNING':
             return JobStatus.RUNNING
-        elif status == 'DONE':
+        elif status == 'SUCCEEDED':
             return JobStatus.DONE
+        elif status == 'PENDING':
+            return JobStatus.INITIALIZING
         else:
             return JobStatus.ERROR
+
+    @property
+    def parameters(self) -> Dict:
+        return self._params
diff --git a/qiskit/providers/ibmq/runtime/runtime_program.py b/qiskit/providers/ibmq/runtime/runtime_program.py
index aa78d5c9c..68a7842cc 100644
--- a/qiskit/providers/ibmq/runtime/runtime_program.py
+++ b/qiskit/providers/ibmq/runtime/runtime_program.py
@@ -11,8 +11,7 @@
 # that they have been altered from the originals.
 
 import logging
-from datetime import datetime
-from typing import Dict, Optional, List, Union
+from typing import Optional, List
 
 logger = logging.getLogger(__name__)
 
@@ -83,6 +82,14 @@ def pprint(self):
             formatted.append(" "*4 + "none")
         print('\n'.join(formatted))
 
+    @property
+    def id(self):
+        return self._id
+
+    @id.setter
+    def id(self, value):
+        pass
+
 
 class ProgramParameter:
 
diff --git a/qiskit/providers/ibmq/utils/runtime.py b/qiskit/providers/ibmq/utils/runtime.py
index 267bc79ea..696f4512a 100644
--- a/qiskit/providers/ibmq/utils/runtime.py
+++ b/qiskit/providers/ibmq/utils/runtime.py
@@ -24,38 +24,64 @@
 
 
 class RuntimeEncoder(json.JSONEncoder):
-    """JSON Encoder used by runtime service."""
+    """JSON Encoder for Numpy arrays and complex numbers."""
 
     def default(self, obj: Any) -> Any:
         if hasattr(obj, 'tolist'):
-            return {'__type__': 'array', '__value__': obj.tolist()}
+            return {'type': 'array', 'value': obj.tolist()}
         if isinstance(obj, complex):
-            return {'__type__': 'complex', '__value__': [obj.real, obj.imag]}
-        if isinstance(obj, Result):
-            return {'__type__': 'result', '__value__': obj.to_dict()}
-        if hasattr(obj, '__class__'):
-            encoded = base64.standard_b64encode(dill.dumps(obj))
-            return {'__type__': 'dill', '__value__': encoded.decode('utf-8')}
-
+            return {'type': 'complex', 'value': [obj.real, obj.imag]}
         return super().default(obj)
 
 
 class RuntimeDecoder(json.JSONDecoder):
-    """JSON Decoder used by runtime service."""
+    """JSON Decoder for Numpy arrays and complex numbers."""
 
     def __init__(self, *args, **kwargs):
         super().__init__(object_hook=self.object_hook, *args, **kwargs)
 
     def object_hook(self, obj):
-        if '__type__' in obj:
-            if obj['__type__'] == 'complex':
-                val = obj['__value__']
+        if 'type' in obj:
+            if obj['type'] == 'complex':
+                val = obj['value']
                 return val[0] + 1j * val[1]
-            if obj['__type__'] == 'array':
-                return np.array(obj['__value__'])
-            if obj['__type__'] == 'result':
-                return Result.from_dict(obj['__value__'])
-            if obj['__type__'] == 'dill':
-                decoded = base64.standard_b64decode(obj['__value__'])
-                return dill.loads(decoded)
+            if obj['type'] == 'array':
+                return np.array(obj['value'])
         return obj
+
+# class RuntimeEncoder(json.JSONEncoder):
+#     """JSON Encoder used by runtime service."""
+#
+#     def default(self, obj: Any) -> Any:
+#         if hasattr(obj, 'tolist'):
+#             return {'__type__': 'array', '__value__': obj.tolist()}
+#         if isinstance(obj, complex):
+#             return {'__type__': 'complex', '__value__': [obj.real, obj.imag]}
+#         if isinstance(obj, Result):
+#             return {'__type__': 'result', '__value__': obj.to_dict()}
+#         if hasattr(obj, '__class__'):
+#             encoded = base64.standard_b64encode(dill.dumps(obj))
+#             return {'__type__': 'dill', '__value__': encoded.decode('utf-8')}
+#
+#         return super().default(obj)
+#
+#
+# class RuntimeDecoder(json.JSONDecoder):
+#     """JSON Decoder used by runtime service."""
+#
+#     def __init__(self, *args, **kwargs):
+#         super().__init__(object_hook=self.object_hook, *args, **kwargs)
+#
+#     def object_hook(self, obj):
+#         if '__type__' in obj:
+#             if obj['__type__'] == 'complex':
+#                 val = obj['__value__']
+#                 return val[0] + 1j * val[1]
+#             if obj['__type__'] == 'array':
+#                 return np.array(obj['__value__'])
+#             if obj['__type__'] == 'result':
+#                 return Result.from_dict(obj['__value__'])
+#             if obj['__type__'] == 'dill':
+#                 decoded = base64.standard_b64decode(obj['__value__'])
+#                 return dill.loads(decoded)
+#         return obj

From 3e860a4deda4e9606a1811974107491a8799b093 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Fri, 26 Mar 2021 09:40:47 -0400
Subject: [PATCH 17/59] add logging

---
 qiskit/providers/ibmq/api/clients/runtime.py  |   5 +-
 qiskit/providers/ibmq/api/rest/runtime.py     |  18 +--
 qiskit/providers/ibmq/api/session.py          |   1 -
 qiskit/providers/ibmq/runtime/constants.py    |   8 +-
 .../ibmq/runtime/ibm_runtime_service.py       |  72 ++++++----
 qiskit/providers/ibmq/runtime/runtime_job.py  | 133 +++++++++++++-----
 .../providers/ibmq/runtime/runtime_program.py |  22 ++-
 7 files changed, 162 insertions(+), 97 deletions(-)

diff --git a/qiskit/providers/ibmq/api/clients/runtime.py b/qiskit/providers/ibmq/api/clients/runtime.py
index 42f2d9784..cbba1bc17 100644
--- a/qiskit/providers/ibmq/api/clients/runtime.py
+++ b/qiskit/providers/ibmq/api/clients/runtime.py
@@ -38,7 +38,8 @@ def __init__(
             access_token: IBM Quantum Experience access token.
             credentials: Account credentials.
         """
-        url = 'https://api-ntc.processing-prod-5dd5718798d097eccc65fac4e78a33ce-0000.us-east.containers.appdomain.cloud'
+        url = os.getenv('NTC_URL', 'https://api-ntc.processing-prod-5dd5718798d097eccc65fac4e78a33ce-0000.us-east.containers.appdomain.cloud')
+        logger.debug(f"Using runtime service url {url}")
         self._session = RetrySession(url, access_token,
                                      **credentials.connection_parameters())
         self.api = Runtime(self._session)
@@ -112,7 +113,7 @@ def program_delete(self, program_id: str) -> None:
 
     def program_job_get(self, job_id):
         response = self.api.program_job(job_id).get()
-        print(f">>>>>> response is {response}")
+        logger.debug(f"Runtime job get response: {response}")
         return response
 
     def program_job_results(self, job_id: str) -> Dict:
diff --git a/qiskit/providers/ibmq/api/rest/runtime.py b/qiskit/providers/ibmq/api/rest/runtime.py
index 891e1cc65..fc07931ab 100644
--- a/qiskit/providers/ibmq/api/rest/runtime.py
+++ b/qiskit/providers/ibmq/api/rest/runtime.py
@@ -81,10 +81,6 @@ def create_program(
             response = self.session.post(url, files=data).json()
         return response
 
-        # data = {'name': program_name,
-        #         'program': (program_name, program_data)}  # type: ignore[dict-item]
-        # return self.session.post(url, files=data).json()
-
     def program_run(
             self,
             program_id: str,
@@ -209,19 +205,11 @@ def delete(self) -> Dict:
         """
         return self.session.delete(self.get_url('self')).json()
 
-    def results(self) -> Dict:
+    def results(self) -> str:
         """Return program job results.
 
         Returns:
             JSON response.
         """
-        r = self.session.get(self.get_url('results'))
-        try:
-            print(f">>>>> result json {r.json()}")
-        except:
-            print(f">>>>> no result json")
-        try:
-            print(f">>>>>> result text {r.text}")
-        except:
-            print(f">>>>>>> no result text")
-
+        response = self.session.get(self.get_url('results'))
+        return response.text
diff --git a/qiskit/providers/ibmq/api/session.py b/qiskit/providers/ibmq/api/session.py
index c6a90f757..3a30b3b91 100644
--- a/qiskit/providers/ibmq/api/session.py
+++ b/qiskit/providers/ibmq/api/session.py
@@ -268,7 +268,6 @@ def request(  # type: ignore[override]
 
         try:
             self._log_request_info(url, method, kwargs)
-            print(f">>>>>> final url is {final_url}")
             response = super().request(method, final_url, headers=headers, **kwargs)
             response.raise_for_status()
         except RequestException as ex:
diff --git a/qiskit/providers/ibmq/runtime/constants.py b/qiskit/providers/ibmq/runtime/constants.py
index 0e0c98f0d..2087c3db8 100644
--- a/qiskit/providers/ibmq/runtime/constants.py
+++ b/qiskit/providers/ibmq/runtime/constants.py
@@ -10,7 +10,7 @@
 # copyright notice, and modified files need to carry a notice indicating
 # that they have been altered from the originals.
 
-"""Values used by the API for different values."""
+"""Values used by the runtime API for different values."""
 
 import enum
 
@@ -22,3 +22,9 @@ class ApiRuntimeJobStatus(enum.Enum):
     RUNNING = 'RUNNING'
     ERROR = 'ERROR'
     SUCCEEDED = 'SUCCEEDED'
+
+
+JOB_FINAL_STATES = (
+    "ERROR",
+    "SUCCEEDED"
+)
diff --git a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
index e4ae73575..87b99b414 100644
--- a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
+++ b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
@@ -13,7 +13,7 @@
 """IBM Quantum runtime service."""
 
 import logging
-from typing import Dict, Callable, Optional, Union
+from typing import Dict, Callable, Optional, Union, List
 import json
 
 from qiskit.providers.ibmq import accountprovider  # pylint: disable=unused-import
@@ -41,7 +41,19 @@ def __init__(self, provider: 'accountprovider.AccountProvider', access_token: st
         self._api_client = RuntimeClient(access_token, provider.credentials)
         self._programs = {}
 
-    def programs(self, refresh: bool = False):
+    def print_programs(self, refresh: bool = False) -> None:
+        """Print information about available runtime programs.
+
+        Args:
+            refresh: If ``True``, re-query the server for the programs. Otherwise
+                return the cached value.
+        """
+        programs = self.programs(refresh)
+        for prog in programs:
+            print("="*50)
+            print(str(prog))
+
+    def programs(self, refresh: bool = False) -> List[RuntimeProgram]:
         """Return available runtime programs.
 
         Args:
@@ -49,43 +61,47 @@ def programs(self, refresh: bool = False):
                 return the cached value.
 
         Returns:
-
+            A list of runtime programs.
         """
         if not self._programs or refresh:
             response = self._api_client.list_programs()
             for prog_dict in response:
-                kwargs = {}
-                if 'cost' in prog_dict:
-                    kwargs['cost'] = prog_dict['cost']
-                if 'data' in prog_dict:
-                    kwargs['data'] = prog_dict['data']
-                program = RuntimeProgram(program_name=prog_dict['name'],
-                                         program_id=prog_dict['id'],
-                                         description=prog_dict.get('description', ""),
-                                         parameters=prog_dict.get('parameters', None),
-                                         return_values=prog_dict.get('return_values', None),
-                                         **kwargs)
-                self._programs[program.id] = program
-
-        for prog in self._programs.values():
-            print("="*50)
-            prog.pprint()
+                program = self._to_program(prog_dict)
+                self._programs[program.program_id] = program
+        return list(self._programs.values())
 
-    def program(self, program_id: str):
+    def program(self, program_id: str, refresh: bool = False) -> RuntimeProgram:
         """Retrieve a runtime program.
 
         Args:
             program_id: Program ID.
+            refresh: If ``True``, re-query the server for the program. Otherwise
+                return the cached value.
 
         Returns:
             Runtime program.
         """
-        if program_id in self._programs:
-            self._programs[program_id].pprint()
-        else:
-            program = RuntimeProgram(**self._api_client.program_get(program_id))
-            self._programs[program.id] = program
-            program.pprint()
+        if program_id not in self._programs or refresh:
+            response = self._api_client.program_get(program_id)
+            self._programs[program_id] = self._to_program(response)
+
+        return self._programs[program_id]
+
+    def _to_program(self, response: Dict) -> RuntimeProgram:
+        """Convert server response to ``RuntimeProgram`` instances.
+
+        Args:
+            response: Server response.
+
+        Returns:
+            A ``RuntimeProgram`` instance.
+        """
+        return RuntimeProgram(program_name=response['name'],
+                              program_id=response['id'],
+                              description=response.get('description', ""),
+                              parameters=response.get('parameters', None),
+                              return_values=response.get('return_values', None),
+                              max_execution_time=response.get('cost', 0))
 
     def run(
             self,
@@ -118,7 +134,8 @@ def run(
 
         backend = self._provider.get_backend(backend_name)
         job = RuntimeJob(backend=backend, api_client=self._api_client,
-                         job_id=response['id'], params=params)
+                         job_id=response['id'], program_id=program_id, params=params,
+                         user_callback=callback)
         return job
 
     def upload_program(
@@ -158,4 +175,5 @@ def job(self, job_id: str):
         response = self._api_client.program_job_get(job_id)
         backend = self._provider.get_backend(response['backend'])
         return RuntimeJob(backend=backend, api_client=self._api_client, job_id=response['id'],
+                          program_id=response.get('program', ""),
                           params=response.get('params', {}))
diff --git a/qiskit/providers/ibmq/runtime/runtime_job.py b/qiskit/providers/ibmq/runtime/runtime_job.py
index 8a1a1eb49..98ded603e 100644
--- a/qiskit/providers/ibmq/runtime/runtime_job.py
+++ b/qiskit/providers/ibmq/runtime/runtime_job.py
@@ -13,25 +13,27 @@
 """IBM Quantum Experience Runtime job."""
 
 from typing import Any, Optional, Callable, Dict
-import queue
-from concurrent import futures
+import time
+import logging
 
-from qiskit.providers.job import JobV1 as Job
-from qiskit.providers.jobstatus import JobStatus
+from qiskit.providers.exceptions import JobTimeoutError
+from qiskit.providers.backend import Backend
 
 from ..api.clients import RuntimeClient
+from .constants import JOB_FINAL_STATES
 
+logger = logging.getLogger(__name__)
 
-class RuntimeJob(Job):
 
-    _executor = futures.ThreadPoolExecutor()
+class RuntimeJob:
 
     def __init__(
             self,
             backend: 'ibmqbackend.IBMQBackend',
             api_client: RuntimeClient,
             job_id: str,
-            params: Dict,
+            program_id: str,
+            params: Optional[Dict] = None,
             user_callback: Optional[Callable] = None
     ) -> None:
         """RuntimeJob constructor.
@@ -40,34 +42,35 @@ def __init__(
             backend: The backend instance used to run this job.
             api_client: Object for connecting to the server.
             job_id: Job ID.
+            program_id: ID of the program this job is for.
+            params: Job parameters.
+            user_callback: User callback function.
         """
-        super().__init__(backend, job_id)
+        self._job_id = job_id
+        self._backend = backend
         self._api_client = api_client
         self._result = None
-        self._params = params
-
+        self._params = params or {}
         self._user_callback = user_callback
-
-    def submit(self):
-        """Unsupported method.
-
-        Note:
-            This method is not supported, please use
-            :meth:`~qiskit.providers.ibmq.ibmqbackend.IBMQBackend.run`
-            to submit a job.
-
-        Raises:
-            NotImplementedError: Upon invocation.
-        """
-        raise NotImplementedError("submit() is not supported. Please use "
-                                  "IBMRuntimeService.run() to submit a runtime job.")
+        self._program_id = program_id
+        self._status = 'PENDING'
 
     def result(
             self,
-            timeout: Optional[float] = None
+            timeout: Optional[float] = None,
+            wait: float = 5
     ) -> Any:
-        """Return the results of the job."""
+        """Return the results of the job.
+
+        Args:
+            timeout: Number of seconds to wait for job.
+            wait: Seconds between queries.
+
+        Returns:
+            Runtime job result.
+        """
         if not self._result:
+            self.wait_for_final_state(timeout=timeout, wait=wait)
             self._result = self._api_client.program_job_results(job_id=self.job_id())
         return self._result
 
@@ -75,19 +78,73 @@ def cancel(self):
         """Attempt to cancel the job."""
         raise NotImplementedError
 
-    def status(self) -> JobStatus:
-        """Return the status of the job."""
-        response = self._api_client.program_job_get(job_id=self.job_id())
-        status = response['status'].upper()
-        if status == 'RUNNING':
-            return JobStatus.RUNNING
-        elif status == 'SUCCEEDED':
-            return JobStatus.DONE
-        elif status == 'PENDING':
-            return JobStatus.INITIALIZING
-        else:
-            return JobStatus.ERROR
+    def status(self) -> str:
+        """Return the status of the job.
+
+        Returns:
+            Status of this job.
+        """
+        if self._status not in JOB_FINAL_STATES:
+            response = self._api_client.program_job_get(job_id=self.job_id())
+            self._status = response['status'].upper()
+        return self._status
+
+    def wait_for_final_state(
+            self,
+            timeout: Optional[float] = None,
+            wait: float = 5
+    ) -> None:
+        """Poll the job status until it progresses to a final state such as ``DONE`` or ``ERROR``.
+
+        Args:
+            timeout: Seconds to wait for the job. If ``None``, wait indefinitely.
+            wait: Seconds between queries.
+
+        Raises:
+            JobTimeoutError: If the job does not reach a final state before the
+                specified timeout.
+        """
+        start_time = time.time()
+        status = self.status()
+        while status not in JOB_FINAL_STATES:
+            elapsed_time = time.time() - start_time
+            if timeout is not None and elapsed_time >= timeout:
+                raise JobTimeoutError(
+                    'Timeout while waiting for job {}.'.format(self.job_id()))
+            time.sleep(wait)
+            status = self.status()
+        return
+
+    def job_id(self) -> str:
+        """Return a unique id identifying the job.
+
+        Returns:
+            Job ID.
+        """
+        return self._job_id
+
+    def backend(self) -> Backend:
+        """Return the backend where this job was executed.
+
+        Returns:
+            Backend used for the job.
+        """
+        return self._backend
 
     @property
     def parameters(self) -> Dict:
+        """Job parameters.
+
+        Returns:
+            Parameters used in this job.
+        """
         return self._params
+
+    @property
+    def program_id(self) -> str:
+        """Returns program ID.
+
+        Returns:
+            ID of the program this job is for.
+        """
+        return self._program_id
diff --git a/qiskit/providers/ibmq/runtime/runtime_program.py b/qiskit/providers/ibmq/runtime/runtime_program.py
index 68a7842cc..2807d839d 100644
--- a/qiskit/providers/ibmq/runtime/runtime_program.py
+++ b/qiskit/providers/ibmq/runtime/runtime_program.py
@@ -26,21 +26,21 @@ def __init__(
             description: str,
             parameters: Optional[List] = None,
             return_values: Optional[List] = None,
-            cost: float = 0,
+            max_execution_time: float = 0,
             data: Optional[bytes] = None
     ) -> None:
         """
 
         Args:
-            program_name:
-            program_id:
-            description:
-            cost:
+            program_name: Program name.
+            program_id: Program ID.
+            description: Program description.
+            max_execution_time: Maximum execution time.
         """
         self.name = program_name
         self._id = program_id
         self._description = description
-        self._cost = cost
+        self._cost = max_execution_time
         self._data = data
         self._parameters = []
         self._return_values = []
@@ -57,7 +57,7 @@ def __init__(
                                                          description=ret['description'],
                                                          return_type=ret['type']))
 
-    def pprint(self):
+    def __str__(self) -> str:
         formatted = [f'{self.name}:',
                      f"  ID: {self._id}",
                      f"  Description: {self._description}",
@@ -80,16 +80,12 @@ def pprint(self):
                 formatted.append(" "*6 + "Type: " + ret.type)
         else:
             formatted.append(" "*4 + "none")
-        print('\n'.join(formatted))
+        return '\n'.join(formatted)
 
     @property
-    def id(self):
+    def program_id(self):
         return self._id
 
-    @id.setter
-    def id(self, value):
-        pass
-
 
 class ProgramParameter:
 

From 06bb63fb3ac653f609c739eaa2b2a0c77b565806 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Fri, 26 Mar 2021 10:30:00 -0400
Subject: [PATCH 18/59] use real coders

---
 qiskit/providers/ibmq/utils/runtime.py | 66 ++++++++------------------
 requirements.txt                       |  1 +
 2 files changed, 21 insertions(+), 46 deletions(-)

diff --git a/qiskit/providers/ibmq/utils/runtime.py b/qiskit/providers/ibmq/utils/runtime.py
index 696f4512a..267bc79ea 100644
--- a/qiskit/providers/ibmq/utils/runtime.py
+++ b/qiskit/providers/ibmq/utils/runtime.py
@@ -24,64 +24,38 @@
 
 
 class RuntimeEncoder(json.JSONEncoder):
-    """JSON Encoder for Numpy arrays and complex numbers."""
+    """JSON Encoder used by runtime service."""
 
     def default(self, obj: Any) -> Any:
         if hasattr(obj, 'tolist'):
-            return {'type': 'array', 'value': obj.tolist()}
+            return {'__type__': 'array', '__value__': obj.tolist()}
         if isinstance(obj, complex):
-            return {'type': 'complex', 'value': [obj.real, obj.imag]}
+            return {'__type__': 'complex', '__value__': [obj.real, obj.imag]}
+        if isinstance(obj, Result):
+            return {'__type__': 'result', '__value__': obj.to_dict()}
+        if hasattr(obj, '__class__'):
+            encoded = base64.standard_b64encode(dill.dumps(obj))
+            return {'__type__': 'dill', '__value__': encoded.decode('utf-8')}
+
         return super().default(obj)
 
 
 class RuntimeDecoder(json.JSONDecoder):
-    """JSON Decoder for Numpy arrays and complex numbers."""
+    """JSON Decoder used by runtime service."""
 
     def __init__(self, *args, **kwargs):
         super().__init__(object_hook=self.object_hook, *args, **kwargs)
 
     def object_hook(self, obj):
-        if 'type' in obj:
-            if obj['type'] == 'complex':
-                val = obj['value']
+        if '__type__' in obj:
+            if obj['__type__'] == 'complex':
+                val = obj['__value__']
                 return val[0] + 1j * val[1]
-            if obj['type'] == 'array':
-                return np.array(obj['value'])
+            if obj['__type__'] == 'array':
+                return np.array(obj['__value__'])
+            if obj['__type__'] == 'result':
+                return Result.from_dict(obj['__value__'])
+            if obj['__type__'] == 'dill':
+                decoded = base64.standard_b64decode(obj['__value__'])
+                return dill.loads(decoded)
         return obj
-
-# class RuntimeEncoder(json.JSONEncoder):
-#     """JSON Encoder used by runtime service."""
-#
-#     def default(self, obj: Any) -> Any:
-#         if hasattr(obj, 'tolist'):
-#             return {'__type__': 'array', '__value__': obj.tolist()}
-#         if isinstance(obj, complex):
-#             return {'__type__': 'complex', '__value__': [obj.real, obj.imag]}
-#         if isinstance(obj, Result):
-#             return {'__type__': 'result', '__value__': obj.to_dict()}
-#         if hasattr(obj, '__class__'):
-#             encoded = base64.standard_b64encode(dill.dumps(obj))
-#             return {'__type__': 'dill', '__value__': encoded.decode('utf-8')}
-#
-#         return super().default(obj)
-#
-#
-# class RuntimeDecoder(json.JSONDecoder):
-#     """JSON Decoder used by runtime service."""
-#
-#     def __init__(self, *args, **kwargs):
-#         super().__init__(object_hook=self.object_hook, *args, **kwargs)
-#
-#     def object_hook(self, obj):
-#         if '__type__' in obj:
-#             if obj['__type__'] == 'complex':
-#                 val = obj['__value__']
-#                 return val[0] + 1j * val[1]
-#             if obj['__type__'] == 'array':
-#                 return np.array(obj['__value__'])
-#             if obj['__type__'] == 'result':
-#                 return Result.from_dict(obj['__value__'])
-#             if obj['__type__'] == 'dill':
-#                 decoded = base64.standard_b64decode(obj['__value__'])
-#                 return dill.loads(decoded)
-#         return obj
diff --git a/requirements.txt b/requirements.txt
index 32c79277c..9a8cbf752 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -6,3 +6,4 @@ websockets>=8
 numpy>=1.13
 urllib3>=1.21.1
 python-dateutil>=2.8.0
+dill>=0.3

From c26dce6f923427542cd3c7bb133bca1b2c2c167a Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Fri, 26 Mar 2021 14:15:57 -0400
Subject: [PATCH 19/59] add user messenger

---
 qiskit/providers/ibmq/api/rest/runtime.py     |  1 -
 qiskit/providers/ibmq/runtime/__init__.py     | 15 +++---
 .../ibmq/runtime/ibm_runtime_service.py       |  2 +-
 .../ibmq/runtime/program/__init__.py          | 16 ++++++
 .../ibmq/runtime/program/program_template.py  | 49 +++++++++++++++++++
 .../ibmq/runtime/program/user_messenger.py    | 46 +++++++++++++++++
 .../{utils/runtime.py => runtime/utils.py}    |  0
 7 files changed, 121 insertions(+), 8 deletions(-)
 create mode 100644 qiskit/providers/ibmq/runtime/program/__init__.py
 create mode 100644 qiskit/providers/ibmq/runtime/program/program_template.py
 create mode 100644 qiskit/providers/ibmq/runtime/program/user_messenger.py
 rename qiskit/providers/ibmq/{utils/runtime.py => runtime/utils.py} (100%)

diff --git a/qiskit/providers/ibmq/api/rest/runtime.py b/qiskit/providers/ibmq/api/rest/runtime.py
index fc07931ab..69b0d78ee 100644
--- a/qiskit/providers/ibmq/api/rest/runtime.py
+++ b/qiskit/providers/ibmq/api/rest/runtime.py
@@ -21,7 +21,6 @@
 from ..session import RetrySession
 
 logger = logging.getLogger(__name__)
-process = None
 
 
 class Runtime(RestAdapterBase):
diff --git a/qiskit/providers/ibmq/runtime/__init__.py b/qiskit/providers/ibmq/runtime/__init__.py
index 609e1e1fb..7c12c7f15 100644
--- a/qiskit/providers/ibmq/runtime/__init__.py
+++ b/qiskit/providers/ibmq/runtime/__init__.py
@@ -11,18 +11,18 @@
 # that they have been altered from the originals.
 
 """
-============================================================
+======================================================
 Runtime Service (:mod:`qiskit.providers.ibmq.runtime`)
-============================================================
+======================================================
 
 .. currentmodule:: qiskit.providers.ibmq.runtime
 
-Modules related to IBM Quantum Experience runtime service.
+Modules related to IBM Quantum Runtime Service.
 
 .. caution::
 
   This package is currently provided in beta form and heavy modifications to
-  both functionality and API are likely to occur.
+  both functionality and API are likely to occur without backward compatibility.
 
 .. note::
 
@@ -33,10 +33,13 @@
 .. autosummary::
    :toctree: ../stubs/
 
-   IBMRuntimeService
-   RuntimeJob
+   IBMQRandomService
+   CQCExtractor
+   CQCExtractorJob
 
 """
 
 from .ibm_runtime_service import IBMRuntimeService
 from .runtime_job import RuntimeJob
+from .runtime_program import RuntimeProgram
+from .program.user_messenger import UserMessenger
diff --git a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
index 87b99b414..06102f5ac 100644
--- a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
+++ b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
@@ -21,7 +21,7 @@
 
 from .runtime_job import RuntimeJob
 from .runtime_program import RuntimeProgram
-from ..utils.runtime import RuntimeEncoder
+from .utils import RuntimeEncoder
 from ..api.clients.runtime import RuntimeClient
 
 logger = logging.getLogger(__name__)
diff --git a/qiskit/providers/ibmq/runtime/program/__init__.py b/qiskit/providers/ibmq/runtime/program/__init__.py
new file mode 100644
index 000000000..7a6e4637b
--- /dev/null
+++ b/qiskit/providers/ibmq/runtime/program/__init__.py
@@ -0,0 +1,16 @@
+# This code is part of Qiskit.
+#
+# (C) Copyright IBM 2021.
+#
+# This code is licensed under the Apache License, Version 2.0. You may
+# obtain a copy of this license in the LICENSE.txt file in the root directory
+# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Any modifications or derivative works of this code must retain this
+# copyright notice, and modified files need to carry a notice indicating
+# that they have been altered from the originals.
+
+"""Runtime program package.
+
+This package contains code used to write runtime programs.
+"""
diff --git a/qiskit/providers/ibmq/runtime/program/program_template.py b/qiskit/providers/ibmq/runtime/program/program_template.py
new file mode 100644
index 000000000..3f761941d
--- /dev/null
+++ b/qiskit/providers/ibmq/runtime/program/program_template.py
@@ -0,0 +1,49 @@
+# This code is part of Qiskit.
+#
+# (C) Copyright IBM 2021.
+#
+# This code is licensed under the Apache License, Version 2.0. You may
+# obtain a copy of this license in the LICENSE.txt file in the root directory
+# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Any modifications or derivative works of this code must retain this
+# copyright notice, and modified files need to carry a notice indicating
+# that they have been altered from the originals.
+
+import sys
+import json
+
+from qiskit import Aer
+from qiskit.providers.backend import BackendV1 as Backend
+from qiskit.providers.ibmq.runtime import UserMessenger
+from qiskit.providers.ibmq.runtime.utils import RuntimeDecoder
+
+
+def program(backend: Backend, user_messenger: UserMessenger, **kwargs):
+    """Function that does classical-quantum calculation."""
+    # UserMessenger can be used to publish interim results.
+    user_messenger.publish("This is an interim result.")
+    return "final result"
+
+
+def main(backend: Backend, user_messenger: UserMessenger, **kwargs):
+    """This is the main entry point of a quantum program.
+
+    Args:
+        backend: Backend for the circuits to run on.
+        user_messenger: Used to communicate with the program consumer.
+    """
+    # Massage the input if necessary.
+    result = program(backend, user_messenger, **kwargs)
+    # UserMessenger can be used to publish final results.
+    user_messenger.publish(result, final_result=True)
+
+
+if __name__ == '__main__':
+    """This is used for testing locally with Aer simulator."""
+    _backend = Aer.get_backend('qasm_simulator')
+    user_params = {}
+    if len(sys.argv) > 1:
+        # If there are user parameters.
+        user_params = json.loads(sys.argv[1], cls=RuntimeDecoder)
+    main(_backend, UserMessenger(), **user_params)
diff --git a/qiskit/providers/ibmq/runtime/program/user_messenger.py b/qiskit/providers/ibmq/runtime/program/user_messenger.py
new file mode 100644
index 000000000..cef0a23e0
--- /dev/null
+++ b/qiskit/providers/ibmq/runtime/program/user_messenger.py
@@ -0,0 +1,46 @@
+# This code is part of Qiskit.
+#
+# (C) Copyright IBM 2021.
+#
+# This code is licensed under the Apache License, Version 2.0. You may
+# obtain a copy of this license in the LICENSE.txt file in the root directory
+# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Any modifications or derivative works of this code must retain this
+# copyright notice, and modified files need to carry a notice indicating
+# that they have been altered from the originals.
+
+"""Base class for handling communication with program consumers."""
+
+import json
+from typing import Any
+
+from ..utils import RuntimeEncoder
+
+
+class UserMessenger:
+    """Base class for handling communication with users"""
+
+    def publish(
+        self,
+        message: Any,
+        final_result: bool = False,
+        encoder: json.JSONEncoder = RuntimeEncoder,
+    ) -> None:
+        """Publish message.
+
+        You can use this method to publish messages, such as interim and final results,
+        to the program consumer. The messages will be made immediately available to the consumer,
+        but they may choose not to receive the messages.
+
+        The `final_result` parameter is used to indicate whether the message is
+        the final result of the program. Final results may be processed differently
+        from interim results.
+
+        Args:
+            message: Message to be published. Can be any type.
+            final_result: Whether this is the final result from the program.
+            encoder: An optional JSON encoder for serializing
+        """
+        # Default implementation for testing.
+        print(json.dumps(message, cls=encoder))
diff --git a/qiskit/providers/ibmq/utils/runtime.py b/qiskit/providers/ibmq/runtime/utils.py
similarity index 100%
rename from qiskit/providers/ibmq/utils/runtime.py
rename to qiskit/providers/ibmq/runtime/utils.py

From 1190457241bd6393a32d09e0bcb2f0fbe797e7dd Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Fri, 26 Mar 2021 14:20:38 -0400
Subject: [PATCH 20/59] fix import

---
 qiskit/providers/ibmq/runtime/program/__init__.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/qiskit/providers/ibmq/runtime/program/__init__.py b/qiskit/providers/ibmq/runtime/program/__init__.py
index 7a6e4637b..feecc01e0 100644
--- a/qiskit/providers/ibmq/runtime/program/__init__.py
+++ b/qiskit/providers/ibmq/runtime/program/__init__.py
@@ -14,3 +14,5 @@
 
 This package contains code used to write runtime programs.
 """
+
+from .user_messenger import UserMessenger

From 5973b5df3784ad66f02899f323b56374f985d77b Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Fri, 26 Mar 2021 14:22:53 -0400
Subject: [PATCH 21/59] fix setup

---
 qiskit/providers/ibmq/runtime/program/__init__.py | 2 --
 setup.py                                          | 3 ++-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/qiskit/providers/ibmq/runtime/program/__init__.py b/qiskit/providers/ibmq/runtime/program/__init__.py
index feecc01e0..7a6e4637b 100644
--- a/qiskit/providers/ibmq/runtime/program/__init__.py
+++ b/qiskit/providers/ibmq/runtime/program/__init__.py
@@ -14,5 +14,3 @@
 
 This package contains code used to write runtime programs.
 """
-
-from .user_messenger import UserMessenger
diff --git a/setup.py b/setup.py
index 13b84d5f5..7c544953e 100644
--- a/setup.py
+++ b/setup.py
@@ -82,7 +82,8 @@
               'qiskit.providers.ibmq.jupyter.dashboard',
               'qiskit.providers.ibmq.random',
               'qiskit.providers.ibmq.experiment',
-              'qiskit.providers.ibmq.runtime'],
+              'qiskit.providers.ibmq.runtime',
+              'qiskit.providers.ibmq.runtime.program'],
     install_requires=REQUIREMENTS,
     include_package_data=True,
     python_requires=">=3.6",

From 0ba3590596739d0ea0d6be8a2466d8e33ea86c0a Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Fri, 26 Mar 2021 15:05:30 -0400
Subject: [PATCH 22/59] add program backend

---
 qiskit/providers/ibmq/runtime/__init__.py     |  1 +
 .../ibmq/runtime/program/program_backend.py   | 66 +++++++++++++++++++
 .../ibmq/runtime/program/program_template.py  |  7 +-
 3 files changed, 70 insertions(+), 4 deletions(-)
 create mode 100644 qiskit/providers/ibmq/runtime/program/program_backend.py

diff --git a/qiskit/providers/ibmq/runtime/__init__.py b/qiskit/providers/ibmq/runtime/__init__.py
index 7c12c7f15..7c076c8a9 100644
--- a/qiskit/providers/ibmq/runtime/__init__.py
+++ b/qiskit/providers/ibmq/runtime/__init__.py
@@ -43,3 +43,4 @@
 from .runtime_job import RuntimeJob
 from .runtime_program import RuntimeProgram
 from .program.user_messenger import UserMessenger
+from .program.program_backend import ProgramBackend
diff --git a/qiskit/providers/ibmq/runtime/program/program_backend.py b/qiskit/providers/ibmq/runtime/program/program_backend.py
new file mode 100644
index 000000000..c2b9de321
--- /dev/null
+++ b/qiskit/providers/ibmq/runtime/program/program_backend.py
@@ -0,0 +1,66 @@
+# This code is part of Qiskit.
+#
+# (C) Copyright IBM 2021.
+#
+# This code is licensed under the Apache License, Version 2.0. You may
+# obtain a copy of this license in the LICENSE.txt file in the root directory
+# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Any modifications or derivative works of this code must retain this
+# copyright notice, and modified files need to carry a notice indicating
+# that they have been altered from the originals.
+
+"""Base class for program backend."""
+
+import logging
+from typing import Union, List, Dict, Optional
+from abc import abstractmethod, ABC
+
+from qiskit.qobj import QasmQobj, PulseQobj
+from qiskit.pulse import Schedule
+from qiskit.providers import BackendV1 as Backend
+from qiskit.providers import JobV1 as Job
+from qiskit.circuit import QuantumCircuit
+
+logger = logging.getLogger(__name__)
+
+
+class ProgramBackend(Backend, ABC):
+    """Base class for a program backend."""
+
+    @abstractmethod
+    def run(
+            self,
+            circuits: Union[QasmQobj, PulseQobj, QuantumCircuit, Schedule,
+                            List[Union[QuantumCircuit, Schedule]]],
+            timeout: Optional[int] = None,
+            **run_config: Dict
+    ) -> Job:
+        """Run on the backend.
+
+        Runtime circuit execution is synchronous, and control will not go
+        back until the execution finishes. You can use the `timeout` parameter
+        to set a timeout value to wait for the execution to finish. Note that if
+        the execution times out, circuit execution results will not be available.
+
+        Args:
+            circuits: An individual or a
+                list of :class:`~qiskit.circuits.QuantumCircuit` or
+                :class:`~qiskit.pulse.Schedule` objects to run on the backend.
+                A :class:`~qiskit.qobj.QasmQobj` or a
+                :class:`~qiskit.qobj.PulseQobj` object is also supported but
+                is deprecated.
+            timeout: Seconds to wait for circuit execution to finish.
+            **run_config: Extra arguments used to configure the run.
+
+        Returns:
+            The job to be executed.
+
+        Raises:
+            IBMQBackendApiError: If an unexpected error occurred while submitting
+                the job.
+            IBMQBackendApiProtocolError: If an unexpected value received from
+                 the server.
+            IBMQBackendValueError: If an input parameter value is not valid.
+        """
+        pass
diff --git a/qiskit/providers/ibmq/runtime/program/program_template.py b/qiskit/providers/ibmq/runtime/program/program_template.py
index 3f761941d..000c6ab19 100644
--- a/qiskit/providers/ibmq/runtime/program/program_template.py
+++ b/qiskit/providers/ibmq/runtime/program/program_template.py
@@ -14,19 +14,18 @@
 import json
 
 from qiskit import Aer
-from qiskit.providers.backend import BackendV1 as Backend
-from qiskit.providers.ibmq.runtime import UserMessenger
+from qiskit.providers.ibmq.runtime import UserMessenger, ProgramBackend
 from qiskit.providers.ibmq.runtime.utils import RuntimeDecoder
 
 
-def program(backend: Backend, user_messenger: UserMessenger, **kwargs):
+def program(backend: ProgramBackend, user_messenger: UserMessenger, **kwargs):
     """Function that does classical-quantum calculation."""
     # UserMessenger can be used to publish interim results.
     user_messenger.publish("This is an interim result.")
     return "final result"
 
 
-def main(backend: Backend, user_messenger: UserMessenger, **kwargs):
+def main(backend: ProgramBackend, user_messenger: UserMessenger, **kwargs):
     """This is the main entry point of a quantum program.
 
     Args:

From 4eba7ff5874b1b0e35b0550757cf7846d8efa9c9 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Wed, 31 Mar 2021 18:29:29 -0400
Subject: [PATCH 23/59] add to_json in encoder

---
 qiskit/providers/ibmq/api/clients/runtime.py  |  15 +-
 qiskit/providers/ibmq/runtime/__init__.py     |  23 ++-
 qiskit/providers/ibmq/runtime/constants.py    |   4 +-
 .../ibmq/runtime/ibm_runtime_service.py       |   8 +-
 qiskit/providers/ibmq/runtime/runtime_job.py  |  35 ++++-
 .../providers/ibmq/runtime/runtime_program.py | 105 ++++++++-----
 qiskit/providers/ibmq/runtime/utils.py        |   4 +
 test/fake_runtime_client.py                   | 147 ++++++++++++++++++
 test/ibmq/test_runtime.py                     |  31 ++++
 9 files changed, 319 insertions(+), 53 deletions(-)
 create mode 100644 test/fake_runtime_client.py

diff --git a/qiskit/providers/ibmq/api/clients/runtime.py b/qiskit/providers/ibmq/api/clients/runtime.py
index cbba1bc17..655c0e2db 100644
--- a/qiskit/providers/ibmq/api/clients/runtime.py
+++ b/qiskit/providers/ibmq/api/clients/runtime.py
@@ -38,7 +38,7 @@ def __init__(
             access_token: IBM Quantum Experience access token.
             credentials: Account credentials.
         """
-        url = os.getenv('NTC_URL', 'https://api-ntc.processing-prod-5dd5718798d097eccc65fac4e78a33ce-0000.us-east.containers.appdomain.cloud')
+        url = os.getenv('NTC_URL')
         logger.debug(f"Using runtime service url {url}")
         self._session = RetrySession(url, access_token,
                                      **credentials.connection_parameters())
@@ -56,7 +56,16 @@ def program_create(
             self,
             program_name: str,
             program_data: Union[bytes, str],
-    ):
+    ) -> Dict:
+        """Create a new program.
+
+        Args:
+            program_name: Name of the program.
+            program_data: Program data.
+
+        Returns:
+            Server response.
+        """
         return self.api.create_program(program_name=program_name, program_data=program_data)
 
     def program_get(self, program_id: str) -> Dict:
@@ -116,7 +125,7 @@ def program_job_get(self, job_id):
         logger.debug(f"Runtime job get response: {response}")
         return response
 
-    def program_job_results(self, job_id: str) -> Dict:
+    def program_job_results(self, job_id: str) -> str:
         """Get the results of a program job.
 
         Args:
diff --git a/qiskit/providers/ibmq/runtime/__init__.py b/qiskit/providers/ibmq/runtime/__init__.py
index 7c076c8a9..71b03f5bf 100644
--- a/qiskit/providers/ibmq/runtime/__init__.py
+++ b/qiskit/providers/ibmq/runtime/__init__.py
@@ -28,15 +28,30 @@
 
   The runtime service is not available to all accounts.
 
+The IBM Quantum Runtime Service allows select users to upload their quantum programs
+that can be invoked by others. A quantum program is a piece of code that takes
+certain inputs, does quantum and sometimes classical processing, and returns the
+results. For example, user A can upload a VQE quantum program that takes a Hamiltonian
+and an optimizer as inputs and returns the minimum eigensolver result. User B
+can then invoke this program, passing in the inputs and obtaining the results,
+with minimal code.
+
+These quantum programs, sometimes called runtime programs, run in a special
+runtime environment that is separate from normal circuit job execution and has
+special performance advantage.
+
+TODO: Add tutorial reference
+
 Classes
 ==========================
 .. autosummary::
    :toctree: ../stubs/
 
-   IBMQRandomService
-   CQCExtractor
-   CQCExtractorJob
-
+   IBMRuntimeService
+   RuntimeJob
+   RuntimeProgram
+   UserMessenger
+   ProgramBackend
 """
 
 from .ibm_runtime_service import IBMRuntimeService
diff --git a/qiskit/providers/ibmq/runtime/constants.py b/qiskit/providers/ibmq/runtime/constants.py
index 2087c3db8..bc2f9b132 100644
--- a/qiskit/providers/ibmq/runtime/constants.py
+++ b/qiskit/providers/ibmq/runtime/constants.py
@@ -20,11 +20,11 @@ class ApiRuntimeJobStatus(enum.Enum):
 
     PENDING = 'PENDING'
     RUNNING = 'RUNNING'
-    ERROR = 'ERROR'
+    FAILED = 'FAILED'
     SUCCEEDED = 'SUCCEEDED'
 
 
 JOB_FINAL_STATES = (
-    "ERROR",
+    "FAILED",
     "SUCCEEDED"
 )
diff --git a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
index 06102f5ac..650521471 100644
--- a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
+++ b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
@@ -21,7 +21,7 @@
 
 from .runtime_job import RuntimeJob
 from .runtime_program import RuntimeProgram
-from .utils import RuntimeEncoder
+from .utils import RuntimeEncoder, RuntimeDecoder
 from ..api.clients.runtime import RuntimeClient
 
 logger = logging.getLogger(__name__)
@@ -174,6 +174,8 @@ def job(self, job_id: str):
         """
         response = self._api_client.program_job_get(job_id)
         backend = self._provider.get_backend(response['backend'])
+        params_str = json.dumps(response.get('params', {}))
+        params = json.loads(params_str, cls=RuntimeDecoder)
         return RuntimeJob(backend=backend, api_client=self._api_client, job_id=response['id'],
-                          program_id=response.get('program', ""),
-                          params=response.get('params', {}))
+                          program_id=response.get('program', {}).get('id', ""),
+                          params=params)
diff --git a/qiskit/providers/ibmq/runtime/runtime_job.py b/qiskit/providers/ibmq/runtime/runtime_job.py
index 98ded603e..166ed9d81 100644
--- a/qiskit/providers/ibmq/runtime/runtime_job.py
+++ b/qiskit/providers/ibmq/runtime/runtime_job.py
@@ -15,12 +15,15 @@
 from typing import Any, Optional, Callable, Dict
 import time
 import logging
+import json
 
 from qiskit.providers.exceptions import JobTimeoutError
 from qiskit.providers.backend import Backend
 
 from ..api.clients import RuntimeClient
 from .constants import JOB_FINAL_STATES
+from .utils import RuntimeDecoder
+from ..job.exceptions import IBMQJobFailureError
 
 logger = logging.getLogger(__name__)
 
@@ -49,7 +52,7 @@ def __init__(
         self._job_id = job_id
         self._backend = backend
         self._api_client = api_client
-        self._result = None
+        self._results = []
         self._params = params or {}
         self._user_callback = user_callback
         self._program_id = program_id
@@ -58,21 +61,43 @@ def __init__(
     def result(
             self,
             timeout: Optional[float] = None,
-            wait: float = 5
+            wait: float = 5,
+            include_interim: bool = False
     ) -> Any:
         """Return the results of the job.
 
+        If ``include_interim=True`` is specified, this method will return a
+        list that includes both interim and final results in the order they
+        were published by the program.
+
         Args:
             timeout: Number of seconds to wait for job.
             wait: Seconds between queries.
+            include_interim: Whether to include interim results.
 
         Returns:
             Runtime job result.
+
+        Raises:
+            IBMQJobFailureError: If the job failed.
         """
-        if not self._result:
+        if not self._results:
             self.wait_for_final_state(timeout=timeout, wait=wait)
-            self._result = self._api_client.program_job_results(job_id=self.job_id())
-        return self._result
+            result_raw = self._api_client.program_job_results(job_id=self.job_id())
+            if self._status == 'FAILED':
+                raise IBMQJobFailureError(f"Unable to retrieve result for job {self.job_id()}. "
+                                          f"Job has failed:\n{result_raw}")
+            result_list = result_raw.split('\n')
+            for res in result_list:
+                if not res:
+                    continue
+                try:
+                    self._results.append(json.loads(res, cls=RuntimeDecoder))
+                except json.JSONDecodeError:
+                    self._results.append(res)
+        if include_interim:
+            return self._results
+        return self._results[-1]
 
     def cancel(self):
         """Attempt to cancel the job."""
diff --git a/qiskit/providers/ibmq/runtime/runtime_program.py b/qiskit/providers/ibmq/runtime/runtime_program.py
index 2807d839d..ac0d686de 100644
--- a/qiskit/providers/ibmq/runtime/runtime_program.py
+++ b/qiskit/providers/ibmq/runtime/runtime_program.py
@@ -11,7 +11,7 @@
 # that they have been altered from the originals.
 
 import logging
-from typing import Optional, List
+from typing import Optional, List, NamedTuple
 
 logger = logging.getLogger(__name__)
 
@@ -26,6 +26,7 @@ def __init__(
             description: str,
             parameters: Optional[List] = None,
             return_values: Optional[List] = None,
+            interim_results: Optional[List] = None,
             max_execution_time: float = 0,
             data: Optional[bytes] = None
     ) -> None:
@@ -35,82 +36,114 @@ def __init__(
             program_name: Program name.
             program_id: Program ID.
             description: Program description.
+            parameters: Documentation on program parameters.
+            return_values: Documentation on program return values.
+            interim_results: Documentation on program interim results.
             max_execution_time: Maximum execution time.
+            data: Program data.
         """
-        self.name = program_name
+        self._name = program_name
         self._id = program_id
         self._description = description
         self._cost = max_execution_time
         self._data = data
         self._parameters = []
         self._return_values = []
-        if parameters is not None:
+        self._interim_results = []
+
+        if parameters:
             for param in parameters:
                 self._parameters.append(
                     ProgramParameter(name=param['name'],
                                      description=param['description'],
-                                     param_type=param['type'],
+                                     type=param['type'],
                                      required=param['required']))
         if return_values is not None:
             for ret in return_values:
-                self._return_values.append(ProgramReturn(name=ret['name'],
+                self._return_values.append(ProgramResult(name=ret['name'],
                                                          description=ret['description'],
-                                                         return_type=ret['type']))
+                                                         type=ret['type']))
+        if interim_results is not None:
+            for intret in interim_results:
+                self._interim_results.append(ProgramResult(name=intret['name'],
+                                                           description=intret['description'],
+                                                           type=intret['type']))
 
     def __str__(self) -> str:
+        def _format_common(items: List):
+            """Add name, description, and type to `formatted`."""
+            for item in items:
+                formatted.append(" "*4 + "- " + item.name + ":")
+                formatted.append(" "*6 + "Description: " + item.description)
+                formatted.append(" "*6 + "Type: " + item.type)
+                if hasattr(item, 'required'):
+                    formatted.append(" "*6 + "Required: " + str(item.required))
+
         formatted = [f'{self.name}:',
                      f"  ID: {self._id}",
                      f"  Description: {self._description}",
                      f"  Parameters:"]
 
         if self._parameters:
-            for param in self._parameters:
-                formatted.append(" "*4 + "- " + param.name + ":")
-                formatted.append(" "*6 + "Description: " + param.description)
-                formatted.append(" "*6 + "Type: " + param.type)
-                formatted.append(" "*6 + "Required: " + str(param.required))
+            _format_common(self._parameters)
+        else:
+            formatted.append(" "*4 + "none")
+
+        formatted.append("  Interim results:")
+        if self._interim_results:
+            _format_common(self._interim_results)
         else:
             formatted.append(" "*4 + "none")
 
         formatted.append("  Returns:")
         if self._return_values:
-            for ret in self._return_values:
-                formatted.append(" "*4 + "- " + ret.name + ":")
-                formatted.append(" "*6 + "Description: " + ret.description)
-                formatted.append(" "*6 + "Type: " + ret.type)
+            _format_common(self._return_values)
         else:
             formatted.append(" "*4 + "none")
         return '\n'.join(formatted)
 
     @property
-    def program_id(self):
-        return self._id
+    def program_id(self) -> str:
+        """Return program ID.
 
+        Returns:
+            Program ID.
+        """
+        return self._id
 
-class ProgramParameter:
+    @property
+    def name(self) -> str:
+        """Return program name.
 
-    def __init__(self, name: str, description: str, param_type: str, required: bool):
+        Returns:
+            Program name.
         """
+        return self._name
 
-        Args:
-            description:
-            param_type:
+    @property
+    def description(self) -> str:
+        """Return program description.
+
+        Returns:
+            Program description.
         """
-        self.name = name
-        self.description = description
-        self.type = param_type
-        self.required = required
+        return self._description
+
+    @property
+    def parameter_doc(self) -> List['ProgramParameter']:
+        return self._parameters
 
 
-class ProgramReturn:
+class ProgramParameter(NamedTuple):
+    """Program parameter."""
+    name: str
+    description: str
+    type: str
+    required: bool
 
-    def __init__(self, name: str, description: str, return_type: str):
-        """
 
-        Args:
-            description:
-            return_type:
-        """
-        self.name = name
-        self.description = description
-        self.type = return_type
+class ProgramResult(NamedTuple):
+    """Program result."""
+    name: str
+    description: str
+    type: str
diff --git a/qiskit/providers/ibmq/runtime/utils.py b/qiskit/providers/ibmq/runtime/utils.py
index 267bc79ea..e08c5085e 100644
--- a/qiskit/providers/ibmq/runtime/utils.py
+++ b/qiskit/providers/ibmq/runtime/utils.py
@@ -33,6 +33,8 @@ def default(self, obj: Any) -> Any:
             return {'__type__': 'complex', '__value__': [obj.real, obj.imag]}
         if isinstance(obj, Result):
             return {'__type__': 'result', '__value__': obj.to_dict()}
+        if hasattr(obj, 'to_json'):
+            return {'__type__': 'to_json', '__value__': obj.to_json()}
         if hasattr(obj, '__class__'):
             encoded = base64.standard_b64encode(dill.dumps(obj))
             return {'__type__': 'dill', '__value__': encoded.decode('utf-8')}
@@ -55,6 +57,8 @@ def object_hook(self, obj):
                 return np.array(obj['__value__'])
             if obj['__type__'] == 'result':
                 return Result.from_dict(obj['__value__'])
+            if obj['__type__'] == 'to_json':
+                return obj['__value__']
             if obj['__type__'] == 'dill':
                 decoded = base64.standard_b64decode(obj['__value__'])
                 return dill.loads(decoded)
diff --git a/test/fake_runtime_client.py b/test/fake_runtime_client.py
new file mode 100644
index 000000000..2c092662c
--- /dev/null
+++ b/test/fake_runtime_client.py
@@ -0,0 +1,147 @@
+# This code is part of Qiskit.
+#
+# (C) Copyright IBM 2021.
+#
+# This code is licensed under the Apache License, Version 2.0. You may
+# obtain a copy of this license in the LICENSE.txt file in the root directory
+# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Any modifications or derivative works of this code must retain this
+# copyright notice, and modified files need to carry a notice indicating
+# that they have been altered from the originals.
+
+"""Fake RuntimeClient."""
+
+import time
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+
+from qiskit.providers.ibmq.credentials import Credentials
+from qiskit.providers.ibmq.api.exceptions import RequestsApiError
+
+
+class BaseFakeProgram:
+    """Base class for faking a program."""
+
+    def __init__(self, program_id, name, data, cost=600):
+        """Initialize a fake program."""
+        self._id = program_id
+        self._name = name
+        self._data = data
+        self._cost = cost
+
+    def to_dict(self, include_data=False):
+        out = {'id': self._id,
+               'name': self._name,
+               'cost': self._cost}
+        if include_data:
+            out['data'] = self._data
+        return out
+
+
+class BaseFakeRuntimeJob:
+    """Base class for faking a runtime job."""
+
+    _job_progress = [
+        "PENDING",
+        "RUNNING",
+        "SUCCEEDED"
+    ]
+
+    _executor = ThreadPoolExecutor()
+
+    def __init__(self, job_id, program_id, hub, group, project, backend_name, params):
+        """Initialize a fake job."""
+        self._job_id = job_id
+        self._status = "PENDING"
+        self._program_id = program_id
+        self._hub = hub
+        self._group = group
+        self._project = project
+        self._backend_name = backend_name
+        self._params = params
+        self._future = self._executor.submit(self._auto_progress)
+        self._result = None
+
+    def _auto_progress(self):
+        """Automatically update job status."""
+        for status in self._job_progress:
+            time.sleep(0.5)
+            self._status = status
+
+        if self._status == "SUCCEEDED":
+            self._result = "foo"
+
+    def to_dict(self):
+        return {'id': self._job_id,
+                'hub': self._hub,
+                'group': self._group,
+                'project': self._project,
+                'backend': self._backend_name,
+                'status': self._status,
+                'params': [self._params]}
+
+
+class BaseFakeRuntimeClient:
+    """Base class for faking the runtime client."""
+
+    def __init__(self):
+        """Initialize a fake runtime client."""
+        self._programs = {}
+        self._jobs = {}
+
+    def list_programs(self):
+        """List all progrmas."""
+        programs = []
+        for prog in self._programs.values():
+            programs.append(prog.to_dict())
+        return programs
+
+    def program_create(self, program_name, program_data):
+        """Create a program."""
+        if isinstance(program_data, str):
+            with open(program_data, 'rb') as file:
+                program_data = file.read()
+        program_id = uuid.uuid4().hex
+        self._programs[program_id] = BaseFakeProgram(program_id, program_name, program_data)
+        return {'id': program_id}
+
+    def program_get(self, program_id: str):
+        """Return a specific program."""
+        return self._programs[program_id].to_dict()
+
+    def program_get_data(self, program_id: str):
+        """Return a specific program and its data."""
+        return self._programs[program_id].to_dict(iclude_data=True)
+
+    def program_run(
+            self,
+            program_id: str,
+            credentials: Credentials,
+            backend_name: str,
+            params: str
+    ):
+        """Run the specified program."""
+        job_id = uuid.uuid4().hex
+        job = BaseFakeRuntimeJob(job_id=job_id, program_id=program_id,
+                                 hub=credentials.hub, group=credentials.group,
+                                 project=credentials.project, backend_name=backend_name,
+                                 params=params)
+        self._jobs[job_id] = job
+        return {'id': job_id}
+
+    def program_delete(self, program_id: str) -> None:
+        """Delete the specified program."""
+        if program_id not in self._programs:
+            raise RequestsApiError("Program not found")
+        del self._programs[program_id]
+
+    def program_job_get(self, job_id):
+        """Get the specific job."""
+        if job_id not in self._jobs:
+            raise RequestsApiError("Job not found")
+        return self._jobs[job_id].to_dict()
+
+    def program_job_results(self, job_id: str):
+        """Get the results of a program job."""
+        pass
diff --git a/test/ibmq/test_runtime.py b/test/ibmq/test_runtime.py
index 7b93d5cc6..24ba303d9 100644
--- a/test/ibmq/test_runtime.py
+++ b/test/ibmq/test_runtime.py
@@ -12,12 +12,16 @@
 
 """Tests for runtime service."""
 
+import unittest
+
 from qiskit.providers.jobstatus import JobStatus
 
 from ..ibmqtestcase import IBMQTestCase
 from ..decorators import requires_provider
+from ..fake_runtime_client import BaseFakeRuntimeClient
 
 
+@unittest.skip("Skip runtime tests")
 class TestRuntime(IBMQTestCase):
 
     @classmethod
@@ -28,6 +32,11 @@ def setUpClass(cls, provider):
         super().setUpClass()
         cls.provider = provider
 
+    def setUp(self):
+        """Initial test setup."""
+        super().setUp()
+        self.provider.runtime._api_client = BaseFakeRuntimeClient()
+
     def test_list_programs(self):
         """Test listing programs."""
         self.provider.runtime.programs()
@@ -50,3 +59,25 @@ def _callback(interim_result):
         backend = self.provider.backend.ibmq_qasm_simulator
         job = self.provider.runtime.run("QKA", backend=backend, params=params, callback=_callback)
         job.result()
+
+
+class TestRuntimeIntegration(IBMQTestCase):
+
+    @classmethod
+    @requires_provider
+    def setUpClass(cls, provider):
+        """Initial class level setup."""
+        # pylint: disable=arguments-differ
+        super().setUpClass()
+        cls.provider = provider
+        try:
+            provider.runtime.programs()
+        except Exception:
+            raise unittest.SkipTest("No access to runtime service.")
+
+    def test_list_programs(self):
+        """Test listing programs."""
+        programs = self.provider.runtime.programs()
+        self.assertTrue(programs)
+        for prog in programs:
+            self.assertTrue(prog.name)

From 0ebd5a7dc4b0c8874c1a62e792a0b3bceb3687d3 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Tue, 6 Apr 2021 22:08:26 -0400
Subject: [PATCH 24/59] interim results

---
 qiskit/providers/ibmq/accountprovider.py      | 124 ++++++++++++-
 qiskit/providers/ibmq/api/clients/__init__.py |   1 +
 qiskit/providers/ibmq/api/clients/runtime.py  |   2 +-
 .../providers/ibmq/api/clients/runtime_ws.py  | 167 ++++++++++++++++++
 qiskit/providers/ibmq/api/rest/runtime.py     |  35 +++-
 qiskit/providers/ibmq/runtime/constants.py    |  31 ++--
 .../ibmq/runtime/ibm_runtime_service.py       |  23 ++-
 .../ibmq/runtime/program/program_backend.py   |   6 +-
 .../ibmq/runtime/program/program_template.py  |  11 +-
 .../ibmq/runtime/program/user_messenger.py    |   5 +-
 qiskit/providers/ibmq/runtime/runtime_job.py  |  95 ++++++++--
 .../providers/ibmq/runtime/runtime_program.py |  52 +++++-
 qiskit/providers/ibmq/runtime/utils.py        |   4 +-
 test/ibmq/test_account_client.py              |   6 +-
 test/ibmq/test_runtime.py                     |  48 ++++-
 .../websocket/test_websocket_integration.py   |   6 +-
 16 files changed, 561 insertions(+), 55 deletions(-)
 create mode 100644 qiskit/providers/ibmq/api/clients/runtime_ws.py

diff --git a/qiskit/providers/ibmq/accountprovider.py b/qiskit/providers/ibmq/accountprovider.py
index d552f4116..25753e9d6 100644
--- a/qiskit/providers/ibmq/accountprovider.py
+++ b/qiskit/providers/ibmq/accountprovider.py
@@ -13,13 +13,21 @@
 """Provider for a single IBM Quantum Experience account."""
 
 import logging
-from typing import Dict, List, Optional, Any, Callable
+from typing import Dict, List, Optional, Any, Callable, Union
 from collections import OrderedDict
 import traceback
 
 from qiskit.providers import ProviderV1 as Provider  # type: ignore[attr-defined]
 from qiskit.providers.models import (QasmBackendConfiguration,
                                      PulseBackendConfiguration)
+from qiskit.circuit import QuantumCircuit, Parameter
+from qiskit.pulse.instruction_schedule_map import InstructionScheduleMap
+from qiskit.pulse.channels import PulseChannel
+from qiskit.providers.backend import BackendV1 as Backend
+from qiskit.providers.basebackend import BaseBackend
+from qiskit.transpiler import Layout
+from qiskit.qobj.utils import MeasLevel, MeasReturnType
+from qiskit.exceptions import QiskitError
 
 from .api.clients import AccountClient
 from .ibmqbackend import IBMQBackend, IBMQSimulator
@@ -190,6 +198,120 @@ def _discover_remote_backends(self, timeout: Optional[float] = None) -> Dict[str
 
         return ret
 
+    def run_circuits(
+            self,
+            circuits: Union[QuantumCircuit, List[QuantumCircuit]],
+            backend: Optional[Union[Backend, BaseBackend]] = None,
+            initial_layout: Optional[Union[Layout, Dict, List]] = None,
+            seed_transpiler: Optional[int] = None,
+            optimization_level: Optional[int] = None,
+            transpiler_options: Optional[dict] = None,
+            scheduling_method: Optional[str] = None,
+            shots: Optional[int] = None,
+            memory: Optional[bool] = None,
+            memory_slots: Optional[int] = None,
+            memory_slot_size: Optional[int] = None,
+            rep_time: Optional[int] = None,
+            rep_delay: Optional[float] = None,
+            parameter_binds: Optional[List[Dict[Parameter, float]]] = None,
+            schedule_circuit=False,
+            inst_map: InstructionScheduleMap = None,
+            meas_map: List[List[int]] = None,
+            init_qubits: Optional[bool] = None,
+            **run_config: Dict
+    ) -> 'RuntimeJob':
+        """Execute the input circuit(s) on a backend using the runtime service.
+
+        Note:
+            This method uses the IBM Quantum runtime service which is not
+            available to all accounts.
+
+        Args:
+            circuits: Circuit(s) to execute.
+
+            backend: Backend to execute circuits on.
+                Transpiler options are automatically grabbed from backend configuration
+                and properties unless otherwise specified.
+
+            initial_layout: Initial position of virtual qubits on physical qubits.
+
+            seed_transpiler: Sets random seed for the stochastic parts of the transpiler.
+
+            optimization_level: How much optimization to perform on the circuits.
+                Higher levels generate more optimized circuits, at the expense of longer
+                transpilation time.
+                If None, level 1 will be chosen as default.
+
+            transpiler_options: Additional transpiler options.
+
+            scheduling_method: Scheduling method.
+
+            shots: Number of repetitions of each circuit, for sampling. Default: 1024.
+
+            memory: If True, per-shot measurement bitstrings are returned as well
+                (provided the backend supports it). Default: False
+
+            memory_slots: Number of classical memory slots used in this job.
+
+            memory_slot_size: Size of each memory slot if the output is Level 0.
+
+            rep_time: Time per program execution in seconds. Must be from the list provided
+                by the backend configuration (``backend.configuration().rep_times``).
+                Defaults to the first entry.
+
+            rep_delay: Delay between programs in seconds. Only supported on certain
+                backends (``backend.configuration().dynamic_reprate_enabled`` ). If supported,
+                ``rep_delay`` will be used instead of ``rep_time`` and must be from the
+                range supplied by the backend (``backend.configuration().rep_delay_range``).
+                Default is given by ``backend.configuration().default_rep_delay``.
+
+            parameter_binds: List of Parameter bindings over which the set of
+                experiments will be executed. Each list element (bind) should be of the form
+                ``{Parameter1: value1, Parameter2: value2, ...}``. All binds will be
+                executed across all experiments, e.g. if parameter_binds is a
+                length-n list, and there are m experiments, a total of :math:`m x n`
+                experiments will be run (one for each experiment/bind pair).
+
+            schedule_circuit: If ``True``, ``circuits`` will be converted to
+                :class:`qiskit.pulse.Schedule` objects prior to execution.
+
+            inst_map: Mapping of circuit operations to pulse schedules. If None, defaults to the
+                ``instruction_schedule_map`` of ``backend``.
+
+            meas_map: List of sets of qubits that must be measured together. If None,
+                defaults to the ``meas_map`` of ``backend``.
+
+            init_qubits: Whether to reset the qubits to the ground state for each shot.
+                Default: ``True``.
+
+            **run_config: Extra arguments used to configure the circuit execution.
+
+        Returns:
+            Runtime job.
+        """
+        inputs = {
+            'circuits': circuits,
+            'initial_layout': initial_layout,
+            'seed_transpiler': seed_transpiler,
+            'optimization_level': optimization_level,
+            'scheduling_method': scheduling_method,
+            'shots': shots,
+            'memory': memory,
+            'memory_slots': memory_slots,
+            'memory_slot_size': memory_slot_size,
+            'rep_time': rep_time,
+            'rep_delay': rep_delay,
+            'parameter_binds': parameter_binds,
+            'schedule_circuit': schedule_circuit,
+            'inst_map': inst_map,
+            'meas_map': meas_map,
+            'init_qubits': init_qubits,
+            'transpiler_options': transpiler_options
+        }
+        inputs.update(run_config)
+        options = {'backend_name': backend.name()}
+        return self.runtime.run('circuit-runner-jessie3', options=options, inputs=inputs)
+
     def service(self, name: str) -> Any:
         """Return the specified service.
 
diff --git a/qiskit/providers/ibmq/api/clients/__init__.py b/qiskit/providers/ibmq/api/clients/__init__.py
index 4e74efd1d..77317db29 100644
--- a/qiskit/providers/ibmq/api/clients/__init__.py
+++ b/qiskit/providers/ibmq/api/clients/__init__.py
@@ -18,3 +18,4 @@
 from .version import VersionClient
 from .websocket import WebsocketClient
 from .runtime import RuntimeClient
+from .runtime_ws import RuntimeWebsocketClient
diff --git a/qiskit/providers/ibmq/api/clients/runtime.py b/qiskit/providers/ibmq/api/clients/runtime.py
index 655c0e2db..2e662fc33 100644
--- a/qiskit/providers/ibmq/api/clients/runtime.py
+++ b/qiskit/providers/ibmq/api/clients/runtime.py
@@ -38,7 +38,7 @@ def __init__(
             access_token: IBM Quantum Experience access token.
             credentials: Account credentials.
         """
-        url = os.getenv('NTC_URL')
+        url = os.getenv('NTC_URL', "")
         logger.debug(f"Using runtime service url {url}")
         self._session = RetrySession(url, access_token,
                                      **credentials.connection_parameters())
diff --git a/qiskit/providers/ibmq/api/clients/runtime_ws.py b/qiskit/providers/ibmq/api/clients/runtime_ws.py
new file mode 100644
index 000000000..843df23c4
--- /dev/null
+++ b/qiskit/providers/ibmq/api/clients/runtime_ws.py
@@ -0,0 +1,167 @@
+# This code is part of Qiskit.
+#
+# (C) Copyright IBM 2021.
+#
+# This code is licensed under the Apache License, Version 2.0. You may
+# obtain a copy of this license in the LICENSE.txt file in the root directory
+# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Any modifications or derivative works of this code must retain this
+# copyright notice, and modified files need to carry a notice indicating
+# that they have been altered from the originals.
+
+"""Client for accessing IBM Quantum runtime service."""
+
+import logging
+import asyncio
+from typing import Optional, Any
+from ssl import SSLError
+
+from websockets import connect, ConnectionClosed
+from websockets.client import WebSocketClientProtocol
+from websockets.exceptions import InvalidURI
+
+from ..exceptions import WebsocketError, WebsocketTimeoutError
+
+logger = logging.getLogger(__name__)
+
+
+class RuntimeWebsocketClient:
+    """Client for websocket communication with the IBM Quantum runtime service."""
+
+    BACKOFF_MAX = 8
+    """Maximum time to wait between retries."""
+
+    def __init__(
+            self,
+            websocket_url: str,
+            access_token: str
+    ) -> None:
+        """WebsocketClient constructor.
+
+        Args:
+            websocket_url: URL for websocket communication with runtime service.
+            access_token: Access token for IBM Quantum Experience.
+        """
+        self._websocket_url = websocket_url.rstrip('/')
+        self._access_token = access_token
+        self._header = {"X-Access-Token": self._access_token}
+        self._ws = None
+
+    async def _connect(self, url: str) -> WebSocketClientProtocol:
+        """Authenticate with the websocket server and return the connection.
+
+        Returns:
+            An open websocket connection.
+
+        Raises:
+            WebsocketError: If the connection to the websocket server could
+                not be established.
+            WebsocketAuthenticationError: If the connection to the websocket
+                was established, but the authentication failed.
+            WebsocketIBMQProtocolError: If the connection to the websocket
+                server was established, but the answer was unexpected.
+        """
+        try:
+            logger.debug('Starting new websocket connection: %s', url)
+            # TODO: Re-enable ping_timeout when server is fixed.
+            websocket = await connect(url, extra_headers=self._header, ping_interval=None)
+            await websocket.recv()  # Ack from server
+
+        # Isolate specific exceptions, so they are not retried in `get_job_status`.
+        except (SSLError, InvalidURI) as ex:
+            raise ex
+
+        # pylint: disable=broad-except
+        except Exception as ex:
+            exception_to_raise = WebsocketError('Failed to connect to the server.')
+
+            logger.info('An exception occurred. Raising "%s" from "%s"',
+                        repr(exception_to_raise), repr(ex))
+            raise exception_to_raise from ex
+
+        logger.debug("Runtime websocket connection established.")
+        return websocket
+
+    async def job_results(
+            self,
+            job_id: str,
+            timeout: Optional[float] = 5,
+            max_retries: int = 5,
+            backoff_factor: float = 0.5
+    ) -> Any:
+        """Return the interim result of a runtime job.
+
+        Args:
+            job_id: ID of the job.
+            timeout: Timeout value, in seconds.
+            max_retries: Max number of retries.
+            backoff_factor: Backoff factor used to calculate the
+                time to wait between retries.
+
+        Returns:
+            The interim result of a job.
+
+        Raises:
+            WebsocketError: If the websocket connection ended unexpectedly.
+            WebsocketTimeoutError: If the timeout has been reached.
+        """
+        url = '{}/stream/jobs/{}'.format(self._websocket_url, job_id)
+
+        current_retry = 0
+
+        while current_retry <= max_retries:
+            try:
+                if self._ws is None:
+                    self._ws = await self._connect(url)
+
+                response = await asyncio.wait_for(self._ws.recv(), timeout=timeout)
+                return response
+
+            except asyncio.TimeoutError:
+                # Timeout during our wait.
+                raise WebsocketTimeoutError(
+                    'Timeout reached while streaming job results.') from None
+            except (WebsocketError, ConnectionClosed) as ex:
+                if isinstance(ex, ConnectionClosed):
+                    self._ws = None
+
+                logger.debug(
+                    f"A websocket error occurred while streaming runtime job result: {str(ex)}")
+                current_retry += 1
+                if current_retry > max_retries:
+                    raise ex
+
+                # Sleep, and then `continue` with retrying.
+                backoff_time = self._backoff_time(backoff_factor, current_retry)
+                logger.info('Retrying websocket after %s seconds: '
+                            'Attempt #%s', backoff_time, current_retry)
+                await asyncio.sleep(backoff_time)  # Block asyncio loop for given backoff time.
+
+        # Execution should not reach here, sanity check.
+        exception_message = 'Max retries exceeded: Failed to establish a websocket ' \
+                            'connection due to a network error.'
+        raise WebsocketError(exception_message)
+
+    def _backoff_time(self, backoff_factor: float, current_retry_attempt: int) -> float:
+        """Calculate the backoff time to wait for.
+
+        Exponential backoff time formula::
+            {backoff_factor} * (2 ** (current_retry_attempt - 1))
+
+        Args:
+            backoff_factor: Backoff factor, in seconds.
+            current_retry_attempt: Current number of retry attempts.
+
+        Returns:
+            The number of seconds to wait for, before making the next retry attempt.
+        """
+        backoff_time = backoff_factor * (2 ** (current_retry_attempt - 1))
+        return min(self.BACKOFF_MAX, backoff_time)
+
+    async def disconnect(self) -> None:
+        """Close the websocket connection."""
+        if self._ws is not None:
+            logger.debug("Closing runtime websocket connection.")
+            await self._ws.close()
+            self._ws = None
diff --git a/qiskit/providers/ibmq/api/rest/runtime.py b/qiskit/providers/ibmq/api/rest/runtime.py
index 69b0d78ee..475c501ab 100644
--- a/qiskit/providers/ibmq/api/rest/runtime.py
+++ b/qiskit/providers/ibmq/api/rest/runtime.py
@@ -24,7 +24,7 @@
 
 
 class Runtime(RestAdapterBase):
-    """Rest adapter for RNG related endpoints."""
+    """Rest adapter for Runtime base endpoints."""
 
     URL_MAP = {
         'programs': '/programs',
@@ -212,3 +212,36 @@ def results(self) -> str:
         """
         response = self.session.get(self.get_url('results'))
         return response.text
+
+
+class Stream(RestAdapterBase):
+    """Rest adapter for streaming related endpoints."""
+
+    URL_MAP = {
+        'jobs': '/jobs'
+    }
+
+    def __init__(
+            self,
+            session: RetrySession,
+            url_prefix: str = ''
+    ) -> None:
+        """ProgramJob constructor.
+
+        Args:
+            session: Session to be used in the adapter.
+            url_prefix: Prefix to use in the URL.
+        """
+        super().__init__(session, '{}/stream'.format(
+            url_prefix))
+
+    def job(self, job_id):
+        """
+
+        Args:
+            job_id:
+
+        Returns:
+
+        """
+        response = self.session.get()
\ No newline at end of file
diff --git a/qiskit/providers/ibmq/runtime/constants.py b/qiskit/providers/ibmq/runtime/constants.py
index bc2f9b132..c001c7610 100644
--- a/qiskit/providers/ibmq/runtime/constants.py
+++ b/qiskit/providers/ibmq/runtime/constants.py
@@ -14,17 +14,26 @@
 
 import enum
 
+from qiskit.providers.jobstatus import JobStatus
 
-class ApiRuntimeJobStatus(enum.Enum):
-    """Possible values used by the API for a runtime job status."""
-
-    PENDING = 'PENDING'
-    RUNNING = 'RUNNING'
-    FAILED = 'FAILED'
-    SUCCEEDED = 'SUCCEEDED'
 
+# class ApiRuntimeJobStatus(enum.Enum):
+#     """Possible values used by the API for a runtime job status."""
+#
+#     PENDING = 'PENDING'
+#     RUNNING = 'RUNNING'
+#     FAILED = 'FAILED'
+#     SUCCEEDED = 'SUCCEEDED'
+#
+#
+# JOB_FINAL_STATES = (
+#     "FAILED",
+#     "SUCCEEDED"
+# )
 
-JOB_FINAL_STATES = (
-    "FAILED",
-    "SUCCEEDED"
-)
+API_TO_JOB_STATUS = {
+    'PENDING': JobStatus.INITIALIZING,
+    'RUNNING': JobStatus.RUNNING,
+    'SUCCEEDED': JobStatus.DONE,
+    'FAILED': JobStatus.ERROR
+}
diff --git a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
index 650521471..08e4f091e 100644
--- a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
+++ b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
@@ -28,7 +28,7 @@
 
 
 class IBMRuntimeService:
-    """IBM Quantum runtime service."""
+    """Class for interacting with the IBM Quantum runtime service."""
 
     def __init__(self, provider: 'accountprovider.AccountProvider', access_token: str) -> None:
         """IBMRuntimeService constructor.
@@ -39,10 +39,11 @@ def __init__(self, provider: 'accountprovider.AccountProvider', access_token: st
         """
         self._provider = provider
         self._api_client = RuntimeClient(access_token, provider.credentials)
+        self._access_token = access_token
         self._programs = {}
 
-    def print_programs(self, refresh: bool = False) -> None:
-        """Print information about available runtime programs.
+    def pprint_programs(self, refresh: bool = False) -> None:
+        """Pretty print information about available runtime programs.
 
         Args:
             refresh: If ``True``, re-query the server for the programs. Otherwise
@@ -64,6 +65,7 @@ def programs(self, refresh: bool = False) -> List[RuntimeProgram]:
             A list of runtime programs.
         """
         if not self._programs or refresh:
+            self._programs = {}
             response = self._api_client.list_programs()
             for prog_dict in response:
                 program = self._to_program(prog_dict)
@@ -107,7 +109,7 @@ def run(
             self,
             program_id: str,
             options: Dict,
-            params: Dict,
+            inputs: Dict,
             callback: Optional[Callable] = None
     ) -> RuntimeJob:
         """Execute the runtime program.
@@ -116,7 +118,7 @@ def run(
             program_id: Program ID.
             options: Runtime options. Currently the only available option is
                 ``backend_name``, which is required.
-            params: Program parameters.
+            inputs: Program input parameters.
             callback: Callback function to be invoked for any interim results.
 
         Returns:
@@ -126,15 +128,16 @@ def run(
             raise QiskitError('"backend_name" is required field in "options"')
 
         backend_name = options['backend_name']
-        params_str = json.dumps(params, cls=RuntimeEncoder)
+        params_str = json.dumps(inputs, cls=RuntimeEncoder)
         response = self._api_client.program_run(program_id=program_id,
                                                 credentials=self._provider.credentials,
                                                 backend_name=backend_name,
                                                 params=params_str)
 
         backend = self._provider.get_backend(backend_name)
-        job = RuntimeJob(backend=backend, api_client=self._api_client,
-                         job_id=response['id'], program_id=program_id, params=params,
+        job = RuntimeJob(backend=backend,
+                         api_client=self._api_client, access_token=self._access_token,
+                         job_id=response['id'], program_id=program_id, params=inputs,
                          user_callback=callback)
         return job
 
@@ -176,6 +179,8 @@ def job(self, job_id: str):
         backend = self._provider.get_backend(response['backend'])
         params_str = json.dumps(response.get('params', {}))
         params = json.loads(params_str, cls=RuntimeDecoder)
-        return RuntimeJob(backend=backend, api_client=self._api_client, job_id=response['id'],
+        return RuntimeJob(backend=backend,
+                          api_client=self._api_client, access_token=self._access_token,
+                          job_id=response['id'],
                           program_id=response.get('program', {}).get('id', ""),
                           params=params)
diff --git a/qiskit/providers/ibmq/runtime/program/program_backend.py b/qiskit/providers/ibmq/runtime/program/program_backend.py
index c2b9de321..c8f5480c2 100644
--- a/qiskit/providers/ibmq/runtime/program/program_backend.py
+++ b/qiskit/providers/ibmq/runtime/program/program_backend.py
@@ -26,7 +26,11 @@
 
 
 class ProgramBackend(Backend, ABC):
-    """Base class for a program backend."""
+    """Base class for a program backend.
+
+    This is a :class:`~qiskit.providers.Backend` class for runtime programs to
+    use in place of :class:`~qiskit.providers.ibmq.IBMQBackend`.
+    """
 
     @abstractmethod
     def run(
diff --git a/qiskit/providers/ibmq/runtime/program/program_template.py b/qiskit/providers/ibmq/runtime/program/program_template.py
index 000c6ab19..d1b094eea 100644
--- a/qiskit/providers/ibmq/runtime/program/program_template.py
+++ b/qiskit/providers/ibmq/runtime/program/program_template.py
@@ -10,6 +10,13 @@
 # copyright notice, and modified files need to carry a notice indicating
 # that they have been altered from the originals.
 
+"""Runtime program template.
+
+The ``main()`` method is the entry point of a runtime program. It takes a
+:class:`ProgramBackend` and a :class:`UserMessenger` that can be used to
+send circuits to the backend and messages to the user, respectively.
+"""
+
 import sys
 import json
 
@@ -26,7 +33,7 @@ def program(backend: ProgramBackend, user_messenger: UserMessenger, **kwargs):
 
 
 def main(backend: ProgramBackend, user_messenger: UserMessenger, **kwargs):
-    """This is the main entry point of a quantum program.
+    """This is the main entry point of a runtime program.
 
     Args:
         backend: Backend for the circuits to run on.
@@ -39,7 +46,7 @@ def main(backend: ProgramBackend, user_messenger: UserMessenger, **kwargs):
 
 
 if __name__ == '__main__':
-    """This is used for testing locally with Aer simulator."""
+    """This is used for testing locally using Aer simulator."""
     _backend = Aer.get_backend('qasm_simulator')
     user_params = {}
     if len(sys.argv) > 1:
diff --git a/qiskit/providers/ibmq/runtime/program/user_messenger.py b/qiskit/providers/ibmq/runtime/program/user_messenger.py
index cef0a23e0..03fbfa527 100644
--- a/qiskit/providers/ibmq/runtime/program/user_messenger.py
+++ b/qiskit/providers/ibmq/runtime/program/user_messenger.py
@@ -19,7 +19,10 @@
 
 
 class UserMessenger:
-    """Base class for handling communication with users"""
+    """Base class for handling communication with program consumers.
+
+    A program consumer is the user that executes the runtime program.
+    """
 
     def publish(
         self,
diff --git a/qiskit/providers/ibmq/runtime/runtime_job.py b/qiskit/providers/ibmq/runtime/runtime_job.py
index 166ed9d81..91099ef23 100644
--- a/qiskit/providers/ibmq/runtime/runtime_job.py
+++ b/qiskit/providers/ibmq/runtime/runtime_job.py
@@ -16,24 +16,34 @@
 import time
 import logging
 import json
+import asyncio
+from concurrent import futures
+import traceback
+import os
 
 from qiskit.providers.exceptions import JobTimeoutError
 from qiskit.providers.backend import Backend
+from qiskit.providers.jobstatus import JobStatus, JOB_FINAL_STATES
 
-from ..api.clients import RuntimeClient
-from .constants import JOB_FINAL_STATES
 from .utils import RuntimeDecoder
+from .constants import API_TO_JOB_STATUS
+from ..api.clients import RuntimeClient, RuntimeWebsocketClient
+from ..api.exceptions import WebsocketTimeoutError
 from ..job.exceptions import IBMQJobFailureError
 
 logger = logging.getLogger(__name__)
 
 
 class RuntimeJob:
+    """Representation of a runtime program execution."""
+
+    _executor = futures.ThreadPoolExecutor()
 
     def __init__(
             self,
             backend: 'ibmqbackend.IBMQBackend',
             api_client: RuntimeClient,
+            access_token: str,
             job_id: str,
             program_id: str,
             params: Optional[Dict] = None,
@@ -44,6 +54,7 @@ def __init__(
         Args:
             backend: The backend instance used to run this job.
             api_client: Object for connecting to the server.
+            access_token: IBM Quantum Experience access token.
             job_id: Job ID.
             program_id: ID of the program this job is for.
             params: Job parameters.
@@ -52,11 +63,17 @@ def __init__(
         self._job_id = job_id
         self._backend = backend
         self._api_client = api_client
+        url = os.getenv('NTC_URL', "")
+        ws_url = url.replace('https', 'wss')
+        self._ws_client = RuntimeWebsocketClient(ws_url, access_token)
         self._results = []
         self._params = params or {}
-        self._user_callback = user_callback
+        # self._user_callback = user_callback
         self._program_id = program_id
-        self._status = 'PENDING'
+        self._status = JobStatus.INITIALIZING
+
+        if user_callback is not None:
+            self.stream_results(user_callback)
 
     def result(
             self,
@@ -84,17 +101,15 @@ def result(
         if not self._results:
             self.wait_for_final_state(timeout=timeout, wait=wait)
             result_raw = self._api_client.program_job_results(job_id=self.job_id())
-            if self._status == 'FAILED':
+            if self._status == JobStatus.ERROR:
                 raise IBMQJobFailureError(f"Unable to retrieve result for job {self.job_id()}. "
                                           f"Job has failed:\n{result_raw}")
+            # TODO - Update when interim results are for streaming only
             result_list = result_raw.split('\n')
             for res in result_list:
                 if not res:
                     continue
-                try:
-                    self._results.append(json.loads(res, cls=RuntimeDecoder))
-                except json.JSONDecodeError:
-                    self._results.append(res)
+                self._results.append(self._decode_data(res))
         if include_interim:
             return self._results
         return self._results[-1]
@@ -103,7 +118,7 @@ def cancel(self):
         """Attempt to cancel the job."""
         raise NotImplementedError
 
-    def status(self) -> str:
+    def status(self) -> JobStatus:
         """Return the status of the job.
 
         Returns:
@@ -111,7 +126,7 @@ def status(self) -> str:
         """
         if self._status not in JOB_FINAL_STATES:
             response = self._api_client.program_job_get(job_id=self.job_id())
-            self._status = response['status'].upper()
+            self._status = API_TO_JOB_STATUS[response['status'].upper()]
         return self._status
 
     def wait_for_final_state(
@@ -140,6 +155,64 @@ def wait_for_final_state(
             status = self.status()
         return
 
+    def stream_results(self, callback: Callable):
+        """Start streaming job results.
+
+        Args:
+            callback: Callback function to be invoked for any interim results.
+                The callback function will receive 2 position parameters:
+                    1. Job ID
+                    2. Job interim result.
+        """
+        self._executor.submit(self._stream_results, user_callback=callback)
+
+    def _stream_results(self, user_callback: Callable) -> None:
+        """Stream interim results.
+
+        Args:
+            user_callback: User callback function.
+        """
+        try:
+            loop = asyncio.get_event_loop()
+        except RuntimeError as ex:
+            # Event loop may not be set in a child thread.
+            if 'There is no current event loop' in str(ex):
+                loop = asyncio.new_event_loop()
+                asyncio.set_event_loop(loop)
+            else:
+                raise
+
+        logger.debug(f"Start result streaming for job {self.job_id()}")
+        try:
+            while True:
+                try:
+                    response = loop.run_until_complete(self._ws_client.job_results(self._job_id))
+                    user_callback(self.job_id(), self._decode_data(response))
+                except WebsocketTimeoutError:
+                    if self.status() in JOB_FINAL_STATES:
+                        logger.debug(f"Job {self.job_id()} finished, stop result streaming.")
+                        return
+        except Exception:  # pylint: disable=broad-except
+            logger.warning(
+                f"An error occurred while streaming results for job {self.job_id()}: " +
+                traceback.format_exc())
+        finally:
+            loop.run_until_complete(self._ws_client.disconnect())
+
+    def _decode_data(self, data: Any) -> Any:
+        """Attempt to decode data using default decoder.
+
+        Args:
+            data: Data to be decoded.
+
+        Returns:
+            Decoded data, or the original data if decoding failed.
+        """
+        try:
+            return json.loads(data, cls=RuntimeDecoder)
+        except json.JSONDecodeError:
+            return data
+
     def job_id(self) -> str:
         """Return a unique id identifying the job.
 
diff --git a/qiskit/providers/ibmq/runtime/runtime_program.py b/qiskit/providers/ibmq/runtime/runtime_program.py
index ac0d686de..e0c5851fb 100644
--- a/qiskit/providers/ibmq/runtime/runtime_program.py
+++ b/qiskit/providers/ibmq/runtime/runtime_program.py
@@ -17,7 +17,25 @@
 
 
 class RuntimeProgram:
-    """Class representing a program definition."""
+    """Class representing program metadata.
+
+    This class contains the metadata describing a program, including its
+    name, ID, description, etc.
+
+    You can use the :class:`~qiskit.providers.ibmq.runtime.IBMRuntimeService`
+    to retrieve the metadata of a specific program or all programs. For example::
+
+        from qiskit import IBMQ
+
+        provider = IBMQ.load_account()
+
+        # To retrieve metadata of all programs.
+        programs = provider.runtime.programs()
+
+        # To retrieve metadata of a single program.
+        program = provider.runtime.program(program_id='circuit-runner')
+        print(f"Program {program.name} takes parameters {program.parameters}")
+    """
 
     def __init__(
             self,
@@ -27,10 +45,9 @@ def __init__(
             parameters: Optional[List] = None,
             return_values: Optional[List] = None,
             interim_results: Optional[List] = None,
-            max_execution_time: float = 0,
-            data: Optional[bytes] = None
+            max_execution_time: float = 0
     ) -> None:
-        """
+        """RuntimeProgram constructor.
 
         Args:
             program_name: Program name.
@@ -40,13 +57,11 @@ def __init__(
             return_values: Documentation on program return values.
             interim_results: Documentation on program interim results.
             max_execution_time: Maximum execution time.
-            data: Program data.
         """
         self._name = program_name
         self._id = program_id
         self._description = description
         self._cost = max_execution_time
-        self._data = data
         self._parameters = []
         self._return_values = []
         self._interim_results = []
@@ -130,9 +145,32 @@ def description(self) -> str:
         return self._description
 
     @property
-    def parameter_doc(self) -> List['ProgramParameter']:
+    def parameters(self) -> List['ProgramParameter']:
+        """Return program parameter definitions.
+
+        Returns:
+            Parameter definitions for this program.
+        """
         return self._parameters
 
+    @property
+    def return_values(self) -> List['ProgramResult']:
+        """Return program return value definitions.
+
+        Returns:
+            Return value definitions for this program.
+        """
+        return self._return_values
+
+    @property
+    def interim_results(self) -> List['ProgramResult']:
+        """Return program interim result definitions.
+
+        Returns:
+            Interim result definitions for this program.
+        """
+        return self._interim_results
+
 
 class ProgramParameter(NamedTuple):
     """Program parameter."""
diff --git a/qiskit/providers/ibmq/runtime/utils.py b/qiskit/providers/ibmq/runtime/utils.py
index e08c5085e..731126d97 100644
--- a/qiskit/providers/ibmq/runtime/utils.py
+++ b/qiskit/providers/ibmq/runtime/utils.py
@@ -10,9 +10,7 @@
 # copyright notice, and modified files need to carry a notice indicating
 # that they have been altered from the originals.
 
-# pylint: disable=method-hidden
-
-"""Custom JSON encoders."""
+"""Utility functions for the runtime service."""
 
 import json
 from typing import Any
diff --git a/test/ibmq/test_account_client.py b/test/ibmq/test_account_client.py
index 359e4d9f3..230b8a80a 100644
--- a/test/ibmq/test_account_client.py
+++ b/test/ibmq/test_account_client.py
@@ -41,7 +41,7 @@ def setUpClass(cls, provider):
         # pylint: disable=arguments-differ
         super().setUpClass()
         cls.provider = provider
-        cls.access_token = cls.provider._api_client.account_api.session.access_token
+        cls.access_token = cls.provider._api_client.account_api.session._access_token
 
     def setUp(self):
         """Initial test setup."""
@@ -174,7 +174,7 @@ def setUpClass(cls, provider):
         # pylint: disable=arguments-differ
         super().setUpClass()
         cls.provider = provider
-        cls.access_token = cls.provider._api_client.account_api.session.access_token
+        cls.access_token = cls.provider._api_client.account_api.session._access_token
 
         backend_name = 'ibmq_qasm_simulator'
         backend = cls.provider.get_backend(backend_name)
@@ -228,7 +228,7 @@ class TestAuthClient(IBMQTestCase):
     def test_valid_login(self, qe_token, qe_url):
         """Test valid authentication."""
         client = AuthClient(qe_token, qe_url)
-        self.assertTrue(client.base_api.session.access_token)
+        self.assertTrue(client.base_api.session._access_token)
 
     @requires_qe_access
     def test_url_404(self, qe_token, qe_url):
diff --git a/test/ibmq/test_runtime.py b/test/ibmq/test_runtime.py
index 24ba303d9..b9557a4da 100644
--- a/test/ibmq/test_runtime.py
+++ b/test/ibmq/test_runtime.py
@@ -13,6 +13,9 @@
 """Tests for runtime service."""
 
 import unittest
+import os
+from io import StringIO
+from unittest.mock import patch
 
 from qiskit.providers.jobstatus import JobStatus
 
@@ -61,6 +64,7 @@ def _callback(interim_result):
         job.result()
 
 
+@unittest.skipIf(not os.environ.get('USE_STAGING_CREDENTIALS', ''), "Only runs on staging")
 class TestRuntimeIntegration(IBMQTestCase):
 
     @classmethod
@@ -80,4 +84,46 @@ def test_list_programs(self):
         programs = self.provider.runtime.programs()
         self.assertTrue(programs)
         for prog in programs:
-            self.assertTrue(prog.name)
+            self._validate_program(prog)
+
+    def test_list_program(self):
+        """Test listing a single program."""
+        program = self.provider.runtime.programs()[0]
+        self._validate_program(program)
+
+    def test_print_programs(self):
+        """Test printing programs."""
+        programs = self.provider.runtime.programs()
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            self.provider.runtime.pprint_programs()
+            for prog in programs:
+                self.assertIn(prog.program_id, mock_stdout)
+                self.assertIn(prog.name, mock_stdout)
+                self.assertIn(prog.description, mock_stdout)
+
+    def test_upload_program(self):
+        """Test uploading a program."""
+        pass
+
+    def test_upload_program_conflict(self):
+        """Test uploading a program with conflicting name."""
+        pass
+
+    def test_upload_program_missing(self):
+        pass
+
+    def test_execute_program(self):
+        pass
+
+    def test_execute_program_bad_params(self):
+        pass
+
+    def test_execute_program_failed(self):
+        pass
+
+    def _validate_program(self, program):
+        self.assertTrue(program)
+        self.assertTrue(program.name)
+        self.assertTrue(program.program_id)
+        self.assertTrue(program.description)
+        self.assertTrue(program.cost)
diff --git a/test/ibmq/websocket/test_websocket_integration.py b/test/ibmq/websocket/test_websocket_integration.py
index 54a7a50f6..1dd763918 100644
--- a/test/ibmq/websocket/test_websocket_integration.py
+++ b/test/ibmq/websocket/test_websocket_integration.py
@@ -101,16 +101,16 @@ def test_websockets_retry_bad_url(self):
         """Test http retry after websocket error due to an invalid URL."""
         job = self.sim_backend.run(self.bell)
 
-        saved_websocket_url = job._api_client.client_ws.websocket_url
+        saved_websocket_url = job._api_client.client_ws._websocket_url
         try:
             # Use fake websocket address.
-            job._api_client.client_ws.websocket_url = 'wss://wss.localhost'
+            job._api_client.client_ws._websocket_url = 'wss://wss.localhost'
 
             # _wait_for_completion() should retry with http successfully
             # after getting websockets error.
             job._wait_for_completion()
         finally:
-            job._api_client.client_ws.websocket_url = saved_websocket_url
+            job._api_client.client_ws._websocket_url = saved_websocket_url
 
         self.assertIs(job._status, JobStatus.DONE)
 

From 2c62d6f30f52c4c35ff6ed0fdef8007841882c2f Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Fri, 9 Apr 2021 19:20:22 -0400
Subject: [PATCH 25/59] move json coder

---
 qiskit/providers/ibmq/api/clients/runtime.py         |  5 ++++-
 qiskit/providers/ibmq/api/rest/runtime.py            | 12 +++++++-----
 qiskit/providers/ibmq/runtime/ibm_runtime_service.py |  4 +++-
 3 files changed, 14 insertions(+), 7 deletions(-)

diff --git a/qiskit/providers/ibmq/api/clients/runtime.py b/qiskit/providers/ibmq/api/clients/runtime.py
index 2e662fc33..ab3fe929c 100644
--- a/qiskit/providers/ibmq/api/clients/runtime.py
+++ b/qiskit/providers/ibmq/api/clients/runtime.py
@@ -56,17 +56,20 @@ def program_create(
             self,
             program_name: str,
             program_data: Union[bytes, str],
+            max_execution_time: int
     ) -> Dict:
         """Create a new program.
 
         Args:
             program_name: Name of the program.
             program_data: Program data.
+            max_execution_time: Maximum execution time.
 
         Returns:
             Server response.
         """
-        return self.api.create_program(program_name=program_name, program_data=program_data)
+        return self.api.create_program(program_name=program_name, program_data=program_data,
+                                       max_execution_time=max_execution_time)
 
     def program_get(self, program_id: str) -> Dict:
         """Return a specific program.
diff --git a/qiskit/providers/ibmq/api/rest/runtime.py b/qiskit/providers/ibmq/api/rest/runtime.py
index 475c501ab..149b9f0c0 100644
--- a/qiskit/providers/ibmq/api/rest/runtime.py
+++ b/qiskit/providers/ibmq/api/rest/runtime.py
@@ -57,26 +57,28 @@ def list_programs(self) -> List[Dict]:
     def create_program(
             self,
             program_name: str,
-            program_data: Union[bytes, str]
+            program_data: Union[bytes, str],
+            max_execution_time: int
     ) -> Dict:
         """Upload a new program.
 
         Args:
             program_name: Name of the program.
             program_data: Program data.
+            max_execution_time: Maximum execution time.
 
         Returns:
             JSON response.
         """
         url = self.get_url('programs')
+        data = {'name': (None, program_name),
+                'cost': (None, str(max_execution_time))}
         if isinstance(program_data, str):
             with open(program_data, 'rb') as file:
-                data = {'name': (None, program_name),
-                        'program': (program_name, file)}  # type: ignore[dict-item]
+                data['program'] = (program_name, file)
                 response = self.session.post(url, files=data).json()
         else:
-            data = {'name': (None, program_name),
-                    'program': (program_name, program_data)}  # type: ignore[dict-item]
+            data['program'] = (program_name, program_data)
             response = self.session.post(url, files=data).json()
         return response
 
diff --git a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
index 08e4f091e..c9abb1a21 100644
--- a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
+++ b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
@@ -145,17 +145,19 @@ def upload_program(
             self,
             name: str,
             data: Union[bytes, str],
+            max_execution_time: int = 0
     ) -> str:
         """Upload a runtime program.
 
         Args:
             name: Name of the program.
             data: Name of the program file or program data to upload.
+            max_execution_time: Maximum execution time.
 
         Returns:
             Program ID.
         """
-        response = self._api_client.program_create(name, data)
+        response = self._api_client.program_create(name, data, max_execution_time)
         return response['id']
 
     def delete_program(self, program_id: str):

From a2f6c9a545b7d1e35188ed15983e793f70f989c2 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Fri, 9 Apr 2021 19:22:44 -0400
Subject: [PATCH 26/59] clarify docstring

---
 qiskit/providers/ibmq/runtime/ibm_runtime_service.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
index c9abb1a21..3afc87ad2 100644
--- a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
+++ b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
@@ -152,7 +152,7 @@ def upload_program(
         Args:
             name: Name of the program.
             data: Name of the program file or program data to upload.
-            max_execution_time: Maximum execution time.
+            max_execution_time: Maximum execution time in seconds.
 
         Returns:
             Program ID.

From d0c2f23617bde6b30560f3c854478e53c7724f5f Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Fri, 9 Apr 2021 19:27:10 -0400
Subject: [PATCH 27/59] return program max_execution_time

---
 qiskit/providers/ibmq/runtime/runtime_program.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/qiskit/providers/ibmq/runtime/runtime_program.py b/qiskit/providers/ibmq/runtime/runtime_program.py
index e0c5851fb..2afcdd67e 100644
--- a/qiskit/providers/ibmq/runtime/runtime_program.py
+++ b/qiskit/providers/ibmq/runtime/runtime_program.py
@@ -45,7 +45,7 @@ def __init__(
             parameters: Optional[List] = None,
             return_values: Optional[List] = None,
             interim_results: Optional[List] = None,
-            max_execution_time: float = 0
+            max_execution_time: int = 0
     ) -> None:
         """RuntimeProgram constructor.
 
@@ -61,7 +61,7 @@ def __init__(
         self._name = program_name
         self._id = program_id
         self._description = description
-        self._cost = max_execution_time
+        self._max_execution_time = max_execution_time
         self._parameters = []
         self._return_values = []
         self._interim_results = []
@@ -171,6 +171,15 @@ def interim_results(self) -> List['ProgramResult']:
         """
         return self._interim_results
 
+    @property
+    def max_execution_time(self) -> int:
+        """Return maximum execution time.
+
+        Returns:
+            Maximum execution time.
+        """
+        return self._max_execution_time
+
 
 class ProgramParameter(NamedTuple):
     """Program parameter."""

From caf8f9505b1842f9d36680146db59d2bb8f7894e Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Mon, 12 Apr 2021 10:35:50 -0400
Subject: [PATCH 28/59] make max_execution_time required

---
 qiskit/providers/ibmq/runtime/ibm_runtime_service.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
index 3afc87ad2..d2c463c38 100644
--- a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
+++ b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
@@ -145,7 +145,7 @@ def upload_program(
             self,
             name: str,
             data: Union[bytes, str],
-            max_execution_time: int = 0
+            max_execution_time: int
     ) -> str:
         """Upload a runtime program.
 

From 2f936050f946b9fadac6058080b850379dab4bce Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Tue, 13 Apr 2021 09:05:04 -0400
Subject: [PATCH 29/59] get runtime url from server

---
 qiskit/providers/ibmq/accountprovider.py      | 19 ++++++++++++-------
 qiskit/providers/ibmq/api/clients/runtime.py  |  5 +----
 .../providers/ibmq/credentials/credentials.py |  1 +
 3 files changed, 14 insertions(+), 11 deletions(-)

diff --git a/qiskit/providers/ibmq/accountprovider.py b/qiskit/providers/ibmq/accountprovider.py
index 25753e9d6..51c87c808 100644
--- a/qiskit/providers/ibmq/accountprovider.py
+++ b/qiskit/providers/ibmq/accountprovider.py
@@ -22,12 +22,9 @@
                                      PulseBackendConfiguration)
 from qiskit.circuit import QuantumCircuit, Parameter
 from qiskit.pulse.instruction_schedule_map import InstructionScheduleMap
-from qiskit.pulse.channels import PulseChannel
 from qiskit.providers.backend import BackendV1 as Backend
 from qiskit.providers.basebackend import BaseBackend
 from qiskit.transpiler import Layout
-from qiskit.qobj.utils import MeasLevel, MeasReturnType
-from qiskit.exceptions import QiskitError
 
 from .api.clients import AccountClient
 from .ibmqbackend import IBMQBackend, IBMQSimulator
@@ -122,11 +119,13 @@ def __init__(self, credentials: Credentials, access_token: str) -> None:
             if credentials.extractor_url else None
         self._experiment = ExperimentService(self, access_token) \
             if credentials.experiment_url else None
-        self._runtime = IBMRuntimeService(self, access_token)
+        self._runtime = IBMRuntimeService(self, access_token) \
+            if credentials.runtime_url else None
 
         self._services = {'backend': self._backend,
                           'random': self._random,
-                          'experiment': self._experiment}
+                          'experiment': self._experiment,
+                          'runtime': self._runtime}
 
     def backends(
             self,
@@ -381,7 +380,7 @@ def random(self) -> IBMQRandomService:
         if self._random:
             return self._random
         else:
-            raise IBMQNotAuthorizedError("You are not authorized to use the random number service.")
+            raise IBMQNotAuthorizedError("You are not authorized to use the service.")
 
     @property
     def runtime(self) -> IBMRuntimeService:
@@ -389,8 +388,14 @@ def runtime(self) -> IBMRuntimeService:
 
         Returns:
             The runtime service instance.
+
+        Raises:
+            IBMQNotAuthorizedError: If the account is not authorized to use the service.
         """
-        return self._runtime
+        if self._runtime:
+            return self._runtime
+        else:
+            raise IBMQNotAuthorizedError("You are not authorized to use the runtime service.")
 
     def __eq__(
             self,
diff --git a/qiskit/providers/ibmq/api/clients/runtime.py b/qiskit/providers/ibmq/api/clients/runtime.py
index ab3fe929c..8510c53d3 100644
--- a/qiskit/providers/ibmq/api/clients/runtime.py
+++ b/qiskit/providers/ibmq/api/clients/runtime.py
@@ -12,7 +12,6 @@
 
 """Client for accessing IBM Quantum runtime service."""
 
-import os
 import logging
 from typing import List, Dict, Union
 
@@ -38,9 +37,7 @@ def __init__(
             access_token: IBM Quantum Experience access token.
             credentials: Account credentials.
         """
-        url = os.getenv('NTC_URL', "")
-        logger.debug(f"Using runtime service url {url}")
-        self._session = RetrySession(url, access_token,
+        self._session = RetrySession(credentials.runtime_url, access_token,
                                      **credentials.connection_parameters())
         self.api = Runtime(self._session)
 
diff --git a/qiskit/providers/ibmq/credentials/credentials.py b/qiskit/providers/ibmq/credentials/credentials.py
index 4eaf2d427..021eaabbb 100644
--- a/qiskit/providers/ibmq/credentials/credentials.py
+++ b/qiskit/providers/ibmq/credentials/credentials.py
@@ -75,6 +75,7 @@ def __init__(
         services = services or {}
         self.extractor_url = services.get('extractorsService', None)
         self.experiment_url = services.get('resultsDB', None)
+        self.runtime_url = services.get('runtime', None)
 
     def is_ibmq(self) -> bool:
         """Return whether the credentials represent an IBM Quantum Experience account."""

From efbf958ef140e19b30b3a38c1a77209614487c60 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Wed, 14 Apr 2021 09:26:05 -0400
Subject: [PATCH 30/59] update ws

---
 qiskit/providers/ibmq/accountprovider.py      |  14 +--
 qiskit/providers/ibmq/api/clients/account.py  |   7 +-
 .../providers/ibmq/api/clients/experiment.py  |   4 +-
 qiskit/providers/ibmq/api/clients/random.py   |   5 +-
 qiskit/providers/ibmq/api/clients/runtime.py  |   4 +-
 .../providers/ibmq/api/clients/runtime_ws.py  |  62 +++++-----
 qiskit/providers/ibmq/api/exceptions.py       |   5 +
 .../providers/ibmq/credentials/credentials.py |   5 +-
 .../ibmq/experiment/experimentservice.py      |   6 +-
 qiskit/providers/ibmq/ibmqfactory.py          |   4 +-
 .../ibmq/random/ibmqrandomservice.py          |   5 +-
 .../ibmq/runtime/ibm_runtime_service.py       |  15 ++-
 qiskit/providers/ibmq/runtime/runtime_job.py  | 108 ++++++++++--------
 13 files changed, 128 insertions(+), 116 deletions(-)

diff --git a/qiskit/providers/ibmq/accountprovider.py b/qiskit/providers/ibmq/accountprovider.py
index 51c87c808..1cba1cb28 100644
--- a/qiskit/providers/ibmq/accountprovider.py
+++ b/qiskit/providers/ibmq/accountprovider.py
@@ -95,18 +95,16 @@ class AccountProvider(Provider):
         in Jupyter Notebook and the Python interpreter.
     """
 
-    def __init__(self, credentials: Credentials, access_token: str) -> None:
+    def __init__(self, credentials: Credentials) -> None:
         """AccountProvider constructor.
 
         Args:
             credentials: IBM Quantum Experience credentials.
-            access_token: IBM Quantum Experience access token.
         """
         super().__init__()
 
         self.credentials = credentials
-        self._api_client = AccountClient(access_token,
-                                         credentials,
+        self._api_client = AccountClient(credentials,
                                          **credentials.connection_parameters())
 
         # Initialize the internal list of backends.
@@ -115,11 +113,9 @@ def __init__(self, credentials: Credentials, access_token: str) -> None:
         self.backends = IBMQDeprecatedBackendService(self.backend)  # type: ignore[assignment]
 
         # Initialize other services.
-        self._random = IBMQRandomService(self, access_token) \
-            if credentials.extractor_url else None
-        self._experiment = ExperimentService(self, access_token) \
-            if credentials.experiment_url else None
-        self._runtime = IBMRuntimeService(self, access_token) \
+        self._random = IBMQRandomService(self) if credentials.extractor_url else None
+        self._experiment = ExperimentService(self) if credentials.experiment_url else None
+        self._runtime = IBMRuntimeService(self) \
             if credentials.runtime_url else None
 
         self._services = {'backend': self._backend,
diff --git a/qiskit/providers/ibmq/api/clients/account.py b/qiskit/providers/ibmq/api/clients/account.py
index e19551355..8700678b2 100644
--- a/qiskit/providers/ibmq/api/clients/account.py
+++ b/qiskit/providers/ibmq/api/clients/account.py
@@ -41,24 +41,23 @@ class AccountClient(BaseClient):
 
     def __init__(
             self,
-            access_token: str,
             credentials: Credentials,
             **request_kwargs: Any
     ) -> None:
         """AccountClient constructor.
 
         Args:
-            access_token: IBM Quantum Experience access token.
             credentials: Account credentials.
             **request_kwargs: Arguments for the request ``Session``.
         """
-        self._session = RetrySession(credentials.base_url, access_token, **request_kwargs)
+        self._session = RetrySession(
+            credentials.base_url, credentials.access_token, **request_kwargs)
         # base_api is used to handle endpoints that don't include h/g/p.
         # account_api is for h/g/p.
         self.base_api = Api(self._session)
         self.account_api = Account(session=self._session, hub=credentials.hub,
                                    group=credentials.group, project=credentials.project)
-        self.client_ws = WebsocketClient(credentials.websockets_url, access_token)
+        self.client_ws = WebsocketClient(credentials.websockets_url, credentials.access_token)
         self._use_websockets = (not credentials.proxies)
 
     # Backend-related public functions.
diff --git a/qiskit/providers/ibmq/api/clients/experiment.py b/qiskit/providers/ibmq/api/clients/experiment.py
index d59e3eb79..372f6a452 100644
--- a/qiskit/providers/ibmq/api/clients/experiment.py
+++ b/qiskit/providers/ibmq/api/clients/experiment.py
@@ -29,16 +29,14 @@ class ExperimentClient(BaseClient):
 
     def __init__(
             self,
-            access_token: str,
             credentials: Credentials
     ) -> None:
         """ExperimentClient constructor.
 
         Args:
-            access_token: IBM Quantum Experience access token.
             credentials: Account credentials.
         """
-        self._session = RetrySession(credentials.experiment_url, access_token,
+        self._session = RetrySession(credentials.experiment_url, credentials.access_token,
                                      **credentials.connection_parameters())
         self.base_api = Api(self._session)
 
diff --git a/qiskit/providers/ibmq/api/clients/random.py b/qiskit/providers/ibmq/api/clients/random.py
index 9df4200e9..202d549c7 100644
--- a/qiskit/providers/ibmq/api/clients/random.py
+++ b/qiskit/providers/ibmq/api/clients/random.py
@@ -28,16 +28,14 @@ class RandomClient:
 
     def __init__(
             self,
-            access_token: str,
             credentials: Credentials,
     ) -> None:
         """RandomClient constructor.
 
         Args:
-            access_token: IBM Quantum Experience access token.
             credentials: Account credentials.
         """
-        self._session = RetrySession(credentials.extractor_url, access_token,
+        self._session = RetrySession(credentials.extractor_url, credentials.access_token,
                                      **credentials.connection_parameters())
         self.random_api = Random(self._session)
 
@@ -47,7 +45,6 @@ def list_services(self) -> List[Dict[str, Any]]:
         Returns:
             RNG services available for this provider.
         """
-        # return [{'name': 'cqc', 'extractors': ['ext1', 'ext2']}]
         return self.random_api.list_services()
 
     def extract(
diff --git a/qiskit/providers/ibmq/api/clients/runtime.py b/qiskit/providers/ibmq/api/clients/runtime.py
index 8510c53d3..06bf6ce2b 100644
--- a/qiskit/providers/ibmq/api/clients/runtime.py
+++ b/qiskit/providers/ibmq/api/clients/runtime.py
@@ -28,16 +28,14 @@ class RuntimeClient:
 
     def __init__(
             self,
-            access_token: str,
             credentials: Credentials,
     ) -> None:
         """RandomClient constructor.
 
         Args:
-            access_token: IBM Quantum Experience access token.
             credentials: Account credentials.
         """
-        self._session = RetrySession(credentials.runtime_url, access_token,
+        self._session = RetrySession(credentials.runtime_url, credentials.access_token,
                                      **credentials.connection_parameters())
         self.api = Runtime(self._session)
 
diff --git a/qiskit/providers/ibmq/api/clients/runtime_ws.py b/qiskit/providers/ibmq/api/clients/runtime_ws.py
index 843df23c4..59e52f14c 100644
--- a/qiskit/providers/ibmq/api/clients/runtime_ws.py
+++ b/qiskit/providers/ibmq/api/clients/runtime_ws.py
@@ -14,14 +14,16 @@
 
 import logging
 import asyncio
-from typing import Optional, Any
+from typing import Any
 from ssl import SSLError
+import queue
+import traceback
 
 from websockets import connect, ConnectionClosed
 from websockets.client import WebSocketClientProtocol
 from websockets.exceptions import InvalidURI
 
-from ..exceptions import WebsocketError, WebsocketTimeoutError
+from ..exceptions import WebsocketError, WebsocketRetryableError
 
 logger = logging.getLogger(__name__)
 
@@ -64,20 +66,16 @@ async def _connect(self, url: str) -> WebSocketClientProtocol:
         """
         try:
             logger.debug('Starting new websocket connection: %s', url)
-            # TODO: Re-enable ping_timeout when server is fixed.
-            websocket = await connect(url, extra_headers=self._header, ping_interval=None)
+            websocket = await connect(url, extra_headers=self._header)
             await websocket.recv()  # Ack from server
 
-        # Isolate specific exceptions, so they are not retried in `get_job_status`.
+        # Isolate specific exceptions, so they are not retried.
         except (SSLError, InvalidURI) as ex:
             raise ex
 
         # pylint: disable=broad-except
         except Exception as ex:
-            exception_to_raise = WebsocketError('Failed to connect to the server.')
-
-            logger.info('An exception occurred. Raising "%s" from "%s"',
-                        repr(exception_to_raise), repr(ex))
+            exception_to_raise = WebsocketRetryableError('Failed to connect to the server.')
             raise exception_to_raise from ex
 
         logger.debug("Runtime websocket connection established.")
@@ -86,7 +84,7 @@ async def _connect(self, url: str) -> WebSocketClientProtocol:
     async def job_results(
             self,
             job_id: str,
-            timeout: Optional[float] = 5,
+            result_queue: queue.Queue,
             max_retries: int = 5,
             backoff_factor: float = 0.5
     ) -> Any:
@@ -94,7 +92,7 @@ async def job_results(
 
         Args:
             job_id: ID of the job.
-            timeout: Timeout value, in seconds.
+            result_queue: Queue used to hold response received from the server.
             max_retries: Max number of retries.
             backoff_factor: Backoff factor used to calculate the
                 time to wait between retries.
@@ -103,8 +101,7 @@ async def job_results(
             The interim result of a job.
 
         Raises:
-            WebsocketError: If the websocket connection ended unexpectedly.
-            WebsocketTimeoutError: If the timeout has been reached.
+            WebsocketError: If a websocket error occurred.
         """
         url = '{}/stream/jobs/{}'.format(self._websocket_url, job_id)
 
@@ -114,29 +111,34 @@ async def job_results(
             try:
                 if self._ws is None:
                     self._ws = await self._connect(url)
-
-                response = await asyncio.wait_for(self._ws.recv(), timeout=timeout)
-                return response
-
-            except asyncio.TimeoutError:
-                # Timeout during our wait.
-                raise WebsocketTimeoutError(
-                    'Timeout reached while streaming job results.') from None
-            except (WebsocketError, ConnectionClosed) as ex:
-                if isinstance(ex, ConnectionClosed):
-                    self._ws = None
-
-                logger.debug(
-                    f"A websocket error occurred while streaming runtime job result: {str(ex)}")
+                while True:
+                    try:
+                        response = await self._ws.recv()
+                        result_queue.put_nowait(response)
+                        current_retry = 0  # Reset counter after a good receive.
+                    except ConnectionClosed as ex:
+                        self._ws = None
+                        if ex.code == 1000:  # Job has finished.
+                            return
+                        exception_to_raise = WebsocketRetryableError(
+                            f"Connection with websocket for job {job_id} "
+                            f"closed unexpectedly: {ex.code}")
+                        raise exception_to_raise
+
+            except WebsocketRetryableError as ex:
+                logger.debug(f"A websocket error occurred while streaming "
+                             f"results for runtime job {job_id}:\n{traceback.format_exc()}")
                 current_retry += 1
                 if current_retry > max_retries:
                     raise ex
 
-                # Sleep, and then `continue` with retrying.
                 backoff_time = self._backoff_time(backoff_factor, current_retry)
-                logger.info('Retrying websocket after %s seconds: '
-                            'Attempt #%s', backoff_time, current_retry)
+                logger.info(f"Retrying websocket after {backoff_time} seconds. "
+                            f"Attemp {current_retry}")
                 await asyncio.sleep(backoff_time)  # Block asyncio loop for given backoff time.
+                continue  # Continues next iteration after `finally` block.
+            finally:
+                await self.disconnect()
 
         # Execution should not reach here, sanity check.
         exception_message = 'Max retries exceeded: Failed to establish a websocket ' \
diff --git a/qiskit/providers/ibmq/api/exceptions.py b/qiskit/providers/ibmq/api/exceptions.py
index 2cc26e842..b2072c2f0 100644
--- a/qiskit/providers/ibmq/api/exceptions.py
+++ b/qiskit/providers/ibmq/api/exceptions.py
@@ -54,6 +54,11 @@ class WebsocketTimeoutError(WebsocketError):
     pass
 
 
+class WebsocketRetryableError(WebsocketError):
+    """A websocket error that can be retried."""
+    pass
+
+
 class AuthenticationLicenseError(ApiError):
     """Exception due to user not having accepted the license agreement."""
     pass
diff --git a/qiskit/providers/ibmq/credentials/credentials.py b/qiskit/providers/ibmq/credentials/credentials.py
index 021eaabbb..9b8455db9 100644
--- a/qiskit/providers/ibmq/credentials/credentials.py
+++ b/qiskit/providers/ibmq/credentials/credentials.py
@@ -48,7 +48,8 @@ def __init__(
             project: Optional[str] = None,
             proxies: Optional[Dict] = None,
             verify: bool = True,
-            services: Optional[Dict] = None
+            services: Optional[Dict] = None,
+            access_token: Optional[str] = None
     ) -> None:
         """Credentials constructor.
 
@@ -62,8 +63,10 @@ def __init__(
             proxies: Proxy configuration.
             verify: If ``False``, ignores SSL certificates errors.
             services: Additional services for this account.
+            access_token: IBM Quantum access token.
         """
         self.token = token
+        self.access_token = access_token
         (self.url, self.base_url,
          self.hub, self.group, self.project) = _unify_ibmq_url(
              url, hub, group, project)
diff --git a/qiskit/providers/ibmq/experiment/experimentservice.py b/qiskit/providers/ibmq/experiment/experimentservice.py
index 39fa98ff2..bec6858fd 100644
--- a/qiskit/providers/ibmq/experiment/experimentservice.py
+++ b/qiskit/providers/ibmq/experiment/experimentservice.py
@@ -73,19 +73,17 @@ class ExperimentService:
 
     def __init__(
             self,
-            provider: 'accountprovider.AccountProvider',
-            access_token: str
+            provider: 'accountprovider.AccountProvider'
     ) -> None:
         """IBMQBackendService constructor.
 
         Args:
             provider: IBM Quantum Experience account provider.
-            access_token: IBM Quantum Experience access token.
         """
         super().__init__()
 
         self._provider = provider
-        self._api_client = ExperimentClient(access_token, provider.credentials)
+        self._api_client = ExperimentClient(provider.credentials)
 
     def backends(self) -> List[Dict]:
         """Return a list of backends.
diff --git a/qiskit/providers/ibmq/ibmqfactory.py b/qiskit/providers/ibmq/ibmqfactory.py
index d6eeb0726..c3102f770 100644
--- a/qiskit/providers/ibmq/ibmqfactory.py
+++ b/qiskit/providers/ibmq/ibmqfactory.py
@@ -461,6 +461,7 @@ def _initialize_providers(self, credentials: Credentials) -> None:
             # Build credentials.
             provider_credentials = Credentials(
                 credentials.token,
+                access_token=auth_client.current_access_token(),
                 url=service_urls['http'],
                 websockets_url=service_urls['ws'],
                 proxies=credentials.proxies,
@@ -470,8 +471,7 @@ def _initialize_providers(self, credentials: Credentials) -> None:
 
             # Build the provider.
             try:
-                provider = AccountProvider(provider_credentials,
-                                           auth_client.current_access_token())
+                provider = AccountProvider(provider_credentials)
                 self._providers[provider_credentials.unique_id()] = provider
             except Exception:  # pylint: disable=broad-except
                 # Catch-all for errors instantiating the provider.
diff --git a/qiskit/providers/ibmq/random/ibmqrandomservice.py b/qiskit/providers/ibmq/random/ibmqrandomservice.py
index 424415337..67f5a9255 100644
--- a/qiskit/providers/ibmq/random/ibmqrandomservice.py
+++ b/qiskit/providers/ibmq/random/ibmqrandomservice.py
@@ -39,16 +39,15 @@ class IBMQRandomService:
         extractor = provider.random.cqc_extractor  # Short hand for above.
     """
 
-    def __init__(self, provider: 'accountprovider.AccountProvider', access_token: str) -> None:
+    def __init__(self, provider: 'accountprovider.AccountProvider') -> None:
         """IBMQRandomService constructor.
 
         Args:
             provider: IBM Quantum Experience account provider.
-            access_token: IBM Quantum Experience access token.
         """
         self._provider = provider
         if provider.credentials.extractor_url:
-            self._random_client = RandomClient(access_token, provider.credentials)
+            self._random_client = RandomClient(provider.credentials)
             self._initialized = False
         else:
             self._random_client = None
diff --git a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
index d2c463c38..d66a481aa 100644
--- a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
+++ b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
@@ -23,6 +23,7 @@
 from .runtime_program import RuntimeProgram
 from .utils import RuntimeEncoder, RuntimeDecoder
 from ..api.clients.runtime import RuntimeClient
+from ..api.clients.runtime_ws import RuntimeWebsocketClient
 
 logger = logging.getLogger(__name__)
 
@@ -30,16 +31,16 @@
 class IBMRuntimeService:
     """Class for interacting with the IBM Quantum runtime service."""
 
-    def __init__(self, provider: 'accountprovider.AccountProvider', access_token: str) -> None:
+    def __init__(self, provider: 'accountprovider.AccountProvider') -> None:
         """IBMRuntimeService constructor.
 
         Args:
             provider: IBM Quantum account provider.
-            access_token: IBM Quantum access token.
         """
         self._provider = provider
-        self._api_client = RuntimeClient(access_token, provider.credentials)
-        self._access_token = access_token
+        self._api_client = RuntimeClient(provider.credentials)
+        self._access_token = provider.credentials.access_token
+        self._ws_url = provider.credentials.runtime_url.replace('https', 'wss')
         self._programs = {}
 
     def pprint_programs(self, refresh: bool = False) -> None:
@@ -136,7 +137,8 @@ def run(
 
         backend = self._provider.get_backend(backend_name)
         job = RuntimeJob(backend=backend,
-                         api_client=self._api_client, access_token=self._access_token,
+                         api_client=self._api_client,
+                         ws_client=RuntimeWebsocketClient(self._ws_url, self._access_token),
                          job_id=response['id'], program_id=program_id, params=inputs,
                          user_callback=callback)
         return job
@@ -182,7 +184,8 @@ def job(self, job_id: str):
         params_str = json.dumps(response.get('params', {}))
         params = json.loads(params_str, cls=RuntimeDecoder)
         return RuntimeJob(backend=backend,
-                          api_client=self._api_client, access_token=self._access_token,
+                          api_client=self._api_client,
+                          ws_client=RuntimeWebsocketClient(self._ws_url, self._access_token),
                           job_id=response['id'],
                           program_id=response.get('program', {}).get('id', ""),
                           params=params)
diff --git a/qiskit/providers/ibmq/runtime/runtime_job.py b/qiskit/providers/ibmq/runtime/runtime_job.py
index 91099ef23..9c00fd042 100644
--- a/qiskit/providers/ibmq/runtime/runtime_job.py
+++ b/qiskit/providers/ibmq/runtime/runtime_job.py
@@ -19,8 +19,9 @@
 import asyncio
 from concurrent import futures
 import traceback
-import os
+import queue
 
+from qiskit.exceptions import QiskitError
 from qiskit.providers.exceptions import JobTimeoutError
 from qiskit.providers.backend import Backend
 from qiskit.providers.jobstatus import JobStatus, JOB_FINAL_STATES
@@ -38,12 +39,13 @@ class RuntimeJob:
     """Representation of a runtime program execution."""
 
     _executor = futures.ThreadPoolExecutor()
+    _result_queue_poison_pill = "_poison_pill"
 
     def __init__(
             self,
             backend: 'ibmqbackend.IBMQBackend',
             api_client: RuntimeClient,
-            access_token: str,
+            ws_client: RuntimeWebsocketClient,
             job_id: str,
             program_id: str,
             params: Optional[Dict] = None,
@@ -54,7 +56,7 @@ def __init__(
         Args:
             backend: The backend instance used to run this job.
             api_client: Object for connecting to the server.
-            access_token: IBM Quantum Experience access token.
+            ws_client: Object for connecting to the server via websocket.
             job_id: Job ID.
             program_id: ID of the program this job is for.
             params: Job parameters.
@@ -63,14 +65,14 @@ def __init__(
         self._job_id = job_id
         self._backend = backend
         self._api_client = api_client
-        url = os.getenv('NTC_URL', "")
-        ws_url = url.replace('https', 'wss')
-        self._ws_client = RuntimeWebsocketClient(ws_url, access_token)
-        self._results = []
+        self._ws_client = ws_client
+        self._results = None
         self._params = params or {}
         # self._user_callback = user_callback
         self._program_id = program_id
         self._status = JobStatus.INITIALIZING
+        self._streaming = False
+        self._result_queue = queue.Queue()
 
         if user_callback is not None:
             self.stream_results(user_callback)
@@ -78,8 +80,7 @@ def __init__(
     def result(
             self,
             timeout: Optional[float] = None,
-            wait: float = 5,
-            include_interim: bool = False
+            wait: float = 5
     ) -> Any:
         """Return the results of the job.
 
@@ -90,7 +91,6 @@ def result(
         Args:
             timeout: Number of seconds to wait for job.
             wait: Seconds between queries.
-            include_interim: Whether to include interim results.
 
         Returns:
             Runtime job result.
@@ -104,15 +104,8 @@ def result(
             if self._status == JobStatus.ERROR:
                 raise IBMQJobFailureError(f"Unable to retrieve result for job {self.job_id()}. "
                                           f"Job has failed:\n{result_raw}")
-            # TODO - Update when interim results are for streaming only
-            result_list = result_raw.split('\n')
-            for res in result_list:
-                if not res:
-                    continue
-                self._results.append(self._decode_data(res))
-        if include_interim:
-            return self._results
-        return self._results[-1]
+            self._results = self._decode_data(result_raw)
+        return self._results
 
     def cancel(self):
         """Attempt to cancel the job."""
@@ -155,7 +148,7 @@ def wait_for_final_state(
             status = self.status()
         return
 
-    def stream_results(self, callback: Callable):
+    def stream_results(self, callback: Callable) -> None:
         """Start streaming job results.
 
         Args:
@@ -164,40 +157,61 @@ def stream_results(self, callback: Callable):
                     1. Job ID
                     2. Job interim result.
         """
-        self._executor.submit(self._stream_results, user_callback=callback)
+        if self._streaming:
+            raise QiskitError("A callback function is already streaming results.")
+        self._streaming = True
+
+        self._executor.submit(self._start_websocket_client,
+                              result_queue=self._result_queue)
+        self._executor.submit(self._stream_results,
+                              result_queue=self._result_queue, user_callback=callback)
 
-    def _stream_results(self, user_callback: Callable) -> None:
+    def _start_websocket_client(
+            self,
+            result_queue: queue.Queue
+    ) -> None:
+        """Start websocket client to stream results."""
+        loop = None
+        try:
+            try:
+                loop = asyncio.get_event_loop()
+            except RuntimeError as ex:
+                # Event loop may not be set in a child thread.
+                if 'There is no current event loop' in str(ex):
+                    loop = asyncio.new_event_loop()
+                    asyncio.set_event_loop(loop)
+                else:
+                    logger.warning(f"Unable to get the event loop: {ex}")
+                    raise
+
+            logger.debug(f"Start websocket client for job {self.job_id()}")
+            loop.run_until_complete(self._ws_client.job_results(self._job_id, result_queue))
+        except Exception:  # pylint: disable=broad-except
+            logger.warning(
+                f"An error occurred while streaming results "
+                f"from the server for job {self.job_id()}:\n{traceback.format_exc()}")
+        finally:
+            result_queue.put_nowait(self._result_queue_poison_pill)
+            if loop is not None:
+                loop.run_until_complete(self._ws_client.disconnect())
+
+    def _stream_results(self, result_queue: queue.Queue, user_callback: Callable) -> None:
         """Stream interim results.
 
         Args:
             user_callback: User callback function.
         """
-        try:
-            loop = asyncio.get_event_loop()
-        except RuntimeError as ex:
-            # Event loop may not be set in a child thread.
-            if 'There is no current event loop' in str(ex):
-                loop = asyncio.new_event_loop()
-                asyncio.set_event_loop(loop)
-            else:
-                raise
-
         logger.debug(f"Start result streaming for job {self.job_id()}")
-        try:
-            while True:
-                try:
-                    response = loop.run_until_complete(self._ws_client.job_results(self._job_id))
-                    user_callback(self.job_id(), self._decode_data(response))
-                except WebsocketTimeoutError:
-                    if self.status() in JOB_FINAL_STATES:
-                        logger.debug(f"Job {self.job_id()} finished, stop result streaming.")
-                        return
-        except Exception:  # pylint: disable=broad-except
-            logger.warning(
-                f"An error occurred while streaming results for job {self.job_id()}: " +
-                traceback.format_exc())
-        finally:
-            loop.run_until_complete(self._ws_client.disconnect())
+        while True:
+            try:
+                response = result_queue.get()
+                if response == self._result_queue_poison_pill:
+                    return
+                user_callback(self.job_id(), self._decode_data(response))
+            except Exception:  # pylint: disable=broad-except
+                logger.warning(
+                    f"An error occurred while streaming results "
+                    f"for job {self.job_id()}:\n{traceback.format_exc()}")
 
     def _decode_data(self, data: Any) -> Any:
         """Attempt to decode data using default decoder.

From 2ac39161a7e74e45393db3bcd6c9764fa30f63ab Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Wed, 14 Apr 2021 11:28:15 -0400
Subject: [PATCH 31/59] add job cancel

---
 qiskit/providers/ibmq/api/clients/runtime.py  | 12 +++++-
 qiskit/providers/ibmq/api/rest/runtime.py     | 38 +++----------------
 .../ibmq/runtime/ibm_runtime_service.py       |  2 +-
 qiskit/providers/ibmq/runtime/runtime_job.py  | 34 ++++++++++++++---
 4 files changed, 44 insertions(+), 42 deletions(-)

diff --git a/qiskit/providers/ibmq/api/clients/runtime.py b/qiskit/providers/ibmq/api/clients/runtime.py
index 06bf6ce2b..d31a157c8 100644
--- a/qiskit/providers/ibmq/api/clients/runtime.py
+++ b/qiskit/providers/ibmq/api/clients/runtime.py
@@ -118,12 +118,12 @@ def program_delete(self, program_id: str) -> None:
         """
         self.api.program(program_id).delete()
 
-    def program_job_get(self, job_id):
+    def job_get(self, job_id):
         response = self.api.program_job(job_id).get()
         logger.debug(f"Runtime job get response: {response}")
         return response
 
-    def program_job_results(self, job_id: str) -> str:
+    def job_results(self, job_id: str) -> str:
         """Get the results of a program job.
 
         Args:
@@ -133,3 +133,11 @@ def program_job_results(self, job_id: str) -> str:
             JSON response.
         """
         return self.api.program_job(job_id).results()
+
+    def job_cancel(self, job_id: str) -> None:
+        """Cancel a job.
+
+        Args:
+            job_id: Program job ID.
+        """
+        self.api.program_job(job_id).cancel()
diff --git a/qiskit/providers/ibmq/api/rest/runtime.py b/qiskit/providers/ibmq/api/rest/runtime.py
index 149b9f0c0..fac0f81c1 100644
--- a/qiskit/providers/ibmq/api/rest/runtime.py
+++ b/qiskit/providers/ibmq/api/rest/runtime.py
@@ -171,7 +171,8 @@ class ProgramJob(RestAdapterBase):
 
     URL_MAP = {
         'self': '',
-        'results': '/results'
+        'results': '/results',
+        'cancel': '/cancel'
     }
 
     def __init__(
@@ -215,35 +216,6 @@ def results(self) -> str:
         response = self.session.get(self.get_url('results'))
         return response.text
 
-
-class Stream(RestAdapterBase):
-    """Rest adapter for streaming related endpoints."""
-
-    URL_MAP = {
-        'jobs': '/jobs'
-    }
-
-    def __init__(
-            self,
-            session: RetrySession,
-            url_prefix: str = ''
-    ) -> None:
-        """ProgramJob constructor.
-
-        Args:
-            session: Session to be used in the adapter.
-            url_prefix: Prefix to use in the URL.
-        """
-        super().__init__(session, '{}/stream'.format(
-            url_prefix))
-
-    def job(self, job_id):
-        """
-
-        Args:
-            job_id:
-
-        Returns:
-
-        """
-        response = self.session.get()
\ No newline at end of file
+    def cancel(self) -> None:
+        """Cancel the job."""
+        self.session.post(self.get_url('cancel'))
diff --git a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
index d66a481aa..4bd48325c 100644
--- a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
+++ b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
@@ -179,7 +179,7 @@ def job(self, job_id: str):
         Returns:
             Runtime job retrieved.
         """
-        response = self._api_client.program_job_get(job_id)
+        response = self._api_client.job_get(job_id)
         backend = self._provider.get_backend(response['backend'])
         params_str = json.dumps(response.get('params', {}))
         params = json.loads(params_str, cls=RuntimeDecoder)
diff --git a/qiskit/providers/ibmq/runtime/runtime_job.py b/qiskit/providers/ibmq/runtime/runtime_job.py
index 9c00fd042..4a026b12e 100644
--- a/qiskit/providers/ibmq/runtime/runtime_job.py
+++ b/qiskit/providers/ibmq/runtime/runtime_job.py
@@ -29,7 +29,6 @@
 from .utils import RuntimeDecoder
 from .constants import API_TO_JOB_STATUS
 from ..api.clients import RuntimeClient, RuntimeWebsocketClient
-from ..api.exceptions import WebsocketTimeoutError
 from ..job.exceptions import IBMQJobFailureError
 
 logger = logging.getLogger(__name__)
@@ -100,16 +99,18 @@ def result(
         """
         if not self._results:
             self.wait_for_final_state(timeout=timeout, wait=wait)
-            result_raw = self._api_client.program_job_results(job_id=self.job_id())
+            result_raw = self._api_client.job_results(job_id=self.job_id())
             if self._status == JobStatus.ERROR:
                 raise IBMQJobFailureError(f"Unable to retrieve result for job {self.job_id()}. "
                                           f"Job has failed:\n{result_raw}")
             self._results = self._decode_data(result_raw)
         return self._results
 
-    def cancel(self):
-        """Attempt to cancel the job."""
-        raise NotImplementedError
+    def cancel(self) -> None:
+        """Cancel the job."""
+        self._api_client.job_cancel(self.job_id())
+        self._cancel_result_streaming()
+        self._status = JobStatus.CANCELLED
 
     def status(self) -> JobStatus:
         """Return the status of the job.
@@ -118,7 +119,7 @@ def status(self) -> JobStatus:
             Status of this job.
         """
         if self._status not in JOB_FINAL_STATES:
-            response = self._api_client.program_job_get(job_id=self.job_id())
+            response = self._api_client.job_get(job_id=self.job_id())
             self._status = API_TO_JOB_STATUS[response['status'].upper()]
         return self._status
 
@@ -165,6 +166,13 @@ def stream_results(self, callback: Callable) -> None:
                               result_queue=self._result_queue)
         self._executor.submit(self._stream_results,
                               result_queue=self._result_queue, user_callback=callback)
+        # TODO - wait for ws to connect before returning?
+
+    def _cancel_result_streaming(self) -> None:
+        """Cancel result streaming."""
+        if not self._streaming:
+            return
+        self._result_queue.put_nowait(self._result_queue_poison_pill)
 
     def _start_websocket_client(
             self,
@@ -206,6 +214,8 @@ def _stream_results(self, result_queue: queue.Queue, user_callback: Callable) ->
             try:
                 response = result_queue.get()
                 if response == self._result_queue_poison_pill:
+                    self._empty_result_queue(result_queue)
+                    self._streaming = False
                     return
                 user_callback(self.job_id(), self._decode_data(response))
             except Exception:  # pylint: disable=broad-except
@@ -213,6 +223,18 @@ def _stream_results(self, result_queue: queue.Queue, user_callback: Callable) ->
                     f"An error occurred while streaming results "
                     f"for job {self.job_id()}:\n{traceback.format_exc()}")
 
+    def _empty_result_queue(self, result_queue: queue.Queue) -> None:
+        """Empty the result queue.
+
+        Args:
+            result_queue: Result queue to empty.
+        """
+        try:
+            while True:
+                result_queue.get_nowait()
+        except queue.Empty:
+            pass
+
     def _decode_data(self, data: Any) -> Any:
         """Attempt to decode data using default decoder.
 

From 00c688269a32a18a7ed73087939cf81417e7ff7f Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Mon, 26 Apr 2021 08:55:36 -0400
Subject: [PATCH 32/59] add doc and tests

---
 docs/apidocs/ibmq-provider.rst                |   1 +
 docs/apidocs/ibmq_runtime.rst                 |   6 +
 qiskit/providers/ibmq/api/session.py          |   2 +-
 qiskit/providers/ibmq/runtime/__init__.py     | 101 ++++-
 qiskit/providers/ibmq/runtime/constants.py    |  25 +-
 qiskit/providers/ibmq/runtime/exceptions.py   |  36 ++
 .../ibmq/runtime/ibm_runtime_service.py       |  89 +++-
 .../ibmq/runtime/program/user_messenger.py    |   6 +-
 qiskit/providers/ibmq/runtime/runtime_job.py  |  52 ++-
 .../providers/ibmq/runtime/runtime_program.py |   6 +-
 test/ibmq/test_runtime.py                     | 394 +++++++++++++++++-
 11 files changed, 655 insertions(+), 63 deletions(-)
 create mode 100644 docs/apidocs/ibmq_runtime.rst
 create mode 100644 qiskit/providers/ibmq/runtime/exceptions.py

diff --git a/docs/apidocs/ibmq-provider.rst b/docs/apidocs/ibmq-provider.rst
index edda6f5a7..01d22de1c 100644
--- a/docs/apidocs/ibmq-provider.rst
+++ b/docs/apidocs/ibmq-provider.rst
@@ -14,3 +14,4 @@ Qiskit IBM Quantum Provider API Reference
    ibmq_utils
    ibmq_random
    ibmq_experiment
+   ibmq_runtime
diff --git a/docs/apidocs/ibmq_runtime.rst b/docs/apidocs/ibmq_runtime.rst
new file mode 100644
index 000000000..2b2689914
--- /dev/null
+++ b/docs/apidocs/ibmq_runtime.rst
@@ -0,0 +1,6 @@
+.. _qiskit-providers-ibmq-runtime:
+
+.. automodule:: qiskit.providers.ibmq.runtime
+   :no-members:
+   :no-inherited-members:
+   :no-special-members:
diff --git a/qiskit/providers/ibmq/api/session.py b/qiskit/providers/ibmq/api/session.py
index 3a30b3b91..d646b61c8 100644
--- a/qiskit/providers/ibmq/api/session.py
+++ b/qiskit/providers/ibmq/api/session.py
@@ -284,7 +284,7 @@ def request(  # type: ignore[override]
                     logger.debug("Response uber-trace-id: %s", ex.response.headers['uber-trace-id'])
                 except Exception:  # pylint: disable=broad-except
                     # the response did not contain the expected json.
-                    message += ". {}".format(ex.response.text)
+                    message += f". {ex.response.text}"
 
             if self.access_token:
                 message = message.replace(self.access_token, '...')
diff --git a/qiskit/providers/ibmq/runtime/__init__.py b/qiskit/providers/ibmq/runtime/__init__.py
index 71b03f5bf..ddfa20229 100644
--- a/qiskit/providers/ibmq/runtime/__init__.py
+++ b/qiskit/providers/ibmq/runtime/__init__.py
@@ -22,15 +22,15 @@
 .. caution::
 
   This package is currently provided in beta form and heavy modifications to
-  both functionality and API are likely to occur without backward compatibility.
+  both functionality and API are likely to occur.
 
 .. note::
 
   The runtime service is not available to all accounts.
 
-The IBM Quantum Runtime Service allows select users to upload their quantum programs
+The IBM Quantum Runtime Service allows authorized users to upload their quantum programs
 that can be invoked by others. A quantum program is a piece of code that takes
-certain inputs, does quantum and sometimes classical processing, and returns the
+certain inputs, performs quantum and classical processing, and returns the
 results. For example, user A can upload a VQE quantum program that takes a Hamiltonian
 and an optimizer as inputs and returns the minimum eigensolver result. User B
 can then invoke this program, passing in the inputs and obtaining the results,
@@ -38,7 +38,100 @@
 
 These quantum programs, sometimes called runtime programs, run in a special
 runtime environment that is separate from normal circuit job execution and has
-special performance advantage.
+special performance advantages.
+
+Listing runtime programs
+------------------------
+
+To list all available runtime programs::
+
+    from qiskit import IBMQ
+
+    provider = IBMQ.load_account()
+
+    # List all available programs.
+    provider.runtime.pprint_programs()
+
+    # Get a single program.
+    program = provider.runtime.program('circuit-runner')
+
+    # Print program definition.
+    print(program)
+
+In the example above, ``provider.runtime`` points to the runtime service class
+:class:`~qiskit.providers.ibmq.runtime.IBMRuntimeService`, which is the main entry
+point for using this service. The example prints the program definitions of all
+available runtime programs and of just the ``circuit-runner`` program. A program
+definition consists of a program's ID, name, description, input parameters,
+return values, interim results, and other information that helps you to know
+more about the program.
+
+Invoking a runtime program
+--------------------------
+
+You can use the :meth:`IBMRuntimeService.run` method to invoke a runtime program.
+For example::
+
+    from qiskit import IBMQ, QuantumCircuit
+
+    provider = IBMQ.load_account()
+    backend = provider.backend.ibmq_qasm_simulator
+
+    # Create a circuit.
+    qc = QuantumCircuit(2, 2)
+    qc.h(0)
+    qc.cx(0, 1)
+    qc.measure_all()
+
+    # Execute the circuit using the "circuit-runner" program.
+    runtime_inputs = {'circuits': circuit, 'measurement_error_mitigation': True}
+    options = {'backend_name': backend.name()}
+    job = provider.runtime.run(program_id="circuit-runner",
+                               options=options,
+                               inputs=runtime_inputs)
+
+    # Get runtime job result.
+    result = job.result()
+
+The example above invokes the ``circuit-runner`` program,
+which compile, executes, and optionally applies measurement error mitigation to
+the circuit result.
+
+Runtime Job
+-----------
+
+When you use the :meth:`IBMRuntimeService.run` method to invoke a runtime
+program, a
+:class:`RuntimeJob` instance is returned. This class has all the basic job
+methods, such as :meth:`RuntimeJob.status`, :meth:`RuntimeJob.result`, and
+:meth:`RuntimeJob.cancel`. Note that it does not have the same methods as regular
+circuit jobs, which are instances of :class:`~qiskit.providers.ibmq.job.IBMQJob`.
+
+Interim results
+---------------
+
+Some runtime programs provide interim results that inform you about program
+progress. You can choose to stream the interim results when you invoke the
+program by passing in the ``callback`` parameter, or at a later time using
+the :meth:`RuntimeJob.stream_results` method. For example::
+
+    from qiskit import IBMQ, QuantumCircuit
+
+    provider = IBMQ.load_account()
+    backend = provider.backend.ibmq_qasm_simulator
+
+    def interim_result_callback(job_id, interim_result):
+        print(interim_result)
+
+    # Stream interim results as soon as the job starts running.
+    job = provider.runtime.run(program_id="circuit-runner",
+                               options=options,
+                               inputs=runtime_inputs,
+                               callback=interim_result_callback)
+
+Uploading a program
+-------------------
+
 
 TODO: Add tutorial reference
 
diff --git a/qiskit/providers/ibmq/runtime/constants.py b/qiskit/providers/ibmq/runtime/constants.py
index c001c7610..a043eec23 100644
--- a/qiskit/providers/ibmq/runtime/constants.py
+++ b/qiskit/providers/ibmq/runtime/constants.py
@@ -10,30 +10,17 @@
 # copyright notice, and modified files need to carry a notice indicating
 # that they have been altered from the originals.
 
-"""Values used by the runtime API for different values."""
-
-import enum
+"""Constant values."""
 
 from qiskit.providers.jobstatus import JobStatus
 
 
-# class ApiRuntimeJobStatus(enum.Enum):
-#     """Possible values used by the API for a runtime job status."""
-#
-#     PENDING = 'PENDING'
-#     RUNNING = 'RUNNING'
-#     FAILED = 'FAILED'
-#     SUCCEEDED = 'SUCCEEDED'
-#
-#
-# JOB_FINAL_STATES = (
-#     "FAILED",
-#     "SUCCEEDED"
-# )
-
 API_TO_JOB_STATUS = {
     'PENDING': JobStatus.INITIALIZING,
+    'QUEUED': JobStatus.QUEUED,
     'RUNNING': JobStatus.RUNNING,
-    'SUCCEEDED': JobStatus.DONE,
-    'FAILED': JobStatus.ERROR
+    'COMPLETED': JobStatus.DONE,
+    'SUCCEEDED': JobStatus.DONE,    # TODO remove when no longer used
+    'FAILED': JobStatus.ERROR,
+    'CANCELLED': JobStatus.CANCELLED
 }
diff --git a/qiskit/providers/ibmq/runtime/exceptions.py b/qiskit/providers/ibmq/runtime/exceptions.py
new file mode 100644
index 000000000..c3ca7c7c5
--- /dev/null
+++ b/qiskit/providers/ibmq/runtime/exceptions.py
@@ -0,0 +1,36 @@
+# This code is part of Qiskit.
+#
+# (C) Copyright IBM 2021.
+#
+# This code is licensed under the Apache License, Version 2.0. You may
+# obtain a copy of this license in the LICENSE.txt file in the root directory
+# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Any modifications or derivative works of this code must retain this
+# copyright notice, and modified files need to carry a notice indicating
+# that they have been altered from the originals.
+
+"""Exceptions related to IBM Quantum runtime service."""
+
+
+from ..exceptions import IBMQError
+
+
+class QiskitRuntimeError(IBMQError):
+    """Base class for errors raised by the runtime service modules."""
+    pass
+
+
+class RuntimeDuplicateProgramError(QiskitRuntimeError):
+    """Error raised when a program being uploaded already exists."""
+    pass
+
+
+class RuntimeProgramNotFound(QiskitRuntimeError):
+    """Error raised when a program is not found."""
+    pass
+
+
+class RuntimeJobFailureError(QiskitRuntimeError):
+    """Error raised when a runtime job failed."""
+    pass
diff --git a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
index 4bd48325c..8bea19963 100644
--- a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
+++ b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
@@ -22,14 +22,59 @@
 from .runtime_job import RuntimeJob
 from .runtime_program import RuntimeProgram
 from .utils import RuntimeEncoder, RuntimeDecoder
+from .exceptions import QiskitRuntimeError, RuntimeDuplicateProgramError, RuntimeProgramNotFound
 from ..api.clients.runtime import RuntimeClient
 from ..api.clients.runtime_ws import RuntimeWebsocketClient
+from ..api.exceptions import RequestsApiError
+from ..exceptions import IBMQNotAuthorizedError
 
 logger = logging.getLogger(__name__)
 
 
 class IBMRuntimeService:
-    """Class for interacting with the IBM Quantum runtime service."""
+    """Class for interacting with the IBM Quantum runtime service.
+
+    The IBM Quantum Runtime Service allows authorized users to upload their quantum programs
+    that can be invoked by others. A quantum program is a piece of code that takes
+    certain inputs, performs quantum and classical processing, and returns the
+    results.
+
+    A sample workflow of using the runtime service::
+
+        from qiskit import IBMQ, QuantumCircuit
+
+        provider = IBMQ.load_account()
+        backend = provider.backend.ibmq_qasm_simulator
+
+        # List all available programs.
+        provider.runtime.pprint_programs()
+
+        # Create a circuit.
+        qc = QuantumCircuit(2, 2)
+        qc.h(0)
+        qc.cx(0, 1)
+        qc.measure_all()
+
+        # Execute the circuit using the "circuit-runner" program.
+        runtime_inputs = {'circuits': circuit, 'measurement_error_mitigation': True}
+        options = {'backend_name': backend.name()}
+        job = provider.runtime.run(program_id="circuit-runner",
+                                   options=options,
+                                   inputs=runtime_inputs)
+
+        # Get runtime job result.
+        result = job.result()
+
+    If the program has any interim results, you can use the ``callback``
+    parameter of the :meth:`run` method to stream the interim results.
+    Alternatively, you can use the :meth:`stream_results` method to stream
+    the results at a later time, but before the job finishes.
+
+    The :meth:`run` method returns a
+    :class:`qiskit.providers.ibmq.runtime.RuntimeJob` object. You can use its
+    methods to perform tasks like checking the job status, getting job result, and
+    canceling the job.
+    """
 
     def __init__(self, provider: 'accountprovider.AccountProvider') -> None:
         """IBMRuntimeService constructor.
@@ -83,9 +128,19 @@ def program(self, program_id: str, refresh: bool = False) -> RuntimeProgram:
 
         Returns:
             Runtime program.
+
+        Raises:
+            RuntimeProgramNotFound: If the program does not exist.
+            QiskitRuntimeError: If the request failed.
         """
         if program_id not in self._programs or refresh:
-            response = self._api_client.program_get(program_id)
+            try:
+                response = self._api_client.program_get(program_id)
+            except RequestsApiError as ex:
+                if ex.status_code == 404:
+                    raise RuntimeProgramNotFound(f"Program not found: {ex.message}") from None
+                raise QiskitRuntimeError(f"Failed to get program: {ex}") from None
+
             self._programs[program_id] = self._to_program(response)
 
         return self._programs[program_id]
@@ -158,8 +213,22 @@ def upload_program(
 
         Returns:
             Program ID.
+
+        Raises:
+            RuntimeDuplicateProgramError: If a program with the same name already exists.
+            IBMQNotAuthorizedError: If you are not authorized to upload programs.
+            QiskitRuntimeError: If the upload failed.
         """
-        response = self._api_client.program_create(name, data, max_execution_time)
+        try:
+            response = self._api_client.program_create(name, data, max_execution_time)
+        except RequestsApiError as ex:
+            if ex.status_code == 409:
+                raise RuntimeDuplicateProgramError(
+                    "Program with the same name already exists.") from None
+            elif ex.status_code == 403:
+                raise IBMQNotAuthorizedError(
+                    "You are not authorized to upload programs.") from None
+            raise QiskitRuntimeError(f"Failed to create program: {ex}") from None
         return response['id']
 
     def delete_program(self, program_id: str):
@@ -167,8 +236,20 @@ def delete_program(self, program_id: str):
 
         Args:
             program_id: Program ID.
+
+        Raises:
+            RuntimeProgramNotFound: If the program doesn't exist.
+            QiskitRuntimeError: If the request failed.
         """
-        self._api_client.program_delete(program_id=program_id)
+        try:
+            self._api_client.program_delete(program_id=program_id)
+        except RequestsApiError as ex:
+            if ex.status_code == 404:
+                raise RuntimeProgramNotFound(f"Program not found: {ex.message}") from None
+            raise QiskitRuntimeError(f"Failed to delete program: {ex}") from None
+
+        if program_id in self._programs:
+            del self._programs[program_id]
 
     def job(self, job_id: str):
         """Retrieve a runtime job.
diff --git a/qiskit/providers/ibmq/runtime/program/user_messenger.py b/qiskit/providers/ibmq/runtime/program/user_messenger.py
index 03fbfa527..fd696aab1 100644
--- a/qiskit/providers/ibmq/runtime/program/user_messenger.py
+++ b/qiskit/providers/ibmq/runtime/program/user_messenger.py
@@ -27,8 +27,8 @@ class UserMessenger:
     def publish(
         self,
         message: Any,
-        final_result: bool = False,
         encoder: json.JSONEncoder = RuntimeEncoder,
+        final: bool = False
     ) -> None:
         """Publish message.
 
@@ -36,14 +36,14 @@ def publish(
         to the program consumer. The messages will be made immediately available to the consumer,
         but they may choose not to receive the messages.
 
-        The `final_result` parameter is used to indicate whether the message is
+        The `final` parameter is used to indicate whether the message is
         the final result of the program. Final results may be processed differently
         from interim results.
 
         Args:
             message: Message to be published. Can be any type.
-            final_result: Whether this is the final result from the program.
             encoder: An optional JSON encoder for serializing
+            final: Whether the message being published is the final result.
         """
         # Default implementation for testing.
         print(json.dumps(message, cls=encoder))
diff --git a/qiskit/providers/ibmq/runtime/runtime_job.py b/qiskit/providers/ibmq/runtime/runtime_job.py
index 4a026b12e..2589c8700 100644
--- a/qiskit/providers/ibmq/runtime/runtime_job.py
+++ b/qiskit/providers/ibmq/runtime/runtime_job.py
@@ -28,14 +28,44 @@
 
 from .utils import RuntimeDecoder
 from .constants import API_TO_JOB_STATUS
+from .exceptions import RuntimeJobFailureError
 from ..api.clients import RuntimeClient, RuntimeWebsocketClient
-from ..job.exceptions import IBMQJobFailureError
 
 logger = logging.getLogger(__name__)
 
 
 class RuntimeJob:
-    """Representation of a runtime program execution."""
+    """Representation of a runtime program execution.
+
+    A new ``RuntimeJob`` instance is returned when you call
+    :meth:`qiskit.providers.ibmq.runtime.IBMRuntimeService.run`
+    to execute a runtime program, and when you call
+    :meth:`qiskit.providers.ibmq.runtime.IBMRuntimeService.job`
+    to retrieve a previously executed job.
+
+    If the program execution is successful, you can inspect the job's status by
+    calling :meth:`status()`. Job status can be one of the
+    :class:`~qiskit.providers.JobStatus` members.
+
+    Some of the methods in this class are blocking, which means control may
+    not be returned immediately. :meth:`result()` is an example
+    of a blocking method::
+
+        job = provider.runtime.run(...)
+
+        try:
+            job_result = job.result()  # It will block until the job finishes.
+            print("The job finished with result {}".format(job_result))
+        except IBMQJobFailureError as ex:
+            print("Job failed!: {}".format(ex))
+
+    If the program has any interim results, you can use the ``callback``
+    parameter of the
+    :meth:`~qiskit.providers.ibmq.runtime.IBMRuntimeService.run`
+    method to stream the interim results.
+    Alternatively, you can use the :meth:`stream_results` method to stream
+    the results at a later time, but before the job finishes.
+    """
 
     _executor = futures.ThreadPoolExecutor()
     _result_queue_poison_pill = "_poison_pill"
@@ -67,7 +97,6 @@ def __init__(
         self._ws_client = ws_client
         self._results = None
         self._params = params or {}
-        # self._user_callback = user_callback
         self._program_id = program_id
         self._status = JobStatus.INITIALIZING
         self._streaming = False
@@ -95,14 +124,14 @@ def result(
             Runtime job result.
 
         Raises:
-            IBMQJobFailureError: If the job failed.
+            RuntimeJobFailureError: If the job failed.
         """
         if not self._results:
             self.wait_for_final_state(timeout=timeout, wait=wait)
             result_raw = self._api_client.job_results(job_id=self.job_id())
             if self._status == JobStatus.ERROR:
-                raise IBMQJobFailureError(f"Unable to retrieve result for job {self.job_id()}. "
-                                          f"Job has failed:\n{result_raw}")
+                raise RuntimeJobFailureError(f"Unable to retrieve result for job {self.job_id()}. "
+                                             f"Job has failed:\n{result_raw}")
             self._results = self._decode_data(result_raw)
         return self._results
 
@@ -120,7 +149,10 @@ def status(self) -> JobStatus:
         """
         if self._status not in JOB_FINAL_STATES:
             response = self._api_client.job_get(job_id=self.job_id())
-            self._status = API_TO_JOB_STATUS[response['status'].upper()]
+            try:
+                self._status = API_TO_JOB_STATUS[response['status'].upper()]
+            except KeyError:
+                raise QiskitError(f"Unknown status: {response['status']}")
         return self._status
 
     def wait_for_final_state(
@@ -266,11 +298,11 @@ def backend(self) -> Backend:
         return self._backend
 
     @property
-    def parameters(self) -> Dict:
-        """Job parameters.
+    def inputs(self) -> Dict:
+        """Job input parameters.
 
         Returns:
-            Parameters used in this job.
+            Input parameters used in this job.
         """
         return self._params
 
diff --git a/qiskit/providers/ibmq/runtime/runtime_program.py b/qiskit/providers/ibmq/runtime/runtime_program.py
index 2afcdd67e..df898e291 100644
--- a/qiskit/providers/ibmq/runtime/runtime_program.py
+++ b/qiskit/providers/ibmq/runtime/runtime_program.py
@@ -94,9 +94,9 @@ def _format_common(items: List):
                 if hasattr(item, 'required'):
                     formatted.append(" "*6 + "Required: " + str(item.required))
 
-        formatted = [f'{self.name}:',
-                     f"  ID: {self._id}",
-                     f"  Description: {self._description}",
+        formatted = [f'{self.program_id}:',
+                     f"  Name: {self.name}",
+                     f"  Description: {self.description}",
                      f"  Parameters:"]
 
         if self._parameters:
diff --git a/test/ibmq/test_runtime.py b/test/ibmq/test_runtime.py
index b9557a4da..9306d64d1 100644
--- a/test/ibmq/test_runtime.py
+++ b/test/ibmq/test_runtime.py
@@ -16,11 +16,18 @@
 import os
 from io import StringIO
 from unittest.mock import patch
+import uuid
+import time
+import threading
 
-from qiskit.providers.jobstatus import JobStatus
+from qiskit.providers.jobstatus import JobStatus, JOB_FINAL_STATES
+from qiskit.providers.ibmq.exceptions import IBMQNotAuthorizedError
+from qiskit.providers.ibmq.runtime.exceptions import (RuntimeDuplicateProgramError,
+                                                      RuntimeProgramNotFound,
+                                                      RuntimeJobFailureError)
 
 from ..ibmqtestcase import IBMQTestCase
-from ..decorators import requires_provider
+from ..decorators import requires_device, requires_provider
 from ..fake_runtime_client import BaseFakeRuntimeClient
 
 
@@ -64,31 +71,128 @@ def _callback(interim_result):
         job.result()
 
 
+# import random
+#
+# from qiskit import transpile
+# from qiskit.circuit.random import random_circuit
+# from qiskit.providers.ibmq.runtime.utils import RuntimeEncoder
+#
+# def prepare_circuits(backend):
+#     circuit = random_circuit(num_qubits=5, depth=4, measure=True,
+#                              seed=random.randint(0, 1000))
+#     return transpile(circuit, backend)
+#
+# def main(backend, user_messenger, **kwargs):
+#     iterations = kwargs.pop('iterations', 5)
+#     interim_results = kwargs.pop('interim_results', {})
+#     final_result = kwargs.pop("final_result", {})
+#     for it in range(iterations):
+#         qc = prepare_circuits(backend)
+#         user_messenger.publish({"iteration": it, "interim_results": interim_results})
+#         backend.run(qc).result()
+#
+#     user_messenger.publish("this is the last message")
+#     print(final_result, cls=RuntimeEncoder)
+
 @unittest.skipIf(not os.environ.get('USE_STAGING_CREDENTIALS', ''), "Only runs on staging")
 class TestRuntimeIntegration(IBMQTestCase):
 
+    RUNTIME_PROGRAM = """
+import random
+
+from qiskit import transpile
+from qiskit.circuit.random import random_circuit
+
+def prepare_circuits(backend):
+    circuit = random_circuit(num_qubits=5, depth=4, measure=True,
+                             seed=random.randint(0, 1000))
+    return transpile(circuit, backend)
+
+def main(backend, user_messenger, **kwargs):
+    iterations = kwargs['iterations']
+    interim_results = kwargs.pop('interim_results', {})
+    final_result = kwargs.pop("final_result", {})
+    for it in range(iterations):
+        qc = prepare_circuits(backend)
+        user_messenger.publish({"iteration": it, "interim_results": interim_results})
+        backend.run(qc).result()
+
+    user_messenger.publish(final_result, final=True)
+    """
+
+    PROGRAM_PREFIX = 'qiskit-test'
+
     @classmethod
-    @requires_provider
-    def setUpClass(cls, provider):
+    @requires_device
+    def setUpClass(cls, backend):
         """Initial class level setup."""
         # pylint: disable=arguments-differ
         super().setUpClass()
-        cls.provider = provider
+        cls.backend = backend
+        cls.provider = backend.provider()
+        cls.program_id = cls.PROGRAM_PREFIX
+        try:
+            cls.program_id = cls.provider.runtime.upload_program(
+                name=cls.PROGRAM_PREFIX,
+                data=cls.RUNTIME_PROGRAM.encode(),
+                max_execution_time=600)
+        except RuntimeDuplicateProgramError:
+            pass
+        except IBMQNotAuthorizedError:
+            raise unittest.SkipTest("No upload access.")
+
+    @classmethod
+    def tearDownClass(cls) -> None:
+        """Class level teardown."""
+        super().tearDownClass()
         try:
-            provider.runtime.programs()
-        except Exception:
-            raise unittest.SkipTest("No access to runtime service.")
+            cls.provider.runtime.delete_program(cls.program_id)
+        except:
+            pass
+
+    def setUp(self) -> None:
+        """Test level setup."""
+        super().setUp()
+        self.to_delete = []
+        self.to_cancel = []
+
+    def tearDown(self) -> None:
+        """Test level teardown."""
+        super().tearDown()
+        # Delete programs
+        for prog in self.to_delete:
+            try:
+                self.provider.runtime.delete_program(prog)
+            except:
+                pass
+
+        # Cancel jobs.
+        for job in self.to_cancel:
+            if job.status() not in JOB_FINAL_STATES:
+                try:
+                    job.cancel()
+                except:
+                    pass
+
+    def test_runtime_service(self):
+        """Test getting runtime service."""
+        self.assertTrue(self.provider.service('runtime'))
 
     def test_list_programs(self):
         """Test listing programs."""
         programs = self.provider.runtime.programs()
         self.assertTrue(programs)
+        found = False
         for prog in programs:
             self._validate_program(prog)
+            if prog.program_id == self.program_id:
+                found = True
+        self.assertTrue(found, f"Program {self.program_id} not found!")
 
     def test_list_program(self):
         """Test listing a single program."""
-        program = self.provider.runtime.programs()[0]
+        program = self.provider.runtime.program(self.program_id)
+        self.assertEqual(self.program_id, program.program_id)
         self._validate_program(program)
 
     def test_print_programs(self):
@@ -96,34 +200,286 @@ def test_print_programs(self):
         programs = self.provider.runtime.programs()
         with patch('sys.stdout', new=StringIO()) as mock_stdout:
             self.provider.runtime.pprint_programs()
+            stdout = mock_stdout.getvalue()
             for prog in programs:
-                self.assertIn(prog.program_id, mock_stdout)
-                self.assertIn(prog.name, mock_stdout)
-                self.assertIn(prog.description, mock_stdout)
+                self.assertIn(prog.program_id, stdout)
+                self.assertIn(prog.name, stdout)
+                self.assertIn(prog.description, stdout)
 
     def test_upload_program(self):
         """Test uploading a program."""
-        pass
+        max_execution_time = 3000
+        program_id = self._upload_program(max_execution_time=max_execution_time)
+        self.assertTrue(program_id)
+        program = self.provider.runtime.program(program_id)
+        self.assertTrue(program)
+        self.assertEqual(max_execution_time, program.max_execution_time)
 
     def test_upload_program_conflict(self):
         """Test uploading a program with conflicting name."""
+        name = self._get_program_name()
+        self._upload_program(name=name)
+        with self.assertRaises(RuntimeDuplicateProgramError):
+            self._upload_program(name=name)
+
+    def test_update_program(self):
+        """Test updating a program."""
+        program_id = self._upload_program()
+        program = self.provider.runtime.program(program_id)
+
+        self.provider.runtime.delete_program(program_id)
+        new_cost = program.max_execution_time + 1000
+        new_id = self._upload_program(name=program.name, max_execution_time=new_cost)
+        updated = self.provider.runtime.program(new_id, refresh=True)
+        self.assertEqual(new_cost, updated.max_execution_time,
+                         f"Program {new_id} does not have the expected cost.")
+
+    def test_delete_program(self):
+        """Test deleting program."""
+        program_id = self._upload_program()
+        self.provider.runtime.delete_program(program_id)
+        with self.assertRaises(RuntimeProgramNotFound):
+            self.provider.runtime.program(program_id, refresh=True)
+
+    def test_double_delete_program(self):
+        """Test deleting a deleted program."""
+        program_id = self._upload_program()
+        self.provider.runtime.delete_program(program_id)
+        with self.assertRaises(RuntimeProgramNotFound):
+            self.provider.runtime.delete_program(program_id)
+
+    def test_run_program(self):
+        """Test running a program."""
+        final_result = {"string": "foo",
+                        "float": 1.5,
+                        "complex": 2+3j,
+                        "class": self.CustomClass("foo")}
+        job = self._run_program(final_result=final_result)
+        result = job.result()
+        my_class = self.CustomClass.from_json(result.pop('class'))
+        self.assertEqual(final_result.pop('class').value, my_class.value)
+        self.assertEqual(final_result, result)
+
+    def test_run_program_failed(self):
+        """Test a failed program execution."""
+        options = {'backend_name': self.backend.name()}
+        job = self.provider.runtime.run(program_id=self.program_id, inputs={}, options=options)
+        self.log.info(f"Runtime job {job.job_id()} submitted.")
+
+        job.wait_for_final_state()
+        self.assertEqual(JobStatus.ERROR, job.status())
+        with self.assertRaises(RuntimeJobFailureError) as cm:
+            job.result()
+        self.assertIn('KeyError', str(cm.exception))
+
+    def test_retrieve_job_queued(self):
+        """Test retrieving a queued job."""
         pass
 
-    def test_upload_program_missing(self):
+    def test_retrieve_job_running(self):
+        """Test retrieving a running job."""
+        job = self._run_program(iterations=10)
+        for _ in range(10):
+            if job.status() == JobStatus.RUNNING:
+                break
+            time.sleep(1)
+        self.assertEqual(JobStatus.RUNNING, job.status())
+        rjob = self.provider.runtime.job(job.job_id())
+        self.assertEqual(job.job_id(), rjob.job_id())
+
+    def test_retrieve_job_done(self):
         pass
 
-    def test_execute_program(self):
+    def test_cancel_job_queued(self):
         pass
 
-    def test_execute_program_bad_params(self):
+    def test_cancel_job_running(self):
         pass
 
-    def test_execute_program_failed(self):
+    def test_cancel_job_done(self):
         pass
 
+    def test_interim_result_callback(self):
+        """Test interim result callback."""
+        def result_callback(job_id, interim_result):
+            nonlocal final_it
+            final_it = interim_result['iteration']
+            nonlocal callback_err
+            if job_id != job.job_id():
+                callback_err.append(f"Unexpected job ID: {job_id}")
+            if interim_result['interim_results'] != int_res:
+                callback_err.append(f"Unexpected interim result: {interim_result}")
+
+        int_res = "foo"
+        final_it = 0
+        callback_err = []
+        iterations = 3
+        job = self._run_program(iterations=iterations, interim_results=int_res,
+                                callback=result_callback)
+        job.wait_for_final_state()
+        self.assertEqual(iterations-1, final_it)
+        self.assertFalse(callback_err)
+        self.assertIsNone(job._ws_client._ws)
+
+    def test_stream_results(self):
+        """Test stream_results method."""
+        def result_callback(job_id, interim_result):
+            nonlocal final_it
+            final_it = interim_result['iteration']
+            nonlocal callback_err
+            if job_id != job.job_id():
+                callback_err.append(f"Unexpected job ID: {job_id}")
+            if interim_result['interim_results'] != int_res:
+                callback_err.append(f"Unexpected interim result: {interim_result}")
+
+        int_res = "bar"
+        final_it = 0
+        callback_err = []
+        iterations = 3
+        job = self._run_program(iterations=iterations, interim_results=int_res)
+
+        for _ in range(10):
+            if job.status() == JobStatus.RUNNING:
+                break
+            time.sleep(1)
+        self.assertEqual(JobStatus.RUNNING, job.status())
+        job.stream_results(result_callback)
+        job.wait_for_final_state()
+        self.assertEqual(iterations-1, final_it)
+        self.assertFalse(callback_err)
+        self.assertIsNone(job._ws_client._ws)
+
+    @unittest.skip("Skip until 267 is fixed")
+    def test_stream_results_done(self):
+        """Test streaming interim results after job is done."""
+        def result_callback(job_id, interim_result):
+            nonlocal called_back
+            called_back = True
+
+        called_back = False
+        job = self._run_program(interim_results="foobar")
+        job.wait_for_final_state()
+        job.stream_results(result_callback)
+        time.sleep(1)
+        self.assertFalse(called_back)
+        self.assertIsNone(job._ws_client._ws)
+
+    def test_callback_error(self):
+        """Test error in callback method."""
+        def result_callback(job_id, interim_result):
+            if interim_result['iteration'] == 0:
+                raise ValueError("Kaboom!")
+            nonlocal final_it
+            final_it = interim_result['iteration']
+
+        final_it = 0
+        iterations = 3
+        with self.assertLogs('qiskit.providers.ibmq.runtime', level='WARNING') as cm:
+            job = self._run_program(iterations=iterations, interim_results="foo",
+                                    callback=result_callback)
+            job.wait_for_final_state()
+
+        self.assertIn("Kaboom", ', '.join(cm.output))
+        self.assertEqual(iterations-1, final_it)
+        self.assertIsNone(job._ws_client._ws)
+
+    @unittest.skip("Skip until 277 is fixed")
+    def test_callback_job_cancelled(self):
+        """Test canceling a job while streaming results."""
+        def result_callback(job_id, interim_result):
+            nonlocal callback_event
+            callback_event.set()
+
+        callback_event = threading.Event()
+        job = self._run_program(iterations=3, interim_results="foo",
+                                callback=result_callback)
+
+        callback_event.wait(10)
+        job.cancel()
+        time.sleep(5)  # Wait for cleanup
+        self.assertIsNone(job._ws_client._ws)
+
+    def test_final_result(self):
+        pass
+
+    def test_job_status(self):
+        pass
+
+    def test_job_inputs(self):
+        """Test job inputs."""
+        inputs = {'iterations': 1,
+                  'interim_results': "foo"}
+        options = {'backend_name': self.backend.name()}
+        job = self.provider.runtime.run(program_id=self.program_id, inputs=inputs,
+                                        options=options)
+        self.log.info(f"Runtime job {job.job_id()} submitted.")
+        self.to_cancel.append(job)
+        self.assertEqual(inputs, job.inputs)
+
+    def test_job_backend(self):
+        """Test job backend."""
+        job = self._run_program()
+        self.assertEqual(self.backend, job.backend())
+
+    def test_job_program_id(self):
+        """Test job program ID."""
+        job = self._run_program()
+        self.assertEqual(self.program_id, job.program_id)
+
+    def test_wait_for_final_state(self):
+        """Test wait for final state."""
+        job = self._run_program()
+        job.wait_for_final_state()
+        self.assertEqual(JobStatus.DONE, job.status())
+
     def _validate_program(self, program):
+        # TODO add more validation
         self.assertTrue(program)
         self.assertTrue(program.name)
         self.assertTrue(program.program_id)
-        self.assertTrue(program.description)
-        self.assertTrue(program.cost)
+        # self.assertTrue(program.description)
+        self.assertTrue(program.max_execution_time)
+
+    def _upload_program(self, name=None, max_execution_time=300):
+        name = name or self._get_program_name()
+        program_id = self.provider.runtime.upload_program(
+            name=name,
+            data=self.RUNTIME_PROGRAM.encode(),
+            max_execution_time=max_execution_time)
+        self.to_delete.append(program_id)
+        return program_id
+
+    def _get_program_name(self):
+        return self.PROGRAM_PREFIX + "_" + uuid.uuid4().hex
+
+    def _run_program(self, program_id=None, iterations=1,
+                     interim_results=None, final_result=None,
+                     callback=None):
+        """Run a program."""
+        inputs = {'iterations': iterations,
+                  'interim_results': interim_results or {},
+                  'final_result': final_result or {}}
+        pid = program_id or self.program_id
+        options = {'backend_name': self.backend.name()}
+        job = self.provider.runtime.run(program_id=pid, inputs=inputs,
+                                        options=options, callback=callback)
+        self.log.info(f"Runtime job {job.job_id()} submitted.")
+        self.to_cancel.append(job)
+        return job
+
+    # iterations = kwargs.pop('iterations', 5)
+    # interim_results = kwargs.pop('interim_results', {})
+    # final_result = kwargs.pop("final_result", {})
+    # test_lp1_sw_renierm
+
+    class CustomClass:
+        """Custom class with serialization methods."""
+        def __init__(self, value):
+            self.value = value
+
+        def to_json(self):
+            return {"value": self.value}
+
+        @classmethod
+        def from_json(cls, data):
+            return cls(**data)

From 7060915c5624e17bc8871530680d2449dde17530 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Mon, 26 Apr 2021 14:08:28 -0400
Subject: [PATCH 33/59] fix lint

---
 qiskit/providers/ibmq/accountprovider.py      |  2 +-
 qiskit/providers/ibmq/api/clients/runtime.py  |  3 +-
 .../providers/ibmq/api/clients/runtime_ws.py  | 21 ++---
 qiskit/providers/ibmq/api/rest/runtime.py     | 10 ++-
 qiskit/providers/ibmq/runtime/constants.py    |  1 -
 qiskit/providers/ibmq/runtime/exceptions.py   |  5 ++
 .../ibmq/runtime/ibm_runtime_service.py       | 12 +--
 .../ibmq/runtime/program/program_backend.py   |  1 +
 .../ibmq/runtime/program/program_template.py  |  9 ++-
 .../ibmq/runtime/program/user_messenger.py    |  9 ++-
 qiskit/providers/ibmq/runtime/runtime_job.py  | 79 +++++++++++--------
 .../providers/ibmq/runtime/runtime_program.py |  2 +
 qiskit/providers/ibmq/runtime/utils.py        |  9 ++-
 test/fake_runtime_client.py                   |  2 +
 test/ibmq/test_runtime.py                     | 67 +++++++++++-----
 15 files changed, 148 insertions(+), 84 deletions(-)

diff --git a/qiskit/providers/ibmq/accountprovider.py b/qiskit/providers/ibmq/accountprovider.py
index 1cba1cb28..a635f1f56 100644
--- a/qiskit/providers/ibmq/accountprovider.py
+++ b/qiskit/providers/ibmq/accountprovider.py
@@ -209,7 +209,7 @@ def run_circuits(
             rep_time: Optional[int] = None,
             rep_delay: Optional[float] = None,
             parameter_binds: Optional[List[Dict[Parameter, float]]] = None,
-            schedule_circuit=False,
+            schedule_circuit: bool = False,
             inst_map: InstructionScheduleMap = None,
             meas_map: List[List[int]] = None,
             init_qubits: Optional[bool] = None,
diff --git a/qiskit/providers/ibmq/api/clients/runtime.py b/qiskit/providers/ibmq/api/clients/runtime.py
index d31a157c8..f019f8216 100644
--- a/qiskit/providers/ibmq/api/clients/runtime.py
+++ b/qiskit/providers/ibmq/api/clients/runtime.py
@@ -119,8 +119,9 @@ def program_delete(self, program_id: str) -> None:
         self.api.program(program_id).delete()
 
     def job_get(self, job_id):
+        """Get job data."""
         response = self.api.program_job(job_id).get()
-        logger.debug(f"Runtime job get response: {response}")
+        logger.debug("Runtime job get response: %s", response)
         return response
 
     def job_results(self, job_id: str) -> str:
diff --git a/qiskit/providers/ibmq/api/clients/runtime_ws.py b/qiskit/providers/ibmq/api/clients/runtime_ws.py
index 59e52f14c..e02ce2ab3 100644
--- a/qiskit/providers/ibmq/api/clients/runtime_ws.py
+++ b/qiskit/providers/ibmq/api/clients/runtime_ws.py
@@ -33,6 +33,7 @@ class RuntimeWebsocketClient:
 
     BACKOFF_MAX = 8
     """Maximum time to wait between retries."""
+    POISON_PILL = "_poison_pill"
 
     def __init__(
             self,
@@ -57,12 +58,8 @@ async def _connect(self, url: str) -> WebSocketClientProtocol:
             An open websocket connection.
 
         Raises:
-            WebsocketError: If the connection to the websocket server could
+            WebsocketRetryableError: If the connection to the websocket server could
                 not be established.
-            WebsocketAuthenticationError: If the connection to the websocket
-                was established, but the authentication failed.
-            WebsocketIBMQProtocolError: If the connection to the websocket
-                server was established, but the answer was unexpected.
         """
         try:
             logger.debug('Starting new websocket connection: %s', url)
@@ -102,6 +99,7 @@ async def job_results(
 
         Raises:
             WebsocketError: If a websocket error occurred.
+            WebsocketRetryableError: If a websocket error occurred and maximum retry reached.
         """
         url = '{}/stream/jobs/{}'.format(self._websocket_url, job_id)
 
@@ -125,16 +123,21 @@ async def job_results(
                             f"closed unexpectedly: {ex.code}")
                         raise exception_to_raise
 
+            except asyncio.CancelledError:
+                logger.debug("Streaming is cancelled.")
+                result_queue.put_nowait(self.POISON_PILL)
+                return
             except WebsocketRetryableError as ex:
-                logger.debug(f"A websocket error occurred while streaming "
-                             f"results for runtime job {job_id}:\n{traceback.format_exc()}")
+                logger.debug("A websocket error occurred while streaming "
+                             "results for runtime job %s:\n%s", job_id, traceback.format_exc())
                 current_retry += 1
                 if current_retry > max_retries:
+                    result_queue.put_nowait(self.POISON_PILL)
                     raise ex
 
                 backoff_time = self._backoff_time(backoff_factor, current_retry)
-                logger.info(f"Retrying websocket after {backoff_time} seconds. "
-                            f"Attemp {current_retry}")
+                logger.info("Retrying websocket after %s seconds. Attemp %s",
+                            backoff_time, current_retry)
                 await asyncio.sleep(backoff_time)  # Block asyncio loop for given backoff time.
                 continue  # Continues next iteration after `finally` block.
             finally:
diff --git a/qiskit/providers/ibmq/api/rest/runtime.py b/qiskit/providers/ibmq/api/rest/runtime.py
index fac0f81c1..d106d8b31 100644
--- a/qiskit/providers/ibmq/api/rest/runtime.py
+++ b/qiskit/providers/ibmq/api/rest/runtime.py
@@ -42,7 +42,15 @@ def program(self, program_id: str) -> 'Program':
         """
         return Program(self.session, program_id)
 
-    def program_job(self, job_id):
+    def program_job(self, job_id: str) -> None:
+        """Return an adapter for the job.
+
+        Args:
+            job_id: Job ID.
+
+        Returns:
+            The program job adapter.
+        """
         return ProgramJob(self.session, job_id)
 
     def list_programs(self) -> List[Dict]:
diff --git a/qiskit/providers/ibmq/runtime/constants.py b/qiskit/providers/ibmq/runtime/constants.py
index a043eec23..43c825dc9 100644
--- a/qiskit/providers/ibmq/runtime/constants.py
+++ b/qiskit/providers/ibmq/runtime/constants.py
@@ -16,7 +16,6 @@
 
 
 API_TO_JOB_STATUS = {
-    'PENDING': JobStatus.INITIALIZING,
     'QUEUED': JobStatus.QUEUED,
     'RUNNING': JobStatus.RUNNING,
     'COMPLETED': JobStatus.DONE,
diff --git a/qiskit/providers/ibmq/runtime/exceptions.py b/qiskit/providers/ibmq/runtime/exceptions.py
index c3ca7c7c5..c8b85e32c 100644
--- a/qiskit/providers/ibmq/runtime/exceptions.py
+++ b/qiskit/providers/ibmq/runtime/exceptions.py
@@ -34,3 +34,8 @@ class RuntimeProgramNotFound(QiskitRuntimeError):
 class RuntimeJobFailureError(QiskitRuntimeError):
     """Error raised when a runtime job failed."""
     pass
+
+
+class RuntimeInvalidStateError(QiskitRuntimeError):
+    """Errors raised when the state is not valid for the operation."""
+    pass
diff --git a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
index 8bea19963..116fadad1 100644
--- a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
+++ b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
@@ -17,7 +17,6 @@
 import json
 
 from qiskit.providers.ibmq import accountprovider  # pylint: disable=unused-import
-from qiskit import QiskitError
 
 from .runtime_job import RuntimeJob
 from .runtime_program import RuntimeProgram
@@ -26,7 +25,7 @@
 from ..api.clients.runtime import RuntimeClient
 from ..api.clients.runtime_ws import RuntimeWebsocketClient
 from ..api.exceptions import RequestsApiError
-from ..exceptions import IBMQNotAuthorizedError
+from ..exceptions import IBMQNotAuthorizedError, IBMQInputValueError
 
 logger = logging.getLogger(__name__)
 
@@ -179,9 +178,12 @@ def run(
 
         Returns:
             A ``RuntimeJob`` instance representing the execution.
+
+        Raises:
+            IBMQInputValueError: If input is invalid.
         """
         if 'backend_name' not in options:
-            raise QiskitError('"backend_name" is required field in "options"')
+            raise IBMQInputValueError('"backend_name" is required field in "options"')
 
         backend_name = options['backend_name']
         params_str = json.dumps(inputs, cls=RuntimeEncoder)
@@ -225,7 +227,7 @@ def upload_program(
             if ex.status_code == 409:
                 raise RuntimeDuplicateProgramError(
                     "Program with the same name already exists.") from None
-            elif ex.status_code == 403:
+            if ex.status_code == 403:
                 raise IBMQNotAuthorizedError(
                     "You are not authorized to upload programs.") from None
             raise QiskitRuntimeError(f"Failed to create program: {ex}") from None
@@ -251,7 +253,7 @@ def delete_program(self, program_id: str):
         if program_id in self._programs:
             del self._programs[program_id]
 
-    def job(self, job_id: str):
+    def job(self, job_id: str) -> RuntimeJob:
         """Retrieve a runtime job.
 
         Args:
diff --git a/qiskit/providers/ibmq/runtime/program/program_backend.py b/qiskit/providers/ibmq/runtime/program/program_backend.py
index c8f5480c2..293d56c05 100644
--- a/qiskit/providers/ibmq/runtime/program/program_backend.py
+++ b/qiskit/providers/ibmq/runtime/program/program_backend.py
@@ -67,4 +67,5 @@ def run(
                  the server.
             IBMQBackendValueError: If an input parameter value is not valid.
         """
+        # pylint: disable=arguments-differ
         pass
diff --git a/qiskit/providers/ibmq/runtime/program/program_template.py b/qiskit/providers/ibmq/runtime/program/program_template.py
index d1b094eea..f696f766d 100644
--- a/qiskit/providers/ibmq/runtime/program/program_template.py
+++ b/qiskit/providers/ibmq/runtime/program/program_template.py
@@ -9,6 +9,8 @@
 # Any modifications or derivative works of this code must retain this
 # copyright notice, and modified files need to carry a notice indicating
 # that they have been altered from the originals.
+# pylint: disable=unused-argument
+# pylint: disable=invalid-name
 
 """Runtime program template.
 
@@ -38,6 +40,7 @@ def main(backend: ProgramBackend, user_messenger: UserMessenger, **kwargs):
     Args:
         backend: Backend for the circuits to run on.
         user_messenger: Used to communicate with the program consumer.
+        kwargs: User inputs.
     """
     # Massage the input if necessary.
     result = program(backend, user_messenger, **kwargs)
@@ -46,10 +49,10 @@ def main(backend: ProgramBackend, user_messenger: UserMessenger, **kwargs):
 
 
 if __name__ == '__main__':
-    """This is used for testing locally using Aer simulator."""
-    _backend = Aer.get_backend('qasm_simulator')
+    # This is used for testing locally using Aer simulator.
+    sim_backend = Aer.get_backend('qasm_simulator')
     user_params = {}
     if len(sys.argv) > 1:
         # If there are user parameters.
         user_params = json.loads(sys.argv[1], cls=RuntimeDecoder)
-    main(_backend, UserMessenger(), **user_params)
+    main(sim_backend, UserMessenger(), **user_params)
diff --git a/qiskit/providers/ibmq/runtime/program/user_messenger.py b/qiskit/providers/ibmq/runtime/program/user_messenger.py
index fd696aab1..eceb6310b 100644
--- a/qiskit/providers/ibmq/runtime/program/user_messenger.py
+++ b/qiskit/providers/ibmq/runtime/program/user_messenger.py
@@ -25,10 +25,10 @@ class UserMessenger:
     """
 
     def publish(
-        self,
-        message: Any,
-        encoder: json.JSONEncoder = RuntimeEncoder,
-        final: bool = False
+            self,
+            message: Any,
+            encoder: json.JSONEncoder = RuntimeEncoder,
+            final: bool = False
     ) -> None:
         """Publish message.
 
@@ -45,5 +45,6 @@ def publish(
             encoder: An optional JSON encoder for serializing
             final: Whether the message being published is the final result.
         """
+        # pylint: disable=unused-argument
         # Default implementation for testing.
         print(json.dumps(message, cls=encoder))
diff --git a/qiskit/providers/ibmq/runtime/runtime_job.py b/qiskit/providers/ibmq/runtime/runtime_job.py
index 2589c8700..0778bc397 100644
--- a/qiskit/providers/ibmq/runtime/runtime_job.py
+++ b/qiskit/providers/ibmq/runtime/runtime_job.py
@@ -10,7 +10,7 @@
 # copyright notice, and modified files need to carry a notice indicating
 # that they have been altered from the originals.
 
-"""IBM Quantum Experience Runtime job."""
+"""IBM Quantum Experience runtime job."""
 
 from typing import Any, Optional, Callable, Dict
 import time
@@ -21,15 +21,15 @@
 import traceback
 import queue
 
-from qiskit.exceptions import QiskitError
 from qiskit.providers.exceptions import JobTimeoutError
 from qiskit.providers.backend import Backend
 from qiskit.providers.jobstatus import JobStatus, JOB_FINAL_STATES
 
 from .utils import RuntimeDecoder
 from .constants import API_TO_JOB_STATUS
-from .exceptions import RuntimeJobFailureError
+from .exceptions import RuntimeJobFailureError, RuntimeInvalidStateError
 from ..api.clients import RuntimeClient, RuntimeWebsocketClient
+from ..exceptions import IBMQError
 
 logger = logging.getLogger(__name__)
 
@@ -72,7 +72,7 @@ class RuntimeJob:
 
     def __init__(
             self,
-            backend: 'ibmqbackend.IBMQBackend',
+            backend: 'IBMQBackend',
             api_client: RuntimeClient,
             ws_client: RuntimeWebsocketClient,
             job_id: str,
@@ -99,7 +99,11 @@ def __init__(
         self._params = params or {}
         self._program_id = program_id
         self._status = JobStatus.INITIALIZING
+
+        # Used for streaming
         self._streaming = False
+        self._streaming_loop = None
+        self._streaming_task = None
         self._result_queue = queue.Queue()
 
         if user_callback is not None:
@@ -146,13 +150,16 @@ def status(self) -> JobStatus:
 
         Returns:
             Status of this job.
+
+        Raises:
+            IBMQError: If an unknown status is returned from the server.
         """
         if self._status not in JOB_FINAL_STATES:
             response = self._api_client.job_get(job_id=self.job_id())
             try:
                 self._status = API_TO_JOB_STATUS[response['status'].upper()]
             except KeyError:
-                raise QiskitError(f"Unknown status: {response['status']}")
+                raise IBMQError(f"Unknown status: {response['status']}")
         return self._status
 
     def wait_for_final_state(
@@ -179,7 +186,6 @@ def wait_for_final_state(
                     'Timeout while waiting for job {}.'.format(self.job_id()))
             time.sleep(wait)
             status = self.status()
-        return
 
     def stream_results(self, callback: Callable) -> None:
         """Start streaming job results.
@@ -189,71 +195,74 @@ def stream_results(self, callback: Callable) -> None:
                 The callback function will receive 2 position parameters:
                     1. Job ID
                     2. Job interim result.
+
+        Raises:
+            RuntimeInvalidStateError: If a callback function is already streaming results.
         """
         if self._streaming:
-            raise QiskitError("A callback function is already streaming results.")
-        self._streaming = True
+            raise RuntimeInvalidStateError("A callback function is already streaming results.")
 
         self._executor.submit(self._start_websocket_client,
                               result_queue=self._result_queue)
         self._executor.submit(self._stream_results,
                               result_queue=self._result_queue, user_callback=callback)
-        # TODO - wait for ws to connect before returning?
 
     def _cancel_result_streaming(self) -> None:
         """Cancel result streaming."""
+        # TODO - consider making this public
         if not self._streaming:
             return
-        self._result_queue.put_nowait(self._result_queue_poison_pill)
+        self._streaming_loop.call_soon_threadsafe(self._streaming_task.cancel())
 
     def _start_websocket_client(
             self,
             result_queue: queue.Queue
     ) -> None:
-        """Start websocket client to stream results."""
-        loop = None
+        """Start websocket client to stream results.
+
+        Args:
+            result_queue: Queue used to pass messages.
+        """
         try:
-            try:
-                loop = asyncio.get_event_loop()
-            except RuntimeError as ex:
-                # Event loop may not be set in a child thread.
-                if 'There is no current event loop' in str(ex):
-                    loop = asyncio.new_event_loop()
-                    asyncio.set_event_loop(loop)
-                else:
-                    logger.warning(f"Unable to get the event loop: {ex}")
-                    raise
-
-            logger.debug(f"Start websocket client for job {self.job_id()}")
-            loop.run_until_complete(self._ws_client.job_results(self._job_id, result_queue))
+            # Need new loop for the thread.
+            self._streaming_loop = asyncio.new_event_loop()
+            asyncio.set_event_loop(self._streaming_loop)
+            # TODO - use asyncio.create_task() when 3.6 is dropped.
+            self._streaming_task = self._streaming_loop.create_task(
+                self._ws_client.job_results(self._job_id, result_queue))
+            self._streaming = True
+
+            logger.debug("Start websocket client for job %s", self.job_id())
+            # loop.run_until_complete(self._ws_client.job_results(self._job_id, result_queue))
+            self._streaming_loop.run_until_complete(self._streaming_task)
         except Exception:  # pylint: disable=broad-except
             logger.warning(
-                f"An error occurred while streaming results "
-                f"from the server for job {self.job_id()}:\n{traceback.format_exc()}")
+                "An error occurred while streaming results "
+                "from the server for job %s:\n%s", self.job_id(), traceback.format_exc())
         finally:
-            result_queue.put_nowait(self._result_queue_poison_pill)
-            if loop is not None:
-                loop.run_until_complete(self._ws_client.disconnect())
+            if self._streaming_loop is not None:
+                self._streaming_loop.run_until_complete(self._ws_client.disconnect())
+            self._streaming = False
 
     def _stream_results(self, result_queue: queue.Queue, user_callback: Callable) -> None:
         """Stream interim results.
 
         Args:
+            result_queue: Queue used to pass websocket messages.
             user_callback: User callback function.
         """
-        logger.debug(f"Start result streaming for job {self.job_id()}")
+        logger.debug("Start result streaming for job %s", self.job_id())
         while True:
             try:
                 response = result_queue.get()
-                if response == self._result_queue_poison_pill:
+                if response == self._ws_client.POISON_PILL:
                     self._empty_result_queue(result_queue)
-                    self._streaming = False
                     return
                 user_callback(self.job_id(), self._decode_data(response))
             except Exception:  # pylint: disable=broad-except
                 logger.warning(
-                    f"An error occurred while streaming results "
-                    f"for job {self.job_id()}:\n{traceback.format_exc()}")
+                    "An error occurred while streaming results "
+                    "for job %s:\n%s", self.job_id(), traceback.format_exc())
 
     def _empty_result_queue(self, result_queue: queue.Queue) -> None:
         """Empty the result queue.
diff --git a/qiskit/providers/ibmq/runtime/runtime_program.py b/qiskit/providers/ibmq/runtime/runtime_program.py
index df898e291..7df0d0860 100644
--- a/qiskit/providers/ibmq/runtime/runtime_program.py
+++ b/qiskit/providers/ibmq/runtime/runtime_program.py
@@ -10,6 +10,8 @@
 # copyright notice, and modified files need to carry a notice indicating
 # that they have been altered from the originals.
 
+"""IBM Quantum Experience runtime program."""
+
 import logging
 from typing import Optional, List, NamedTuple
 
diff --git a/qiskit/providers/ibmq/runtime/utils.py b/qiskit/providers/ibmq/runtime/utils.py
index 731126d97..cf9f388cd 100644
--- a/qiskit/providers/ibmq/runtime/utils.py
+++ b/qiskit/providers/ibmq/runtime/utils.py
@@ -9,22 +9,24 @@
 # Any modifications or derivative works of this code must retain this
 # copyright notice, and modified files need to carry a notice indicating
 # that they have been altered from the originals.
+# pylint: disable=method-hidden
 
 """Utility functions for the runtime service."""
 
 import json
 from typing import Any
-import numpy as np
-import dill
 import base64
 
+import dill
+import numpy as np
+
 from qiskit.result import Result
 
 
 class RuntimeEncoder(json.JSONEncoder):
     """JSON Encoder used by runtime service."""
 
-    def default(self, obj: Any) -> Any:
+    def default(self, obj: Any) -> Any:  # pylint: disable=arguments-differ
         if hasattr(obj, 'tolist'):
             return {'__type__': 'array', '__value__': obj.tolist()}
         if isinstance(obj, complex):
@@ -47,6 +49,7 @@ def __init__(self, *args, **kwargs):
         super().__init__(object_hook=self.object_hook, *args, **kwargs)
 
     def object_hook(self, obj):
+        """Called to decode object."""
         if '__type__' in obj:
             if obj['__type__'] == 'complex':
                 val = obj['__value__']
diff --git a/test/fake_runtime_client.py b/test/fake_runtime_client.py
index 2c092662c..7ccc3c8fc 100644
--- a/test/fake_runtime_client.py
+++ b/test/fake_runtime_client.py
@@ -31,6 +31,7 @@ def __init__(self, program_id, name, data, cost=600):
         self._cost = cost
 
     def to_dict(self, include_data=False):
+        """Convert this program to a dictionary format."""
         out = {'id': self._id,
                'name': self._name,
                'cost': self._cost}
@@ -73,6 +74,7 @@ def _auto_progress(self):
             self._result = "foo"
 
     def to_dict(self):
+        """Convert to dictionary format."""
         return {'id': self._job_id,
                 'hub': self._hub,
                 'group': self._group,
diff --git a/test/ibmq/test_runtime.py b/test/ibmq/test_runtime.py
index 9306d64d1..d0015c91d 100644
--- a/test/ibmq/test_runtime.py
+++ b/test/ibmq/test_runtime.py
@@ -18,7 +18,6 @@
 from unittest.mock import patch
 import uuid
 import time
-import threading
 
 from qiskit.providers.jobstatus import JobStatus, JOB_FINAL_STATES
 from qiskit.providers.ibmq.exceptions import IBMQNotAuthorizedError
@@ -33,6 +32,7 @@
 
 @unittest.skip("Skip runtime tests")
 class TestRuntime(IBMQTestCase):
+    """Class for testing runtime modules."""
 
     @classmethod
     @requires_provider
@@ -63,6 +63,7 @@ def test_run_program(self):
         self.assertTrue(job.result())
 
     def test_interim_results(self):
+        """Test interim results."""
         def _callback(interim_result):
             print(f"interim result {interim_result}")
         params = {'param1': 'foo'}
@@ -96,6 +97,7 @@ def _callback(interim_result):
 
 @unittest.skipIf(not os.environ.get('USE_STAGING_CREDENTIALS', ''), "Only runs on staging")
 class TestRuntimeIntegration(IBMQTestCase):
+    """Integration tests for runtime modules."""
 
     RUNTIME_PROGRAM = """
 import random
@@ -147,7 +149,7 @@ def tearDownClass(cls) -> None:
         super().tearDownClass()
         try:
             cls.provider.runtime.delete_program(cls.program_id)
-        except:
+        except Exception:  # pylint: disable=broad-except
             pass
 
     def setUp(self) -> None:
@@ -163,7 +165,7 @@ def tearDown(self) -> None:
         for prog in self.to_delete:
             try:
                 self.provider.runtime.delete_program(prog)
-            except:
+            except Exception:  # pylint: disable=broad-except
                 pass
 
         # Cancel jobs.
@@ -171,7 +173,7 @@ def tearDown(self) -> None:
             if job.status() not in JOB_FINAL_STATES:
                 try:
                     job.cancel()
-                except:
+                except Exception:  # pylint: disable=broad-except
                     pass
 
     def test_runtime_service(self):
@@ -264,13 +266,13 @@ def test_run_program_failed(self):
         """Test a failed program execution."""
         options = {'backend_name': self.backend.name()}
         job = self.provider.runtime.run(program_id=self.program_id, inputs={}, options=options)
-        self.log.info(f"Runtime job {job.job_id()} submitted.")
+        self.log.info("Runtime job %s submitted.", job.job_id())
 
         job.wait_for_final_state()
         self.assertEqual(JobStatus.ERROR, job.status())
-        with self.assertRaises(RuntimeJobFailureError) as cm:
+        with self.assertRaises(RuntimeJobFailureError) as err_cm:
             job.result()
-        self.assertIn('KeyError', str(cm.exception))
+        self.assertIn('KeyError', str(err_cm.exception))
 
     def test_retrieve_job_queued(self):
         """Test retrieving a queued job."""
@@ -288,15 +290,26 @@ def test_retrieve_job_running(self):
         self.assertEqual(job.job_id(), rjob.job_id())
 
     def test_retrieve_job_done(self):
+        """Test retrieving a finished job."""
         pass
 
     def test_cancel_job_queued(self):
+        """Test canceling a queued job."""
         pass
 
+    @unittest.skip("Skip until fixed")
     def test_cancel_job_running(self):
-        pass
+        """Test canceling a running job."""
+        job = self._run_program(iterations=3, interim_results="foobar")
+        while job.status() != JobStatus.RUNNING:
+            time.sleep(5)
+        job.cancel()
+        self.assertEqual(job.status(), JobStatus.CANCELLED)
+        rjob = self.provider.runtime.job(job.job_id())
+        self.assertEqual(rjob.status(), JobStatus.CANCELLED)
 
     def test_cancel_job_done(self):
+        """Test canceling a finished job."""
         pass
 
     def test_interim_result_callback(self):
@@ -353,6 +366,7 @@ def result_callback(job_id, interim_result):
     def test_stream_results_done(self):
         """Test streaming interim results after job is done."""
         def result_callback(job_id, interim_result):
+            # pylint: disable=unused-argument
             nonlocal called_back
             called_back = True
 
@@ -367,6 +381,7 @@ def result_callback(job_id, interim_result):
     def test_callback_error(self):
         """Test error in callback method."""
         def result_callback(job_id, interim_result):
+            # pylint: disable=unused-argument
             if interim_result['iteration'] == 0:
                 raise ValueError("Kaboom!")
             nonlocal final_it
@@ -374,35 +389,40 @@ def result_callback(job_id, interim_result):
 
         final_it = 0
         iterations = 3
-        with self.assertLogs('qiskit.providers.ibmq.runtime', level='WARNING') as cm:
+        with self.assertLogs('qiskit.providers.ibmq.runtime', level='WARNING') as err_cm:
             job = self._run_program(iterations=iterations, interim_results="foo",
                                     callback=result_callback)
             job.wait_for_final_state()
 
-        self.assertIn("Kaboom", ', '.join(cm.output))
+        self.assertIn("Kaboom", ', '.join(err_cm.output))
         self.assertEqual(iterations-1, final_it)
         self.assertIsNone(job._ws_client._ws)
 
-    @unittest.skip("Skip until 277 is fixed")
-    def test_callback_job_cancelled(self):
+    # @unittest.skip("Skip until 277 is fixed")
+    def test_callback_job_cancelled_running(self):
         """Test canceling a job while streaming results."""
         def result_callback(job_id, interim_result):
-            nonlocal callback_event
-            callback_event.set()
+            # pylint: disable=unused-argument
+            nonlocal final_it
+            final_it = interim_result['iteration']
 
-        callback_event = threading.Event()
-        job = self._run_program(iterations=3, interim_results="foo",
+        final_it = 0
+        iterations = 3
+        job = self._run_program(iterations=iterations, interim_results="foo",
                                 callback=result_callback)
-
-        callback_event.wait(10)
+        while job.status() != JobStatus.RUNNING:
+            time.sleep(5)
         job.cancel()
-        time.sleep(5)  # Wait for cleanup
+        time.sleep(3)  # Wait for cleanup
         self.assertIsNone(job._ws_client._ws)
+        self.assertLess(final_it, iterations)
 
     def test_final_result(self):
+        """Test getting final result."""
         pass
 
     def test_job_status(self):
+        """Test job status."""
         pass
 
     def test_job_inputs(self):
@@ -412,7 +432,7 @@ def test_job_inputs(self):
         options = {'backend_name': self.backend.name()}
         job = self.provider.runtime.run(program_id=self.program_id, inputs=inputs,
                                         options=options)
-        self.log.info(f"Runtime job {job.job_id()} submitted.")
+        self.log.info("Runtime job %s submitted.", job.job_id())
         self.to_cancel.append(job)
         self.assertEqual(inputs, job.inputs)
 
@@ -433,6 +453,7 @@ def test_wait_for_final_state(self):
         self.assertEqual(JobStatus.DONE, job.status())
 
     def _validate_program(self, program):
+        """Validate a program."""
         # TODO add more validation
         self.assertTrue(program)
         self.assertTrue(program.name)
@@ -441,6 +462,7 @@ def _validate_program(self, program):
         self.assertTrue(program.max_execution_time)
 
     def _upload_program(self, name=None, max_execution_time=300):
+        """Upload a new program."""
         name = name or self._get_program_name()
         program_id = self.provider.runtime.upload_program(
             name=name,
@@ -450,6 +472,7 @@ def _upload_program(self, name=None, max_execution_time=300):
         return program_id
 
     def _get_program_name(self):
+        """Return a unique program name."""
         return self.PROGRAM_PREFIX + "_" + uuid.uuid4().hex
 
     def _run_program(self, program_id=None, iterations=1,
@@ -463,7 +486,7 @@ def _run_program(self, program_id=None, iterations=1,
         options = {'backend_name': self.backend.name()}
         job = self.provider.runtime.run(program_id=pid, inputs=inputs,
                                         options=options, callback=callback)
-        self.log.info(f"Runtime job {job.job_id()} submitted.")
+        self.log.info("Runtime job %s submitted.", job.job_id())
         self.to_cancel.append(job)
         return job
 
@@ -478,8 +501,10 @@ def __init__(self, value):
             self.value = value
 
         def to_json(self):
+            """To JSON serializable."""
             return {"value": self.value}
 
         @classmethod
         def from_json(cls, data):
+            """From JSON serializable."""
             return cls(**data)

From 40f29bc1fa533e3762ee5e32dc3389ae77be2ee6 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Mon, 26 Apr 2021 14:23:50 -0400
Subject: [PATCH 34/59] fix doc

---
 qiskit/providers/ibmq/credentials/credentials.py | 4 +++-
 qiskit/providers/ibmq/runtime/runtime_job.py     | 4 ++--
 2 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/qiskit/providers/ibmq/credentials/credentials.py b/qiskit/providers/ibmq/credentials/credentials.py
index 9b8455db9..61280c22a 100644
--- a/qiskit/providers/ibmq/credentials/credentials.py
+++ b/qiskit/providers/ibmq/credentials/credentials.py
@@ -85,7 +85,9 @@ def is_ibmq(self) -> bool:
         return all([self.hub, self.group, self.project])
 
     def __eq__(self, other: object) -> bool:
-        return self.__dict__ == other.__dict__
+        if not isinstance(other, Credentials):
+            return False
+        return (self.token == other.token) & (self.unique_id() == other.unique_id())
 
     def unique_id(self) -> HubGroupProject:
         """Return a value that uniquely identifies these credentials.
diff --git a/qiskit/providers/ibmq/runtime/runtime_job.py b/qiskit/providers/ibmq/runtime/runtime_job.py
index 0778bc397..12bdd7e04 100644
--- a/qiskit/providers/ibmq/runtime/runtime_job.py
+++ b/qiskit/providers/ibmq/runtime/runtime_job.py
@@ -24,6 +24,7 @@
 from qiskit.providers.exceptions import JobTimeoutError
 from qiskit.providers.backend import Backend
 from qiskit.providers.jobstatus import JobStatus, JOB_FINAL_STATES
+import qiskit.providers.ibmq.ibmqbackend as ibmqbackend
 
 from .utils import RuntimeDecoder
 from .constants import API_TO_JOB_STATUS
@@ -68,11 +69,10 @@ class RuntimeJob:
     """
 
     _executor = futures.ThreadPoolExecutor()
-    _result_queue_poison_pill = "_poison_pill"
 
     def __init__(
             self,
-            backend: 'IBMQBackend',
+            backend: 'ibmqbackend.IBMQBackend',
             api_client: RuntimeClient,
             ws_client: RuntimeWebsocketClient,
             job_id: str,

From 89ea5b4c4b95b234e0f7b91119dad30174980949 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Mon, 26 Apr 2021 14:47:11 -0400
Subject: [PATCH 35/59] fix doc

---
 qiskit/providers/ibmq/accountprovider.py     | 3 ++-
 qiskit/providers/ibmq/runtime/runtime_job.py | 2 +-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/qiskit/providers/ibmq/accountprovider.py b/qiskit/providers/ibmq/accountprovider.py
index a635f1f56..92af788bc 100644
--- a/qiskit/providers/ibmq/accountprovider.py
+++ b/qiskit/providers/ibmq/accountprovider.py
@@ -25,6 +25,7 @@
 from qiskit.providers.backend import BackendV1 as Backend
 from qiskit.providers.basebackend import BaseBackend
 from qiskit.transpiler import Layout
+import qiskit.providers.ibmq.runtime.runtime_job as runtime_job  # pylint: disable=unused-import
 
 from .api.clients import AccountClient
 from .ibmqbackend import IBMQBackend, IBMQSimulator
@@ -214,7 +215,7 @@ def run_circuits(
             meas_map: List[List[int]] = None,
             init_qubits: Optional[bool] = None,
             **run_config: Dict
-    ) -> 'RuntimeJob':
+    ) -> 'runtime_job.RuntimeJob':
         """Execute the input circuit(s) on a backend using the runtime service.
 
         Note:
diff --git a/qiskit/providers/ibmq/runtime/runtime_job.py b/qiskit/providers/ibmq/runtime/runtime_job.py
index 12bdd7e04..4d2dec67c 100644
--- a/qiskit/providers/ibmq/runtime/runtime_job.py
+++ b/qiskit/providers/ibmq/runtime/runtime_job.py
@@ -24,7 +24,7 @@
 from qiskit.providers.exceptions import JobTimeoutError
 from qiskit.providers.backend import Backend
 from qiskit.providers.jobstatus import JobStatus, JOB_FINAL_STATES
-import qiskit.providers.ibmq.ibmqbackend as ibmqbackend
+import qiskit.providers.ibmq.ibmqbackend as ibmqbackend  # pylint: disable=unused-import
 
 from .utils import RuntimeDecoder
 from .constants import API_TO_JOB_STATUS

From addb11d24e5e4fed80b8765151dee00e5b764b26 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Tue, 27 Apr 2021 08:17:12 -0400
Subject: [PATCH 36/59] add more tests

---
 .../ibmq/runtime/ibm_runtime_service.py       |  14 +-
 qiskit/providers/ibmq/runtime/runtime_job.py  |   3 +-
 test/decorators.py                            |  69 ++++--
 test/ibmq/runtime/__init__.py                 |  13 ++
 .../{ => ibmq/runtime}/fake_runtime_client.py |  58 ++++-
 test/ibmq/runtime/test_runtime.py             | 175 +++++++++++++++
 .../test_runtime_integration.py}              | 210 +++++++-----------
 test/ibmq/runtime/utils.py                    |  42 ++++
 8 files changed, 420 insertions(+), 164 deletions(-)
 create mode 100644 test/ibmq/runtime/__init__.py
 rename test/{ => ibmq/runtime}/fake_runtime_client.py (72%)
 create mode 100644 test/ibmq/runtime/test_runtime.py
 rename test/ibmq/{test_runtime.py => runtime/test_runtime_integration.py} (75%)
 create mode 100644 test/ibmq/runtime/utils.py

diff --git a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
index 116fadad1..dc21928ed 100644
--- a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
+++ b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
@@ -264,11 +264,19 @@ def job(self, job_id: str) -> RuntimeJob:
         """
         response = self._api_client.job_get(job_id)
         backend = self._provider.get_backend(response['backend'])
-        params_str = json.dumps(response.get('params', {}))
-        params = json.loads(params_str, cls=RuntimeDecoder)
+        params = response.get('params', {})
+        if isinstance(params, list):
+            if len(params) > 0:
+                params = params[0]
+            else:
+                params = {}
+        if not isinstance(params, str):
+            params = json.dumps(params)
+
+        decoded = json.loads(params, cls=RuntimeDecoder)
         return RuntimeJob(backend=backend,
                           api_client=self._api_client,
                           ws_client=RuntimeWebsocketClient(self._ws_url, self._access_token),
                           job_id=response['id'],
                           program_id=response.get('program', {}).get('id', ""),
-                          params=params)
+                          params=decoded)
diff --git a/qiskit/providers/ibmq/runtime/runtime_job.py b/qiskit/providers/ibmq/runtime/runtime_job.py
index 4d2dec67c..7fccd4a5c 100644
--- a/qiskit/providers/ibmq/runtime/runtime_job.py
+++ b/qiskit/providers/ibmq/runtime/runtime_job.py
@@ -24,7 +24,7 @@
 from qiskit.providers.exceptions import JobTimeoutError
 from qiskit.providers.backend import Backend
 from qiskit.providers.jobstatus import JobStatus, JOB_FINAL_STATES
-import qiskit.providers.ibmq.ibmqbackend as ibmqbackend  # pylint: disable=unused-import
+from qiskit.providers.ibmq import ibmqbackend  # pylint: disable=unused-import
 
 from .utils import RuntimeDecoder
 from .constants import API_TO_JOB_STATUS
@@ -193,6 +193,7 @@ def stream_results(self, callback: Callable) -> None:
         Args:
             callback: Callback function to be invoked for any interim results.
                 The callback function will receive 2 position parameters:
+
                     1. Job ID
                     2. Job interim result.
 
diff --git a/test/decorators.py b/test/decorators.py
index 930dc152c..68636c16f 100644
--- a/test/decorators.py
+++ b/test/decorators.py
@@ -191,36 +191,67 @@ def requires_device(func):
     @requires_qe_access
     def _wrapper(obj, *args, **kwargs):
 
-        _enable_account(kwargs.pop('qe_token'), kwargs.pop('qe_url'))
-
         backend_name = os.getenv('QE_STAGING_DEVICE', None) if \
             os.getenv('USE_STAGING_CREDENTIALS', '') else os.getenv('QE_DEVICE', None)
 
-        _backend = None
-        provider = _get_custom_provider(IBMQ) or list(IBMQ._providers.values())[0]
+        _backend = _get_backend(qe_token=kwargs.pop('qe_token'),
+                                qe_url=kwargs.pop('qe_url'),
+                                backend_name=backend_name)
+        kwargs.update({'backend': _backend})
+        return func(obj, *args, **kwargs)
 
-        if backend_name:
-            # Put desired provider as the first in the list.
-            providers = [provider] + IBMQ.providers()
-            for provider in providers:
-                backends = provider.backends(name=backend_name)
-                if backends:
-                    _backend = backends[0]
-                    break
-        else:
-            _backend = least_busy(provider.backends(
-                simulator=False, min_num_qubits=5))
-
-        if not _backend:
-            raise Exception('Unable to find a suitable backend.')
+    return _wrapper
 
-        kwargs.update({'backend': _backend})
 
+def requires_runtime_device(func):
+    """Decorator that retrieves the appropriate backend to use for testing.
+
+    Args:
+        func (callable): test function to be decorated.
+
+    Returns:
+        callable: the decorated function.
+    """
+    @wraps(func)
+    @requires_qe_access
+    def _wrapper(obj, *args, **kwargs):
+
+        backend_name = os.getenv('QE_STAGING_RUNTIME_DEVICE', None) if \
+            os.getenv('USE_STAGING_CREDENTIALS', '') else os.getenv('QE_RUNTIME_DEVICE', None)
+        _backend = _get_backend(qe_token=kwargs.pop('qe_token'),
+                                qe_url=kwargs.pop('qe_url'),
+                                backend_name=backend_name)
+        kwargs.update({'backend': _backend})
         return func(obj, *args, **kwargs)
 
     return _wrapper
 
 
+def _get_backend(qe_token, qe_url, backend_name):
+    """Get the specified backend."""
+    _enable_account(qe_token, qe_url)
+
+    _backend = None
+    provider = _get_custom_provider(IBMQ) or list(IBMQ._providers.values())[0]
+
+    if backend_name:
+        # Put desired provider as the first in the list.
+        providers = [provider] + IBMQ.providers()
+        for provider in providers:
+            backends = provider.backends(name=backend_name)
+            if backends:
+                _backend = backends[0]
+                break
+    else:
+        _backend = least_busy(provider.backends(
+            simulator=False, min_num_qubits=5))
+
+    if not _backend:
+        raise Exception('Unable to find a suitable backend.')
+
+    return _backend
+
+
 def _get_credentials():
     """Finds the credentials for a specific test and options.
 
diff --git a/test/ibmq/runtime/__init__.py b/test/ibmq/runtime/__init__.py
new file mode 100644
index 000000000..99de8ba78
--- /dev/null
+++ b/test/ibmq/runtime/__init__.py
@@ -0,0 +1,13 @@
+# This code is part of Qiskit.
+#
+# (C) Copyright IBM 2021.
+#
+# This code is licensed under the Apache License, Version 2.0. You may
+# obtain a copy of this license in the LICENSE.txt file in the root directory
+# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Any modifications or derivative works of this code must retain this
+# copyright notice, and modified files need to carry a notice indicating
+# that they have been altered from the originals.
+
+"""Runtime related tests."""
diff --git a/test/fake_runtime_client.py b/test/ibmq/runtime/fake_runtime_client.py
similarity index 72%
rename from test/fake_runtime_client.py
rename to test/ibmq/runtime/fake_runtime_client.py
index 7ccc3c8fc..67bc880cd 100644
--- a/test/fake_runtime_client.py
+++ b/test/ibmq/runtime/fake_runtime_client.py
@@ -44,7 +44,7 @@ class BaseFakeRuntimeJob:
     """Base class for faking a runtime job."""
 
     _job_progress = [
-        "PENDING",
+        "QUEUED",
         "RUNNING",
         "SUCCEEDED"
     ]
@@ -54,7 +54,7 @@ class BaseFakeRuntimeJob:
     def __init__(self, job_id, program_id, hub, group, project, backend_name, params):
         """Initialize a fake job."""
         self._job_id = job_id
-        self._status = "PENDING"
+        self._status = "QUEUED"
         self._program_id = program_id
         self._hub = hub
         self._group = group
@@ -83,6 +83,27 @@ def to_dict(self):
                 'status': self._status,
                 'params': [self._params]}
 
+    def result(self):
+        """Return job result."""
+        return self._result
+
+
+class FailedRuntimeJob(BaseFakeRuntimeJob):
+    """Base class for faking a runtime job."""
+
+    _job_progress = [
+        "QUEUED",
+        "RUNNING",
+        "FAILED"
+    ]
+
+    def _auto_progress(self):
+        """Automatically update job status."""
+        super()._auto_progress()
+
+        if self._status == "FAILED":
+            self._result = "Kaboom!"
+
 
 class BaseFakeRuntimeClient:
     """Base class for faking the runtime client."""
@@ -91,6 +112,13 @@ def __init__(self):
         """Initialize a fake runtime client."""
         self._programs = {}
         self._jobs = {}
+        self._job_classes = []
+
+    def set_job_classes(self, classes):
+        """Set job classes to use."""
+        if not isinstance(classes, list):
+            classes = [classes]
+        self._job_classes = classes
 
     def list_programs(self):
         """List all progrmas."""
@@ -99,17 +127,20 @@ def list_programs(self):
             programs.append(prog.to_dict())
         return programs
 
-    def program_create(self, program_name, program_data):
+    def program_create(self, program_name, program_data, max_execution_time):
         """Create a program."""
         if isinstance(program_data, str):
             with open(program_data, 'rb') as file:
                 program_data = file.read()
         program_id = uuid.uuid4().hex
-        self._programs[program_id] = BaseFakeProgram(program_id, program_name, program_data)
+        self._programs[program_id] = BaseFakeProgram(program_id, program_name, program_data,
+                                                     max_execution_time)
         return {'id': program_id}
 
     def program_get(self, program_id: str):
         """Return a specific program."""
+        if program_id not in self._programs:
+            raise RequestsApiError("Program not found", status_code=404)
         return self._programs[program_id].to_dict()
 
     def program_get_data(self, program_id: str):
@@ -125,10 +156,11 @@ def program_run(
     ):
         """Run the specified program."""
         job_id = uuid.uuid4().hex
-        job = BaseFakeRuntimeJob(job_id=job_id, program_id=program_id,
-                                 hub=credentials.hub, group=credentials.group,
-                                 project=credentials.project, backend_name=backend_name,
-                                 params=params)
+        job_cls = self._job_classes.pop(0) if len(self._job_classes) > 0 else BaseFakeRuntimeJob
+        job = job_cls(job_id=job_id, program_id=program_id,
+                      hub=credentials.hub, group=credentials.group,
+                      project=credentials.project, backend_name=backend_name,
+                      params=params)
         self._jobs[job_id] = job
         return {'id': job_id}
 
@@ -138,12 +170,16 @@ def program_delete(self, program_id: str) -> None:
             raise RequestsApiError("Program not found")
         del self._programs[program_id]
 
-    def program_job_get(self, job_id):
+    def job_get(self, job_id):
         """Get the specific job."""
         if job_id not in self._jobs:
-            raise RequestsApiError("Job not found")
+            raise RequestsApiError("Job not found", status_code=404)
         return self._jobs[job_id].to_dict()
 
-    def program_job_results(self, job_id: str):
+    def job_results(self, job_id):
         """Get the results of a program job."""
+        return self._jobs[job_id].result()
+
+    def job_cancel(self, job_id):
+        """Cancel the job."""
         pass
diff --git a/test/ibmq/runtime/test_runtime.py b/test/ibmq/runtime/test_runtime.py
new file mode 100644
index 000000000..a295b1979
--- /dev/null
+++ b/test/ibmq/runtime/test_runtime.py
@@ -0,0 +1,175 @@
+# This code is part of Qiskit.
+#
+# (C) Copyright IBM 2021.
+#
+# This code is licensed under the Apache License, Version 2.0. You may
+# obtain a copy of this license in the LICENSE.txt file in the root directory
+# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Any modifications or derivative works of this code must retain this
+# copyright notice, and modified files need to carry a notice indicating
+# that they have been altered from the originals.
+
+"""Tests for runtime service."""
+
+import json
+from io import StringIO
+from unittest.mock import patch
+from unittest import mock
+import uuid
+
+import numpy as np
+from qiskit.result import Result
+from qiskit.providers.jobstatus import JobStatus
+from qiskit.providers.ibmq.runtime.utils import RuntimeEncoder, RuntimeDecoder
+from qiskit.providers.ibmq.accountprovider import AccountProvider
+from qiskit.providers.ibmq.runtime import IBMRuntimeService, RuntimeJob
+from qiskit.providers.ibmq.runtime.exceptions import RuntimeProgramNotFound
+
+
+from ...ibmqtestcase import IBMQTestCase
+from .fake_runtime_client import BaseFakeRuntimeClient
+from .utils import SerializableClass, UnserializableClass
+
+
+class TestRuntime(IBMQTestCase):
+    """Class for testing runtime modules."""
+
+    def setUp(self):
+        """Initial test setup."""
+        super().setUp()
+        self.runtime = IBMRuntimeService(mock.MagicMock(sepc=AccountProvider))
+        self.runtime._api_client = BaseFakeRuntimeClient()
+
+    def test_coder(self):
+        """Test runtime encoder and decoder."""
+        result = Result(backend_name='ibmqx2',
+                        backend_version='1.1',
+                        qobj_id='12345',
+                        job_id='67890',
+                        success=False,
+                        results=[])
+
+        data = {"string": "foo",
+                "float": 1.5,
+                "complex": 2+3j,
+                "array": np.array([[1, 2, 3], [4, 5, 6]]),
+                "result": result,
+                "sclass": SerializableClass("foo"),
+                "usclass": UnserializableClass("bar"),
+                }
+        encoded = json.dumps(data, cls=RuntimeEncoder)
+        decoded = json.loads(encoded, cls=RuntimeDecoder)
+        decoded["sclass"] = SerializableClass.from_json(**decoded['sclass'])
+
+        decoded_result = decoded.pop('result')
+        data.pop('result')
+
+        decoded_array = decoded.pop('array')
+        orig_array = data.pop('array')
+
+        self.assertEqual(decoded, data)
+        self.assertIsInstance(decoded_result, Result)
+        self.assertTrue((decoded_array == orig_array).all())
+
+    def test_list_programs(self):
+        """Test listing programs."""
+        program_id = self._upload_program()
+        programs = self.runtime.programs()
+        all_ids = [prog.program_id for prog in programs]
+        self.assertIn(program_id, all_ids)
+
+    def test_list_program(self):
+        """Test listing a single program."""
+        program_id = self._upload_program()
+        program = self.runtime.program(program_id)
+        self.assertEqual(program_id, program.program_id)
+
+    def test_print_programs(self):
+        """Test printing programs."""
+        ids = []
+        for idx in range(3):
+            ids.append(self._upload_program(name=f"name_{idx}"))
+
+        programs = self.runtime.programs()
+        with patch('sys.stdout', new=StringIO()) as mock_stdout:
+            self.runtime.pprint_programs()
+            stdout = mock_stdout.getvalue()
+            for prog in programs:
+                self.assertIn(prog.program_id, stdout)
+                self.assertIn(prog.name, stdout)
+                # self.assertIn(prog.description, stdout)  TODO - add when enabled
+
+    def test_upload_program(self):
+        """Test uploading a program."""
+        max_execution_time = 3000
+        program_id = self._upload_program(max_execution_time=max_execution_time)
+        self.assertTrue(program_id)
+        program = self.runtime.program(program_id)
+        self.assertTrue(program)
+        self.assertEqual(max_execution_time, program.max_execution_time)
+
+    def test_delete_program(self):
+        """Test deleting program."""
+        program_id = self._upload_program()
+        self.runtime.delete_program(program_id)
+        with self.assertRaises(RuntimeProgramNotFound):
+            self.runtime.program(program_id, refresh=True)
+
+    def test_double_delete_program(self):
+        """Test deleting a deleted program."""
+        program_id = self._upload_program()
+        self.runtime.delete_program(program_id)
+        with self.assertRaises(RuntimeProgramNotFound):
+            self.runtime.delete_program(program_id)
+
+    def test_run_program(self):
+        """Test running program."""
+        program_id = self._upload_program()
+        params = {'param1': 'foo'}
+        job = self._run_program(program_id, inputs=params)
+        self.assertTrue(job.job_id())
+        self.assertIsInstance(job, RuntimeJob)
+        self.assertIsInstance(job.status(), JobStatus)
+        self.assertEqual(job.inputs, params)
+        job.wait_for_final_state()
+        self.assertEqual(job.status(), JobStatus.DONE)
+        self.assertTrue(job.result())
+
+    # def test_run_program_failed(self):
+    #     """Test a failed program execution."""
+    #     options = {'backend_name': self.backend.name()}
+    #     job = self.provider.runtime.run(program_id=self.program_id, inputs={}, options=options)
+    #     self.log.info("Runtime job %s submitted.", job.job_id())
+    #
+    #     job.wait_for_final_state()
+    #     self.assertEqual(JobStatus.ERROR, job.status())
+    #     with self.assertRaises(RuntimeJobFailureError) as err_cm:
+    #         job.result()
+    #     self.assertIn('KeyError', str(err_cm.exception))
+    #
+    # def test_interim_results(self):
+    #     """Test interim results."""
+    #     def _callback(interim_result):
+    #         print(f"interim result {interim_result}")
+    #     params = {'param1': 'foo'}
+    #     backend = self.provider.backend.ibmq_qasm_simulator
+    #     job = self.provider.runtime.run("QKA", backend=backend, params=params, callback=_callback)
+    #     job.result()
+
+    def _upload_program(self, name=None, max_execution_time=300):
+        """Upload a new program."""
+        name = name or uuid.uuid4().hex
+        data = "A fancy program"
+        program_id = self.runtime.upload_program(
+            name=name,
+            data=data.encode(),
+            max_execution_time=max_execution_time)
+        return program_id
+
+    def _run_program(self, program_id, inputs=None):
+        """Run a program."""
+        options = {'backend_name': "some_backend"}
+        job = self.runtime.run(program_id=program_id, inputs=inputs,
+                               options=options)
+        return job
diff --git a/test/ibmq/test_runtime.py b/test/ibmq/runtime/test_runtime_integration.py
similarity index 75%
rename from test/ibmq/test_runtime.py
rename to test/ibmq/runtime/test_runtime_integration.py
index d0015c91d..5e205bd23 100644
--- a/test/ibmq/test_runtime.py
+++ b/test/ibmq/runtime/test_runtime_integration.py
@@ -14,10 +14,9 @@
 
 import unittest
 import os
-from io import StringIO
-from unittest.mock import patch
 import uuid
 import time
+import random
 
 from qiskit.providers.jobstatus import JobStatus, JOB_FINAL_STATES
 from qiskit.providers.ibmq.exceptions import IBMQNotAuthorizedError
@@ -25,75 +24,13 @@
                                                       RuntimeProgramNotFound,
                                                       RuntimeJobFailureError)
 
-from ..ibmqtestcase import IBMQTestCase
-from ..decorators import requires_device, requires_provider
-from ..fake_runtime_client import BaseFakeRuntimeClient
-
-
-@unittest.skip("Skip runtime tests")
-class TestRuntime(IBMQTestCase):
-    """Class for testing runtime modules."""
-
-    @classmethod
-    @requires_provider
-    def setUpClass(cls, provider):
-        """Initial class level setup."""
-        # pylint: disable=arguments-differ
-        super().setUpClass()
-        cls.provider = provider
-
-    def setUp(self):
-        """Initial test setup."""
-        super().setUp()
-        self.provider.runtime._api_client = BaseFakeRuntimeClient()
-
-    def test_list_programs(self):
-        """Test listing programs."""
-        self.provider.runtime.programs()
-
-    def test_run_program(self):
-        """Test running program."""
-        params = {'param1': 'foo'}
-        backend = self.provider.backend.ibmq_qasm_simulator
-        job = self.provider.runtime.run("QKA", backend=backend, params=params)
-        self.assertTrue(job.job_id())
-        self.assertIsInstance(job.status(), JobStatus)
-        job.wait_for_final_state()
-        self.assertEqual(job.status(), JobStatus.DONE)
-        self.assertTrue(job.result())
+from ...ibmqtestcase import IBMQTestCase
+from ...decorators import requires_runtime_device
+from .utils import SerializableClass
 
-    def test_interim_results(self):
-        """Test interim results."""
-        def _callback(interim_result):
-            print(f"interim result {interim_result}")
-        params = {'param1': 'foo'}
-        backend = self.provider.backend.ibmq_qasm_simulator
-        job = self.provider.runtime.run("QKA", backend=backend, params=params, callback=_callback)
-        job.result()
 
+# os.environ['USE_STAGING_CREDENTIALS'] = 'true'
 
-# import random
-#
-# from qiskit import transpile
-# from qiskit.circuit.random import random_circuit
-# from qiskit.providers.ibmq.runtime.utils import RuntimeEncoder
-#
-# def prepare_circuits(backend):
-#     circuit = random_circuit(num_qubits=5, depth=4, measure=True,
-#                              seed=random.randint(0, 1000))
-#     return transpile(circuit, backend)
-#
-# def main(backend, user_messenger, **kwargs):
-#     iterations = kwargs.pop('iterations', 5)
-#     interim_results = kwargs.pop('interim_results', {})
-#     final_result = kwargs.pop("final_result", {})
-#     for it in range(iterations):
-#         qc = prepare_circuits(backend)
-#         user_messenger.publish({"iteration": it, "interim_results": interim_results})
-#         backend.run(qc).result()
-#
-#     user_messenger.publish("this is the last message")
-#     print(final_result, cls=RuntimeEncoder)
 
 @unittest.skipIf(not os.environ.get('USE_STAGING_CREDENTIALS', ''), "Only runs on staging")
 class TestRuntimeIntegration(IBMQTestCase):
@@ -125,7 +62,7 @@ def main(backend, user_messenger, **kwargs):
     PROGRAM_PREFIX = 'qiskit-test'
 
     @classmethod
-    @requires_device
+    @requires_runtime_device
     def setUpClass(cls, backend):
         """Initial class level setup."""
         # pylint: disable=arguments-differ
@@ -197,17 +134,6 @@ def test_list_program(self):
         self.assertEqual(self.program_id, program.program_id)
         self._validate_program(program)
 
-    def test_print_programs(self):
-        """Test printing programs."""
-        programs = self.provider.runtime.programs()
-        with patch('sys.stdout', new=StringIO()) as mock_stdout:
-            self.provider.runtime.pprint_programs()
-            stdout = mock_stdout.getvalue()
-            for prog in programs:
-                self.assertIn(prog.program_id, stdout)
-                self.assertIn(prog.name, stdout)
-                self.assertIn(prog.description, stdout)
-
     def test_upload_program(self):
         """Test uploading a program."""
         max_execution_time = 3000
@@ -252,15 +178,10 @@ def test_double_delete_program(self):
 
     def test_run_program(self):
         """Test running a program."""
-        final_result = {"string": "foo",
-                        "float": 1.5,
-                        "complex": 2+3j,
-                        "class": self.CustomClass("foo")}
-        job = self._run_program(final_result=final_result)
+        job = self._run_program(final_result="foo")
         result = job.result()
-        my_class = self.CustomClass.from_json(result.pop('class'))
-        self.assertEqual(final_result.pop('class').value, my_class.value)
-        self.assertEqual(final_result, result)
+        self.assertEqual(JobStatus.DONE, job.status())
+        self.assertEqual("foo", result)
 
     def test_run_program_failed(self):
         """Test a failed program execution."""
@@ -276,31 +197,46 @@ def test_run_program_failed(self):
 
     def test_retrieve_job_queued(self):
         """Test retrieving a queued job."""
-        pass
+        _ = self._run_program(iterations=10)
+        job = self._run_program(iterations=2)
+        while job.status() != JobStatus.QUEUED:
+            time.sleep(1)
+        rjob = self.provider.runtime.job(job.job_id())
+        self.assertEqual(job.job_id(), rjob.job_id())
+        self.assertEqual(self.program_id, job.program_id)
 
     def test_retrieve_job_running(self):
         """Test retrieving a running job."""
         job = self._run_program(iterations=10)
-        for _ in range(10):
-            if job.status() == JobStatus.RUNNING:
-                break
-            time.sleep(1)
-        self.assertEqual(JobStatus.RUNNING, job.status())
+        while job.status() != JobStatus.RUNNING:
+            time.sleep(5)
         rjob = self.provider.runtime.job(job.job_id())
         self.assertEqual(job.job_id(), rjob.job_id())
+        self.assertEqual(self.program_id, job.program_id)
 
     def test_retrieve_job_done(self):
         """Test retrieving a finished job."""
-        pass
+        job = self._run_program()
+        job.wait_for_final_state()
+        rjob = self.provider.runtime.job(job.job_id())
+        self.assertEqual(job.job_id(), rjob.job_id())
+        self.assertEqual(self.program_id, job.program_id)
 
     def test_cancel_job_queued(self):
         """Test canceling a queued job."""
-        pass
+        _ = self._run_program(iterations=10)
+        job = self._run_program(iterations=2)
+        while job.status() != JobStatus.QUEUED:
+            time.sleep(1)
+        job.cancel()
+        self.assertEqual(job.status(), JobStatus.CANCELLED)
+        rjob = self.provider.runtime.job(job.job_id())
+        self.assertEqual(rjob.status(), JobStatus.CANCELLED)
 
     @unittest.skip("Skip until fixed")
     def test_cancel_job_running(self):
         """Test canceling a running job."""
-        job = self._run_program(iterations=3, interim_results="foobar")
+        job = self._run_program(iterations=3)
         while job.status() != JobStatus.RUNNING:
             time.sleep(5)
         job.cancel()
@@ -310,7 +246,9 @@ def test_cancel_job_running(self):
 
     def test_cancel_job_done(self):
         """Test canceling a finished job."""
-        pass
+        job = self._run_program()
+        job.wait_for_final_state()
+        job.cancel()
 
     def test_interim_result_callback(self):
         """Test interim result callback."""
@@ -399,8 +337,8 @@ def result_callback(job_id, interim_result):
         self.assertIsNone(job._ws_client._ws)
 
     # @unittest.skip("Skip until 277 is fixed")
-    def test_callback_job_cancelled_running(self):
-        """Test canceling a job while streaming results."""
+    def test_callback_cancel_job(self):
+        """Test canceling a running job while streaming results."""
         def result_callback(job_id, interim_result):
             # pylint: disable=unused-argument
             nonlocal final_it
@@ -408,33 +346,52 @@ def result_callback(job_id, interim_result):
 
         final_it = 0
         iterations = 3
-        job = self._run_program(iterations=iterations, interim_results="foo",
-                                callback=result_callback)
-        while job.status() != JobStatus.RUNNING:
-            time.sleep(5)
-        job.cancel()
-        time.sleep(3)  # Wait for cleanup
-        self.assertIsNone(job._ws_client._ws)
-        self.assertLess(final_it, iterations)
+        sub_tests = [JobStatus.QUEUED, JobStatus.RUNNING]
+
+        for status in sub_tests:
+            with self.subTest(status=status):
+                if status == JobStatus.QUEUED:
+                    _ = self._run_program(iterations=10)
+
+                job = self._run_program(iterations=iterations, interim_results="foo",
+                                        callback=result_callback)
+                while job.status() != status:
+                    time.sleep(5)
+                job.cancel()
+                time.sleep(3)  # Wait for cleanup
+                self.assertIsNone(job._ws_client._ws)
+                self.assertLess(final_it, iterations)
 
     def test_final_result(self):
         """Test getting final result."""
-        pass
+        final_result = self._get_complex_types()
+        job = self._run_program(final_result=final_result)
+        result = job.result()
+        self._assert_complex_types_equal(final_result, result)
+
+        rresults = self.provider.runtime.job(job.job_id()).result()
+        self._assert_complex_types_equal(final_result, rresults)
 
     def test_job_status(self):
         """Test job status."""
-        pass
+        job = self._run_program(iterations=1)
+        time.sleep(random.randint(1, 5))
+        job.status()
 
     def test_job_inputs(self):
         """Test job inputs."""
+        interim_results = self._get_complex_types()
         inputs = {'iterations': 1,
-                  'interim_results': "foo"}
+                  'interim_results': interim_results}
         options = {'backend_name': self.backend.name()}
         job = self.provider.runtime.run(program_id=self.program_id, inputs=inputs,
                                         options=options)
         self.log.info("Runtime job %s submitted.", job.job_id())
         self.to_cancel.append(job)
         self.assertEqual(inputs, job.inputs)
+        rjob = self.provider.runtime.job(job.job_id())
+        rinterim_results = rjob.inputs['interim_results']
+        self._assert_complex_types_equal(interim_results, rinterim_results)
 
     def test_job_backend(self):
         """Test job backend."""
@@ -475,6 +432,18 @@ def _get_program_name(self):
         """Return a unique program name."""
         return self.PROGRAM_PREFIX + "_" + uuid.uuid4().hex
 
+    def _get_complex_types(self):
+        return {"string": "foo",
+                "float": 1.5,
+                "complex": 2+3j,
+                "class": SerializableClass("foo")}
+
+    def _assert_complex_types_equal(self, expected, received):
+        """Verify the received data in complex types is expected."""
+        if 'class' in received:
+            received['class'] = SerializableClass.from_json(**received['class'])
+        self.assertEqual(expected, received)
+
     def _run_program(self, program_id=None, iterations=1,
                      interim_results=None, final_result=None,
                      callback=None):
@@ -489,22 +458,3 @@ def _run_program(self, program_id=None, iterations=1,
         self.log.info("Runtime job %s submitted.", job.job_id())
         self.to_cancel.append(job)
         return job
-
-    # iterations = kwargs.pop('iterations', 5)
-    # interim_results = kwargs.pop('interim_results', {})
-    # final_result = kwargs.pop("final_result", {})
-    # test_lp1_sw_renierm
-
-    class CustomClass:
-        """Custom class with serialization methods."""
-        def __init__(self, value):
-            self.value = value
-
-        def to_json(self):
-            """To JSON serializable."""
-            return {"value": self.value}
-
-        @classmethod
-        def from_json(cls, data):
-            """From JSON serializable."""
-            return cls(**data)
diff --git a/test/ibmq/runtime/utils.py b/test/ibmq/runtime/utils.py
new file mode 100644
index 000000000..81ee35793
--- /dev/null
+++ b/test/ibmq/runtime/utils.py
@@ -0,0 +1,42 @@
+# This code is part of Qiskit.
+#
+# (C) Copyright IBM 2020.
+#
+# This code is licensed under the Apache License, Version 2.0. You may
+# obtain a copy of this license in the LICENSE.txt file in the root directory
+# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Any modifications or derivative works of this code must retain this
+# copyright notice, and modified files need to carry a notice indicating
+# that they have been altered from the originals.
+
+"""Utility functions for runtime testing."""
+
+
+class SerializableClass:
+    """Custom class with serialization methods."""
+
+    def __init__(self, value):
+        self.value = value
+
+    def to_json(self):
+        """To JSON serializable."""
+        return {"value": self.value}
+
+    @classmethod
+    def from_json(cls, value):
+        """From JSON serializable."""
+        return cls(value=value)
+
+    def __eq__(self, other):
+        return self.value == other.value
+
+
+class UnserializableClass:
+    """Custom class without serialization methods."""
+
+    def __init__(self, value):
+        self.value = value
+
+    def __eq__(self, other):
+        return self.value == other.value

From 72cdf156e3b6298f2248e084238a995727d631a1 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Tue, 27 Apr 2021 09:06:46 -0400
Subject: [PATCH 37/59] fix mypy

---
 .github/workflows/main.yml                    |  1 -
 qiskit/providers/ibmq/api/clients/runtime.py  | 13 ++-
 .../providers/ibmq/api/clients/runtime_ws.py  |  2 +-
 qiskit/providers/ibmq/api/rest/runtime.py     |  2 +-
 .../ibmq/runtime/ibm_runtime_service.py       |  4 +-
 .../ibmq/runtime/program/program_backend.py   |  4 +-
 .../ibmq/runtime/program/user_messenger.py    |  2 +-
 qiskit/providers/ibmq/runtime/runtime_job.py  |  7 +-
 .../providers/ibmq/runtime/runtime_program.py |  2 +-
 qiskit/providers/ibmq/runtime/utils.py        |  4 +-
 test/ibmq/runtime/fake_runtime_client.py      | 23 ++++-
 test/ibmq/runtime/test_runtime.py             | 84 +++++++++++++++----
 test/ibmq/runtime/test_runtime_integration.py |  4 +-
 13 files changed, 112 insertions(+), 40 deletions(-)

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
index 72e4dd095..30b75bd4c 100644
--- a/.github/workflows/main.yml
+++ b/.github/workflows/main.yml
@@ -78,7 +78,6 @@ jobs:
         run: |
           python -m pip install --upgrade pip
           pip install -U -c constraints.txt -r requirements-dev.txt
-          pip install -U git+https://github.com/Qiskit/qiskit-aer.git
           pip install -c constraints.txt -e .
       - name: Run Tests
         run: make test
diff --git a/qiskit/providers/ibmq/api/clients/runtime.py b/qiskit/providers/ibmq/api/clients/runtime.py
index f019f8216..23763fba8 100644
--- a/qiskit/providers/ibmq/api/clients/runtime.py
+++ b/qiskit/providers/ibmq/api/clients/runtime.py
@@ -118,8 +118,15 @@ def program_delete(self, program_id: str) -> None:
         """
         self.api.program(program_id).delete()
 
-    def job_get(self, job_id):
-        """Get job data."""
+    def job_get(self, job_id: str) -> Dict:
+        """Get job data.
+
+        Args:
+            job_id: Job ID.
+
+        Returns:
+            JSON response.
+        """
         response = self.api.program_job(job_id).get()
         logger.debug("Runtime job get response: %s", response)
         return response
@@ -131,7 +138,7 @@ def job_results(self, job_id: str) -> str:
             job_id: Program job ID.
 
         Returns:
-            JSON response.
+            Job result.
         """
         return self.api.program_job(job_id).results()
 
diff --git a/qiskit/providers/ibmq/api/clients/runtime_ws.py b/qiskit/providers/ibmq/api/clients/runtime_ws.py
index e02ce2ab3..7f7f018e9 100644
--- a/qiskit/providers/ibmq/api/clients/runtime_ws.py
+++ b/qiskit/providers/ibmq/api/clients/runtime_ws.py
@@ -167,6 +167,6 @@ def _backoff_time(self, backoff_factor: float, current_retry_attempt: int) -> fl
     async def disconnect(self) -> None:
         """Close the websocket connection."""
         if self._ws is not None:
-            logger.debug("Closing runtime websocket connection.")
+            logger.debug("Closing runtime websocket connection.")  # type: ignore[unreachable]
             await self._ws.close()
             self._ws = None
diff --git a/qiskit/providers/ibmq/api/rest/runtime.py b/qiskit/providers/ibmq/api/rest/runtime.py
index d106d8b31..80a14ddcd 100644
--- a/qiskit/providers/ibmq/api/rest/runtime.py
+++ b/qiskit/providers/ibmq/api/rest/runtime.py
@@ -42,7 +42,7 @@ def program(self, program_id: str) -> 'Program':
         """
         return Program(self.session, program_id)
 
-    def program_job(self, job_id: str) -> None:
+    def program_job(self, job_id: str) -> 'ProgramJob':
         """Return an adapter for the job.
 
         Args:
diff --git a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
index dc21928ed..58be173f0 100644
--- a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
+++ b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
@@ -85,7 +85,7 @@ def __init__(self, provider: 'accountprovider.AccountProvider') -> None:
         self._api_client = RuntimeClient(provider.credentials)
         self._access_token = provider.credentials.access_token
         self._ws_url = provider.credentials.runtime_url.replace('https', 'wss')
-        self._programs = {}
+        self._programs = {}  # type: Dict
 
     def pprint_programs(self, refresh: bool = False) -> None:
         """Pretty print information about available runtime programs.
@@ -233,7 +233,7 @@ def upload_program(
             raise QiskitRuntimeError(f"Failed to create program: {ex}") from None
         return response['id']
 
-    def delete_program(self, program_id: str):
+    def delete_program(self, program_id: str) -> None:
         """Delete a runtime program.
 
         Args:
diff --git a/qiskit/providers/ibmq/runtime/program/program_backend.py b/qiskit/providers/ibmq/runtime/program/program_backend.py
index 293d56c05..6d54e52e1 100644
--- a/qiskit/providers/ibmq/runtime/program/program_backend.py
+++ b/qiskit/providers/ibmq/runtime/program/program_backend.py
@@ -18,8 +18,8 @@
 
 from qiskit.qobj import QasmQobj, PulseQobj
 from qiskit.pulse import Schedule
-from qiskit.providers import BackendV1 as Backend
-from qiskit.providers import JobV1 as Job
+from qiskit.providers.backend import BackendV1 as Backend
+from qiskit.providers.job import JobV1 as Job
 from qiskit.circuit import QuantumCircuit
 
 logger = logging.getLogger(__name__)
diff --git a/qiskit/providers/ibmq/runtime/program/user_messenger.py b/qiskit/providers/ibmq/runtime/program/user_messenger.py
index eceb6310b..2a4df7571 100644
--- a/qiskit/providers/ibmq/runtime/program/user_messenger.py
+++ b/qiskit/providers/ibmq/runtime/program/user_messenger.py
@@ -27,7 +27,7 @@ class UserMessenger:
     def publish(
             self,
             message: Any,
-            encoder: json.JSONEncoder = RuntimeEncoder,
+            encoder: json.JSONEncoder = RuntimeEncoder,  # type: ignore[assignment]
             final: bool = False
     ) -> None:
         """Publish message.
diff --git a/qiskit/providers/ibmq/runtime/runtime_job.py b/qiskit/providers/ibmq/runtime/runtime_job.py
index 7fccd4a5c..c6b5bba9e 100644
--- a/qiskit/providers/ibmq/runtime/runtime_job.py
+++ b/qiskit/providers/ibmq/runtime/runtime_job.py
@@ -104,7 +104,7 @@ def __init__(
         self._streaming = False
         self._streaming_loop = None
         self._streaming_task = None
-        self._result_queue = queue.Queue()
+        self._result_queue = queue.Queue()  # type: queue.Queue
 
         if user_callback is not None:
             self.stream_results(user_callback)
@@ -226,7 +226,7 @@ def _start_websocket_client(
         """
         try:
             # Need new loop for the thread.
-            self._streaming_loop = asyncio.new_event_loop()
+            self._streaming_loop = asyncio.new_event_loop()  # type: ignore[assignment]
             asyncio.set_event_loop(self._streaming_loop)
             # TODO - use asyncio.create_task() when 3.6 is dropped.
             self._streaming_task = self._streaming_loop.create_task(
@@ -242,7 +242,8 @@ def _start_websocket_client(
                 "from the server for job %s:\n%s", self.job_id(), traceback.format_exc())
         finally:
             if self._streaming_loop is not None:
-                self._streaming_loop.run_until_complete(self._ws_client.disconnect())
+                self._streaming_loop.run_until_complete(  # type: ignore[unreachable]
+                    self._ws_client.disconnect())
             self._streaming = False
 
     def _stream_results(self, result_queue: queue.Queue, user_callback: Callable) -> None:
diff --git a/qiskit/providers/ibmq/runtime/runtime_program.py b/qiskit/providers/ibmq/runtime/runtime_program.py
index 7df0d0860..b6ab11c78 100644
--- a/qiskit/providers/ibmq/runtime/runtime_program.py
+++ b/qiskit/providers/ibmq/runtime/runtime_program.py
@@ -87,7 +87,7 @@ def __init__(
                                                            type=intret['type']))
 
     def __str__(self) -> str:
-        def _format_common(items: List):
+        def _format_common(items: List) -> None:
             """Add name, description, and type to `formatted`."""
             for item in items:
                 formatted.append(" "*4 + "- " + item.name + ":")
diff --git a/qiskit/providers/ibmq/runtime/utils.py b/qiskit/providers/ibmq/runtime/utils.py
index cf9f388cd..5e1cecc5d 100644
--- a/qiskit/providers/ibmq/runtime/utils.py
+++ b/qiskit/providers/ibmq/runtime/utils.py
@@ -45,10 +45,10 @@ def default(self, obj: Any) -> Any:  # pylint: disable=arguments-differ
 class RuntimeDecoder(json.JSONDecoder):
     """JSON Decoder used by runtime service."""
 
-    def __init__(self, *args, **kwargs):
+    def __init__(self, *args: Any, **kwargs: Any):
         super().__init__(object_hook=self.object_hook, *args, **kwargs)
 
-    def object_hook(self, obj):
+    def object_hook(self, obj: Any) -> Any:
         """Called to decode object."""
         if '__type__' in obj:
             if obj['__type__'] == 'complex':
diff --git a/test/ibmq/runtime/fake_runtime_client.py b/test/ibmq/runtime/fake_runtime_client.py
index 67bc880cd..4459c675a 100644
--- a/test/ibmq/runtime/fake_runtime_client.py
+++ b/test/ibmq/runtime/fake_runtime_client.py
@@ -81,7 +81,8 @@ def to_dict(self):
                 'project': self._project,
                 'backend': self._backend_name,
                 'status': self._status,
-                'params': [self._params]}
+                'params': [self._params],
+                'program': {'id': self._program_id}}
 
     def result(self):
         """Return job result."""
@@ -89,7 +90,7 @@ def result(self):
 
 
 class FailedRuntimeJob(BaseFakeRuntimeJob):
-    """Base class for faking a runtime job."""
+    """Class for faking a failed runtime job."""
 
     _job_progress = [
         "QUEUED",
@@ -105,6 +106,20 @@ def _auto_progress(self):
             self._result = "Kaboom!"
 
 
+class CancelableRuntimeJob(BaseFakeRuntimeJob):
+    """Class for faking a cancelable runtime job."""
+
+    _job_progress = [
+        "QUEUED",
+        "RUNNING"
+    ]
+
+    def cancel(self):
+        """Cancel the job."""
+        self._future.cancel()
+        self._status = "CANCELLED"
+
+
 class BaseFakeRuntimeClient:
     """Base class for faking the runtime client."""
 
@@ -167,7 +182,7 @@ def program_run(
     def program_delete(self, program_id: str) -> None:
         """Delete the specified program."""
         if program_id not in self._programs:
-            raise RequestsApiError("Program not found")
+            raise RequestsApiError("Program not found", status_code=404)
         del self._programs[program_id]
 
     def job_get(self, job_id):
@@ -182,4 +197,4 @@ def job_results(self, job_id):
 
     def job_cancel(self, job_id):
         """Cancel the job."""
-        pass
+        self._jobs[job_id].cancel()
diff --git a/test/ibmq/runtime/test_runtime.py b/test/ibmq/runtime/test_runtime.py
index a295b1979..d73607a40 100644
--- a/test/ibmq/runtime/test_runtime.py
+++ b/test/ibmq/runtime/test_runtime.py
@@ -17,6 +17,8 @@
 from unittest.mock import patch
 from unittest import mock
 import uuid
+import time
+import random
 
 import numpy as np
 from qiskit.result import Result
@@ -24,11 +26,12 @@
 from qiskit.providers.ibmq.runtime.utils import RuntimeEncoder, RuntimeDecoder
 from qiskit.providers.ibmq.accountprovider import AccountProvider
 from qiskit.providers.ibmq.runtime import IBMRuntimeService, RuntimeJob
-from qiskit.providers.ibmq.runtime.exceptions import RuntimeProgramNotFound
+from qiskit.providers.ibmq.runtime.exceptions import (RuntimeProgramNotFound,
+                                                      RuntimeJobFailureError)
 
 
 from ...ibmqtestcase import IBMQTestCase
-from .fake_runtime_client import BaseFakeRuntimeClient
+from .fake_runtime_client import BaseFakeRuntimeClient, FailedRuntimeJob, CancelableRuntimeJob
 from .utils import SerializableClass, UnserializableClass
 
 
@@ -125,9 +128,8 @@ def test_double_delete_program(self):
 
     def test_run_program(self):
         """Test running program."""
-        program_id = self._upload_program()
         params = {'param1': 'foo'}
-        job = self._run_program(program_id, inputs=params)
+        job = self._run_program(inputs=params)
         self.assertTrue(job.job_id())
         self.assertIsInstance(job, RuntimeJob)
         self.assertIsInstance(job.status(), JobStatus)
@@ -136,18 +138,62 @@ def test_run_program(self):
         self.assertEqual(job.status(), JobStatus.DONE)
         self.assertTrue(job.result())
 
-    # def test_run_program_failed(self):
-    #     """Test a failed program execution."""
-    #     options = {'backend_name': self.backend.name()}
-    #     job = self.provider.runtime.run(program_id=self.program_id, inputs={}, options=options)
-    #     self.log.info("Runtime job %s submitted.", job.job_id())
-    #
-    #     job.wait_for_final_state()
-    #     self.assertEqual(JobStatus.ERROR, job.status())
-    #     with self.assertRaises(RuntimeJobFailureError) as err_cm:
-    #         job.result()
-    #     self.assertIn('KeyError', str(err_cm.exception))
-    #
+    def test_run_program_failed(self):
+        """Test a failed program execution."""
+        job = self._run_program(job_classes=FailedRuntimeJob)
+        job.wait_for_final_state()
+        self.assertEqual(JobStatus.ERROR, job.status())
+        with self.assertRaises(RuntimeJobFailureError):
+            job.result()
+
+    def test_retrieve_job(self):
+        """Test retrieving a job."""
+        program_id = self._upload_program()
+        params = {'param1': 'foo'}
+        job = self._run_program(program_id, inputs=params)
+        rjob = self.runtime.job(job.job_id())
+        self.assertEqual(job.job_id(), rjob.job_id())
+        self.assertEqual(program_id, rjob.program_id)
+
+    def test_cancel_job(self):
+        """Test canceling a job."""
+        job = self._run_program(job_classes=CancelableRuntimeJob)
+        time.sleep(1)
+        job.cancel()
+        self.assertEqual(job.status(), JobStatus.CANCELLED)
+        rjob = self.runtime.job(job.job_id())
+        self.assertEqual(rjob.status(), JobStatus.CANCELLED)
+
+    def test_final_result(self):
+        """Test getting final result."""
+        job = self._run_program()
+        result = job.result()
+        self.assertTrue(result)
+
+    def test_job_status(self):
+        """Test job status."""
+        job = self._run_program()
+        time.sleep(random.randint(1, 5))
+        self.assertTrue(job.status())
+
+    def test_job_inputs(self):
+        """Test job inputs."""
+        inputs = {"param1": "foo", "param2": "bar"}
+        job = self._run_program(inputs=inputs)
+        self.assertEqual(inputs, job.inputs)
+
+    def test_job_program_id(self):
+        """Test job program ID."""
+        program_id = self._upload_program()
+        job = self._run_program(program_id=program_id)
+        self.assertEqual(program_id, job.program_id)
+
+    def test_wait_for_final_state(self):
+        """Test wait for final state."""
+        job = self._run_program()
+        job.wait_for_final_state()
+        self.assertEqual(JobStatus.DONE, job.status())
+
     # def test_interim_results(self):
     #     """Test interim results."""
     #     def _callback(interim_result):
@@ -167,9 +213,13 @@ def _upload_program(self, name=None, max_execution_time=300):
             max_execution_time=max_execution_time)
         return program_id
 
-    def _run_program(self, program_id, inputs=None):
+    def _run_program(self, program_id=None, inputs=None, job_classes=None):
         """Run a program."""
         options = {'backend_name': "some_backend"}
+        if job_classes:
+            self.runtime._api_client.set_job_classes(job_classes)
+        if program_id is None:
+            program_id = self._upload_program()
         job = self.runtime.run(program_id=program_id, inputs=inputs,
                                options=options)
         return job
diff --git a/test/ibmq/runtime/test_runtime_integration.py b/test/ibmq/runtime/test_runtime_integration.py
index 5e205bd23..6961ff287 100644
--- a/test/ibmq/runtime/test_runtime_integration.py
+++ b/test/ibmq/runtime/test_runtime_integration.py
@@ -203,7 +203,7 @@ def test_retrieve_job_queued(self):
             time.sleep(1)
         rjob = self.provider.runtime.job(job.job_id())
         self.assertEqual(job.job_id(), rjob.job_id())
-        self.assertEqual(self.program_id, job.program_id)
+        self.assertEqual(self.program_id, rjob.program_id)
 
     def test_retrieve_job_running(self):
         """Test retrieving a running job."""
@@ -376,7 +376,7 @@ def test_job_status(self):
         """Test job status."""
         job = self._run_program(iterations=1)
         time.sleep(random.randint(1, 5))
-        job.status()
+        self.assertTrue(job.status())
 
     def test_job_inputs(self):
         """Test job inputs."""

From 487f6ce45b63b5f16f69a1951cdc02ca50cab840 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Tue, 27 Apr 2021 12:55:37 -0400
Subject: [PATCH 38/59] fix serializable class

---
 qiskit/providers/ibmq/accountprovider.py      | 2 +-
 test/ibmq/runtime/test_runtime.py             | 2 +-
 test/ibmq/runtime/test_runtime_integration.py | 2 +-
 test/ibmq/runtime/utils.py                    | 8 +++++---
 4 files changed, 8 insertions(+), 6 deletions(-)

diff --git a/qiskit/providers/ibmq/accountprovider.py b/qiskit/providers/ibmq/accountprovider.py
index 92af788bc..a724fbe35 100644
--- a/qiskit/providers/ibmq/accountprovider.py
+++ b/qiskit/providers/ibmq/accountprovider.py
@@ -25,7 +25,7 @@
 from qiskit.providers.backend import BackendV1 as Backend
 from qiskit.providers.basebackend import BaseBackend
 from qiskit.transpiler import Layout
-import qiskit.providers.ibmq.runtime.runtime_job as runtime_job  # pylint: disable=unused-import
+from qiskit.providers.ibmq.runtime import runtime_job  # pylint: disable=unused-import
 
 from .api.clients import AccountClient
 from .ibmqbackend import IBMQBackend, IBMQSimulator
diff --git a/test/ibmq/runtime/test_runtime.py b/test/ibmq/runtime/test_runtime.py
index d73607a40..c0a29a4f5 100644
--- a/test/ibmq/runtime/test_runtime.py
+++ b/test/ibmq/runtime/test_runtime.py
@@ -63,7 +63,7 @@ def test_coder(self):
                 }
         encoded = json.dumps(data, cls=RuntimeEncoder)
         decoded = json.loads(encoded, cls=RuntimeDecoder)
-        decoded["sclass"] = SerializableClass.from_json(**decoded['sclass'])
+        decoded["sclass"] = SerializableClass.from_json(decoded['sclass'])
 
         decoded_result = decoded.pop('result')
         data.pop('result')
diff --git a/test/ibmq/runtime/test_runtime_integration.py b/test/ibmq/runtime/test_runtime_integration.py
index 6961ff287..a7a93cfc5 100644
--- a/test/ibmq/runtime/test_runtime_integration.py
+++ b/test/ibmq/runtime/test_runtime_integration.py
@@ -441,7 +441,7 @@ def _get_complex_types(self):
     def _assert_complex_types_equal(self, expected, received):
         """Verify the received data in complex types is expected."""
         if 'class' in received:
-            received['class'] = SerializableClass.from_json(**received['class'])
+            received['class'] = SerializableClass.from_json(received['class'])
         self.assertEqual(expected, received)
 
     def _run_program(self, program_id=None, iterations=1,
diff --git a/test/ibmq/runtime/utils.py b/test/ibmq/runtime/utils.py
index 81ee35793..9dc368f07 100644
--- a/test/ibmq/runtime/utils.py
+++ b/test/ibmq/runtime/utils.py
@@ -12,6 +12,8 @@
 
 """Utility functions for runtime testing."""
 
+import json
+
 
 class SerializableClass:
     """Custom class with serialization methods."""
@@ -21,12 +23,12 @@ def __init__(self, value):
 
     def to_json(self):
         """To JSON serializable."""
-        return {"value": self.value}
+        return json.dumps({"value": self.value})
 
     @classmethod
-    def from_json(cls, value):
+    def from_json(cls, json_str):
         """From JSON serializable."""
-        return cls(value=value)
+        return cls(**json.loads(json_str))
 
     def __eq__(self, other):
         return self.value == other.value

From 22d63e66d7c4cdda6a93e3ab366cd372e513bee1 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Tue, 27 Apr 2021 21:51:58 -0400
Subject: [PATCH 39/59] add release notes

---
 qiskit/providers/ibmq/runtime/__init__.py     | 25 ++++++-------
 .../ibmq/runtime/ibm_runtime_service.py       | 22 +++++++-----
 .../ibmq/runtime/program/program_backend.py   |  1 +
 .../ibmq/runtime/program/user_messenger.py    |  1 +
 qiskit/providers/ibmq/runtime/runtime_job.py  |  4 +--
 .../providers/ibmq/runtime/runtime_program.py |  4 +--
 .../notes/runtime-f9a57a8286fa6197.yaml       | 36 +++++++++++++++++++
 7 files changed, 67 insertions(+), 26 deletions(-)
 create mode 100644 releasenotes/notes/runtime-f9a57a8286fa6197.yaml

diff --git a/qiskit/providers/ibmq/runtime/__init__.py b/qiskit/providers/ibmq/runtime/__init__.py
index ddfa20229..6c76a888c 100644
--- a/qiskit/providers/ibmq/runtime/__init__.py
+++ b/qiskit/providers/ibmq/runtime/__init__.py
@@ -17,7 +17,7 @@
 
 .. currentmodule:: qiskit.providers.ibmq.runtime
 
-Modules related to IBM Quantum Runtime Service.
+Modules related to Qiskit Runtime Service.
 
 .. caution::
 
@@ -28,17 +28,14 @@
 
   The runtime service is not available to all accounts.
 
-The IBM Quantum Runtime Service allows authorized users to upload their quantum programs
-that can be invoked by others. A quantum program is a piece of code that takes
-certain inputs, performs quantum and classical processing, and returns the
-results. For example, user A can upload a VQE quantum program that takes a Hamiltonian
-and an optimizer as inputs and returns the minimum eigensolver result. User B
-can then invoke this program, passing in the inputs and obtaining the results,
-with minimal code.
+The Qiskit Runtime Service allows authorized users to upload their quantum programs.
+A quantum program is a piece of code that takes certain inputs, performs
+quantum and classical processing, and returns the results. Other
+authorized users can invoke these quantum programs by simply passing in parameters.
 
 These quantum programs, sometimes called runtime programs, run in a special
-runtime environment that is separate from normal circuit job execution and has
-special performance advantages.
+runtime environment that significantly reduces waiting time during computational
+iterations.
 
 Listing runtime programs
 ------------------------
@@ -59,7 +56,7 @@
     print(program)
 
 In the example above, ``provider.runtime`` points to the runtime service class
-:class:`~qiskit.providers.ibmq.runtime.IBMRuntimeService`, which is the main entry
+:class:`IBMRuntimeService`, which is the main entry
 point for using this service. The example prints the program definitions of all
 available runtime programs and of just the ``circuit-runner`` program. A program
 definition consists of a program's ID, name, description, input parameters,
@@ -75,7 +72,7 @@
     from qiskit import IBMQ, QuantumCircuit
 
     provider = IBMQ.load_account()
-    backend = provider.backend.ibmq_qasm_simulator
+    backend = provider.backend.ibmq_montreal
 
     # Create a circuit.
     qc = QuantumCircuit(2, 2)
@@ -94,7 +91,7 @@
     result = job.result()
 
 The example above invokes the ``circuit-runner`` program,
-which compile, executes, and optionally applies measurement error mitigation to
+which compiles, executes, and optionally applies measurement error mitigation to
 the circuit result.
 
 Runtime Job
@@ -111,7 +108,7 @@
 ---------------
 
 Some runtime programs provide interim results that inform you about program
-progress. You can choose to stream the interim results when you invoke the
+progress. You can choose to stream the interim results when you run the
 program by passing in the ``callback`` parameter, or at a later time using
 the :meth:`RuntimeJob.stream_results` method. For example::
 
diff --git a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
index 58be173f0..e40eedece 100644
--- a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
+++ b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
@@ -10,7 +10,7 @@
 # copyright notice, and modified files need to carry a notice indicating
 # that they have been altered from the originals.
 
-"""IBM Quantum runtime service."""
+"""Qiskit runtime service."""
 
 import logging
 from typing import Dict, Callable, Optional, Union, List
@@ -31,12 +31,14 @@
 
 
 class IBMRuntimeService:
-    """Class for interacting with the IBM Quantum runtime service.
+    """Class for interacting with the Qiskit runtime service.
 
-    The IBM Quantum Runtime Service allows authorized users to upload their quantum programs
-    that can be invoked by others. A quantum program is a piece of code that takes
+    The Qiskit Runtime Service allows authorized users to upload their quantum programs
+    that can be invoked by other users. A quantum program is a piece of code that takes
     certain inputs, performs quantum and classical processing, and returns the
-    results.
+    results. Quantum programs, also known as runtime programs, run in a special
+    runtime environment that significantly reduces waiting time during computational
+    iterations.
 
     A sample workflow of using the runtime service::
 
@@ -70,9 +72,9 @@ class IBMRuntimeService:
     the results at a later time, but before the job finishes.
 
     The :meth:`run` method returns a
-    :class:`qiskit.providers.ibmq.runtime.RuntimeJob` object. You can use its
-    methods to perform tasks like checking the job status, getting job result, and
-    canceling the job.
+    :class:`~qiskit.providers.ibmq.runtime.RuntimeJob` object. You can use its
+    methods to perform tasks like checking job status, getting job result, and
+    canceling job.
     """
 
     def __init__(self, provider: 'accountprovider.AccountProvider') -> None:
@@ -175,6 +177,10 @@ def run(
                 ``backend_name``, which is required.
             inputs: Program input parameters.
             callback: Callback function to be invoked for any interim results.
+                The callback function will receive 2 positional parameters:
+
+                    1. Job ID
+                    2. Job interim result.
 
         Returns:
             A ``RuntimeJob`` instance representing the execution.
diff --git a/qiskit/providers/ibmq/runtime/program/program_backend.py b/qiskit/providers/ibmq/runtime/program/program_backend.py
index 6d54e52e1..0333c55fa 100644
--- a/qiskit/providers/ibmq/runtime/program/program_backend.py
+++ b/qiskit/providers/ibmq/runtime/program/program_backend.py
@@ -30,6 +30,7 @@ class ProgramBackend(Backend, ABC):
 
     This is a :class:`~qiskit.providers.Backend` class for runtime programs to
     use in place of :class:`~qiskit.providers.ibmq.IBMQBackend`.
+    This class can be used when writing a new runtime program.
     """
 
     @abstractmethod
diff --git a/qiskit/providers/ibmq/runtime/program/user_messenger.py b/qiskit/providers/ibmq/runtime/program/user_messenger.py
index 2a4df7571..e14802a40 100644
--- a/qiskit/providers/ibmq/runtime/program/user_messenger.py
+++ b/qiskit/providers/ibmq/runtime/program/user_messenger.py
@@ -22,6 +22,7 @@ class UserMessenger:
     """Base class for handling communication with program consumers.
 
     A program consumer is the user that executes the runtime program.
+    This class can be used when writing a new runtime program.
     """
 
     def publish(
diff --git a/qiskit/providers/ibmq/runtime/runtime_job.py b/qiskit/providers/ibmq/runtime/runtime_job.py
index c6b5bba9e..228fc937c 100644
--- a/qiskit/providers/ibmq/runtime/runtime_job.py
+++ b/qiskit/providers/ibmq/runtime/runtime_job.py
@@ -57,7 +57,7 @@ class RuntimeJob:
         try:
             job_result = job.result()  # It will block until the job finishes.
             print("The job finished with result {}".format(job_result))
-        except IBMQJobFailureError as ex:
+        except RuntimeJobFailureError as ex:
             print("Job failed!: {}".format(ex))
 
     If the program has any interim results, you can use the ``callback``
@@ -192,7 +192,7 @@ def stream_results(self, callback: Callable) -> None:
 
         Args:
             callback: Callback function to be invoked for any interim results.
-                The callback function will receive 2 position parameters:
+                The callback function will receive 2 positional parameters:
 
                     1. Job ID
                     2. Job interim result.
diff --git a/qiskit/providers/ibmq/runtime/runtime_program.py b/qiskit/providers/ibmq/runtime/runtime_program.py
index b6ab11c78..38659a696 100644
--- a/qiskit/providers/ibmq/runtime/runtime_program.py
+++ b/qiskit/providers/ibmq/runtime/runtime_program.py
@@ -10,7 +10,7 @@
 # copyright notice, and modified files need to carry a notice indicating
 # that they have been altered from the originals.
 
-"""IBM Quantum Experience runtime program."""
+"""Qiskit runtime program."""
 
 import logging
 from typing import Optional, List, NamedTuple
@@ -21,7 +21,7 @@
 class RuntimeProgram:
     """Class representing program metadata.
 
-    This class contains the metadata describing a program, including its
+    This class contains the metadata describing a program, such as its
     name, ID, description, etc.
 
     You can use the :class:`~qiskit.providers.ibmq.runtime.IBMRuntimeService`
diff --git a/releasenotes/notes/runtime-f9a57a8286fa6197.yaml b/releasenotes/notes/runtime-f9a57a8286fa6197.yaml
new file mode 100644
index 000000000..cf6320168
--- /dev/null
+++ b/releasenotes/notes/runtime-f9a57a8286fa6197.yaml
@@ -0,0 +1,36 @@
+---
+prelude: >
+    This release introduces a new feature ``Qiskit Runtime Service``.
+    This new service allows authorized users to upload quantum programs that
+    can be invoked by others. These quantum programs run in a special
+    environment that significantly reduces waiting time during computational
+    iterations.
+features:
+  - |
+    This release introduces a new feature ``Qiskit Runtime Service``. This
+    new service allows authorized users to upload quantum programs. Other
+    authorized users can invoke these quantum programs by simply passing
+    in parameters. These quantum programs run in a speical runtime
+    environment that significantly reduces waiting time during computational
+    iterations.
+
+    An example of using this new service::
+
+      from qiskit import IBMQ
+
+      provider = IBMQ.load_account()
+      # Print all avaiable programs.
+      provider.runtime.pprint_programs()
+
+      # Prepare the inputs. See program documentation on input parameters.
+      inputs = {...}
+      options = {"backend_name": provider.backend.ibmq_montreal.name()}
+
+      job = provider.runtime.run(program_id="circuit-runner",
+                                 options=options,
+                                 inputs=inputs)
+      # Check job status.
+      print(f"job status is {job.status()}")
+
+      # Get job result.
+      result = job.result()

From 43272f65264e4980501004ee2c05005505e03a14 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Tue, 27 Apr 2021 22:10:48 -0400
Subject: [PATCH 40/59] fix program name

---
 qiskit/providers/ibmq/accountprovider.py        |  2 +-
 qiskit/providers/ibmq/api/clients/runtime.py    |  2 +-
 qiskit/providers/ibmq/api/clients/runtime_ws.py |  1 +
 qiskit/providers/ibmq/runtime/__init__.py       |  2 +-
 qiskit/providers/ibmq/runtime/runtime_job.py    |  3 +--
 test/decorators.py                              |  2 ++
 test/ibmq/runtime/test_runtime.py               | 11 ++---------
 test/ibmq/runtime/test_runtime_integration.py   |  3 ---
 test/ibmq/runtime/utils.py                      |  2 +-
 9 files changed, 10 insertions(+), 18 deletions(-)

diff --git a/qiskit/providers/ibmq/accountprovider.py b/qiskit/providers/ibmq/accountprovider.py
index a724fbe35..6d15a9a33 100644
--- a/qiskit/providers/ibmq/accountprovider.py
+++ b/qiskit/providers/ibmq/accountprovider.py
@@ -306,7 +306,7 @@ def run_circuits(
         }
         inputs.update(run_config)
         options = {'backend_name': backend.name()}
-        return self.runtime.run('circuit-runner-jessie3', options=options, inputs=inputs)
+        return self.runtime.run('circuit-runner', options=options, inputs=inputs)
 
     def service(self, name: str) -> Any:
         """Return the specified service.
diff --git a/qiskit/providers/ibmq/api/clients/runtime.py b/qiskit/providers/ibmq/api/clients/runtime.py
index 23763fba8..6a98cedd5 100644
--- a/qiskit/providers/ibmq/api/clients/runtime.py
+++ b/qiskit/providers/ibmq/api/clients/runtime.py
@@ -1,6 +1,6 @@
 # This code is part of Qiskit.
 #
-# (C) Copyright IBM 2020.
+# (C) Copyright IBM 2021.
 #
 # This code is licensed under the Apache License, Version 2.0. You may
 # obtain a copy of this license in the LICENSE.txt file in the root directory
diff --git a/qiskit/providers/ibmq/api/clients/runtime_ws.py b/qiskit/providers/ibmq/api/clients/runtime_ws.py
index 7f7f018e9..2e8fbd0f6 100644
--- a/qiskit/providers/ibmq/api/clients/runtime_ws.py
+++ b/qiskit/providers/ibmq/api/clients/runtime_ws.py
@@ -34,6 +34,7 @@ class RuntimeWebsocketClient:
     BACKOFF_MAX = 8
     """Maximum time to wait between retries."""
     POISON_PILL = "_poison_pill"
+    """Used to inform consumer to stop."""
 
     def __init__(
             self,
diff --git a/qiskit/providers/ibmq/runtime/__init__.py b/qiskit/providers/ibmq/runtime/__init__.py
index 6c76a888c..ef4db0437 100644
--- a/qiskit/providers/ibmq/runtime/__init__.py
+++ b/qiskit/providers/ibmq/runtime/__init__.py
@@ -130,7 +130,7 @@ def interim_result_callback(job_id, interim_result):
 -------------------
 
 
-TODO: Add tutorial reference
+TODO: Add doc about uploading a program
 
 Classes
 ==========================
diff --git a/qiskit/providers/ibmq/runtime/runtime_job.py b/qiskit/providers/ibmq/runtime/runtime_job.py
index 228fc937c..029338229 100644
--- a/qiskit/providers/ibmq/runtime/runtime_job.py
+++ b/qiskit/providers/ibmq/runtime/runtime_job.py
@@ -10,7 +10,7 @@
 # copyright notice, and modified files need to carry a notice indicating
 # that they have been altered from the originals.
 
-"""IBM Quantum Experience runtime job."""
+"""Qiskit runtime job."""
 
 from typing import Any, Optional, Callable, Dict
 import time
@@ -234,7 +234,6 @@ def _start_websocket_client(
             self._streaming = True
 
             logger.debug("Start websocket client for job %s", self.job_id())
-            # loop.run_until_complete(self._ws_client.job_results(self._job_id, result_queue))
             self._streaming_loop.run_until_complete(self._streaming_task)
         except Exception:  # pylint: disable=broad-except
             logger.warning(
diff --git a/test/decorators.py b/test/decorators.py
index 68636c16f..db6d99337 100644
--- a/test/decorators.py
+++ b/test/decorators.py
@@ -218,6 +218,8 @@ def _wrapper(obj, *args, **kwargs):
 
         backend_name = os.getenv('QE_STAGING_RUNTIME_DEVICE', None) if \
             os.getenv('USE_STAGING_CREDENTIALS', '') else os.getenv('QE_RUNTIME_DEVICE', None)
+        if not backend_name:
+            raise SkipTest("Runtime device not specified")
         _backend = _get_backend(qe_token=kwargs.pop('qe_token'),
                                 qe_url=kwargs.pop('qe_url'),
                                 backend_name=backend_name)
diff --git a/test/ibmq/runtime/test_runtime.py b/test/ibmq/runtime/test_runtime.py
index c0a29a4f5..cdafde03f 100644
--- a/test/ibmq/runtime/test_runtime.py
+++ b/test/ibmq/runtime/test_runtime.py
@@ -194,15 +194,6 @@ def test_wait_for_final_state(self):
         job.wait_for_final_state()
         self.assertEqual(JobStatus.DONE, job.status())
 
-    # def test_interim_results(self):
-    #     """Test interim results."""
-    #     def _callback(interim_result):
-    #         print(f"interim result {interim_result}")
-    #     params = {'param1': 'foo'}
-    #     backend = self.provider.backend.ibmq_qasm_simulator
-    #     job = self.provider.runtime.run("QKA", backend=backend, params=params, callback=_callback)
-    #     job.result()
-
     def _upload_program(self, name=None, max_execution_time=300):
         """Upload a new program."""
         name = name or uuid.uuid4().hex
@@ -223,3 +214,5 @@ def _run_program(self, program_id=None, inputs=None, job_classes=None):
         job = self.runtime.run(program_id=program_id, inputs=inputs,
                                options=options)
         return job
+
+    # TODO add websocket tests
diff --git a/test/ibmq/runtime/test_runtime_integration.py b/test/ibmq/runtime/test_runtime_integration.py
index a7a93cfc5..acdbd337f 100644
--- a/test/ibmq/runtime/test_runtime_integration.py
+++ b/test/ibmq/runtime/test_runtime_integration.py
@@ -29,9 +29,6 @@
 from .utils import SerializableClass
 
 
-# os.environ['USE_STAGING_CREDENTIALS'] = 'true'
-
-
 @unittest.skipIf(not os.environ.get('USE_STAGING_CREDENTIALS', ''), "Only runs on staging")
 class TestRuntimeIntegration(IBMQTestCase):
     """Integration tests for runtime modules."""
diff --git a/test/ibmq/runtime/utils.py b/test/ibmq/runtime/utils.py
index 9dc368f07..93c1bf796 100644
--- a/test/ibmq/runtime/utils.py
+++ b/test/ibmq/runtime/utils.py
@@ -1,6 +1,6 @@
 # This code is part of Qiskit.
 #
-# (C) Copyright IBM 2020.
+# (C) Copyright IBM 2021.
 #
 # This code is licensed under the Apache License, Version 2.0. You may
 # obtain a copy of this license in the LICENSE.txt file in the root directory

From 935391817b5681755da7cca12034018dad361340 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Wed, 28 Apr 2021 08:29:58 -0400
Subject: [PATCH 41/59] fix tests

---
 test/ibmq/test_account_client.py                  | 3 +--
 test/ibmq/test_random.py                          | 2 +-
 test/ibmq/websocket/test_websocket_integration.py | 6 +++---
 3 files changed, 5 insertions(+), 6 deletions(-)

diff --git a/test/ibmq/test_account_client.py b/test/ibmq/test_account_client.py
index 230b8a80a..e18e50c27 100644
--- a/test/ibmq/test_account_client.py
+++ b/test/ibmq/test_account_client.py
@@ -69,8 +69,7 @@ def tearDown(self) -> None:
 
     def _get_client(self):
         """Helper for instantiating an AccountClient."""
-        return AccountClient(self.access_token,
-                             self.provider.credentials)
+        return AccountClient(self.provider.credentials)
 
     def test_exception_message(self):
         """Check exception has proper message."""
diff --git a/test/ibmq/test_random.py b/test/ibmq/test_random.py
index 18641cea0..a74b6a5df 100644
--- a/test/ibmq/test_random.py
+++ b/test/ibmq/test_random.py
@@ -103,7 +103,7 @@ def setUpClass(cls, provider):
         # pylint: disable=arguments-differ
         super().setUpClass()
         cls.provider = provider
-        random_service = IBMQRandomService(provider, None)
+        random_service = IBMQRandomService(provider)
         random_service._random_client = FakeRandomClient()
         random_service._initialized = False
         cls.provider._random = random_service
diff --git a/test/ibmq/websocket/test_websocket_integration.py b/test/ibmq/websocket/test_websocket_integration.py
index 1dd763918..54a7a50f6 100644
--- a/test/ibmq/websocket/test_websocket_integration.py
+++ b/test/ibmq/websocket/test_websocket_integration.py
@@ -101,16 +101,16 @@ def test_websockets_retry_bad_url(self):
         """Test http retry after websocket error due to an invalid URL."""
         job = self.sim_backend.run(self.bell)
 
-        saved_websocket_url = job._api_client.client_ws._websocket_url
+        saved_websocket_url = job._api_client.client_ws.websocket_url
         try:
             # Use fake websocket address.
-            job._api_client.client_ws._websocket_url = 'wss://wss.localhost'
+            job._api_client.client_ws.websocket_url = 'wss://wss.localhost'
 
             # _wait_for_completion() should retry with http successfully
             # after getting websockets error.
             job._wait_for_completion()
         finally:
-            job._api_client.client_ws._websocket_url = saved_websocket_url
+            job._api_client.client_ws.websocket_url = saved_websocket_url
 
         self.assertIs(job._status, JobStatus.DONE)
 

From 0d350e402495e96f670ce77aa855d057edfc1049 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Thu, 29 Apr 2021 12:34:18 -0400
Subject: [PATCH 42/59] add metadata

---
 qiskit/providers/ibmq/accountprovider.py      |  20 ++
 qiskit/providers/ibmq/api/clients/runtime.py  |  22 +-
 .../providers/ibmq/api/clients/runtime_ws.py  |   5 -
 qiskit/providers/ibmq/api/rest/runtime.py     |  27 +-
 qiskit/providers/ibmq/runtime/__init__.py     |  73 +++++-
 qiskit/providers/ibmq/runtime/exceptions.py   |   5 +
 .../ibmq/runtime/ibm_runtime_service.py       | 235 +++++++++++++++++-
 .../ibmq/runtime/program/__init__.py          |   2 +-
 .../ibmq/runtime/program/program_backend.py   |   3 +-
 .../program/program_metadata_sample.json      |  18 ++
 .../ibmq/runtime/program/program_template.py  |   3 +
 qiskit/providers/ibmq/runtime/runtime_job.py  |  30 ++-
 .../providers/ibmq/runtime/runtime_program.py |  59 ++++-
 test/ibmq/runtime/test_runtime_integration.py |  69 ++++-
 14 files changed, 514 insertions(+), 57 deletions(-)
 create mode 100644 qiskit/providers/ibmq/runtime/program/program_metadata_sample.json

diff --git a/qiskit/providers/ibmq/accountprovider.py b/qiskit/providers/ibmq/accountprovider.py
index 6d15a9a33..ba55e12f7 100644
--- a/qiskit/providers/ibmq/accountprovider.py
+++ b/qiskit/providers/ibmq/accountprovider.py
@@ -338,6 +338,26 @@ def services(self) -> Dict:
         """
         return {key: val for key, val in self._services.items() if val is not None}
 
+    def has_service(self, name: str) -> bool:
+        """Check if this provider has access to the service.
+
+        Args:
+            name: Name of the service.
+
+        Returns:
+            Whether the provider has access to the service.
+
+        Raises:
+            IBMQInputValueError: If an unknown service name is specified.
+        """
+        if name not in self._services:
+            raise IBMQInputValueError(f"Unknown service {name} specified.")
+
+        if self._services[name] is None:
+            return False
+
+        return True
+
     @property
     def backend(self) -> IBMQBackendService:
         """Return the backend service.
diff --git a/qiskit/providers/ibmq/api/clients/runtime.py b/qiskit/providers/ibmq/api/clients/runtime.py
index 6a98cedd5..f29535c47 100644
--- a/qiskit/providers/ibmq/api/clients/runtime.py
+++ b/qiskit/providers/ibmq/api/clients/runtime.py
@@ -131,6 +131,14 @@ def job_get(self, job_id: str) -> Dict:
         logger.debug("Runtime job get response: %s", response)
         return response
 
+    def jobs_get(self) -> List:
+        """Get job data for all jobs.
+
+        Returns:
+            A list of job data.
+        """
+        return self.api.jobs_get()
+
     def job_results(self, job_id: str) -> str:
         """Get the results of a program job.
 
@@ -146,6 +154,18 @@ def job_cancel(self, job_id: str) -> None:
         """Cancel a job.
 
         Args:
-            job_id: Program job ID.
+            job_id: Runtime job ID.
         """
         self.api.program_job(job_id).cancel()
+
+    def job_delete(self, job_id: str) -> None:
+        """Delete a job.
+
+        Args:
+            job_id: Runtime job ID.
+        """
+        self.api.program_job(job_id).delete()
+
+    def logout(self) -> None:
+        """Clear authorization cache."""
+        self.api.logout()
diff --git a/qiskit/providers/ibmq/api/clients/runtime_ws.py b/qiskit/providers/ibmq/api/clients/runtime_ws.py
index 2e8fbd0f6..8ac75f3da 100644
--- a/qiskit/providers/ibmq/api/clients/runtime_ws.py
+++ b/qiskit/providers/ibmq/api/clients/runtime_ws.py
@@ -33,8 +33,6 @@ class RuntimeWebsocketClient:
 
     BACKOFF_MAX = 8
     """Maximum time to wait between retries."""
-    POISON_PILL = "_poison_pill"
-    """Used to inform consumer to stop."""
 
     def __init__(
             self,
@@ -116,7 +114,6 @@ async def job_results(
                         result_queue.put_nowait(response)
                         current_retry = 0  # Reset counter after a good receive.
                     except ConnectionClosed as ex:
-                        self._ws = None
                         if ex.code == 1000:  # Job has finished.
                             return
                         exception_to_raise = WebsocketRetryableError(
@@ -126,14 +123,12 @@ async def job_results(
 
             except asyncio.CancelledError:
                 logger.debug("Streaming is cancelled.")
-                result_queue.put_nowait(self.POISON_PILL)
                 return
             except WebsocketRetryableError as ex:
                 logger.debug("A websocket error occurred while streaming "
                              "results for runtime job %s:\n%s", job_id, traceback.format_exc())
                 current_retry += 1
                 if current_retry > max_retries:
-                    result_queue.put_nowait(self.POISON_PILL)
                     raise ex
 
                 backoff_time = self._backoff_time(backoff_factor, current_retry)
diff --git a/qiskit/providers/ibmq/api/rest/runtime.py b/qiskit/providers/ibmq/api/rest/runtime.py
index 80a14ddcd..001594819 100644
--- a/qiskit/providers/ibmq/api/rest/runtime.py
+++ b/qiskit/providers/ibmq/api/rest/runtime.py
@@ -28,7 +28,8 @@ class Runtime(RestAdapterBase):
 
     URL_MAP = {
         'programs': '/programs',
-        'jobs': '/jobs'
+        'jobs': '/jobs',
+        'logout': '/logout'
     }
 
     def program(self, program_id: str) -> 'Program':
@@ -124,6 +125,20 @@ def program_run(
         data = json.dumps(payload)
         return self.session.post(url, data=data).json()
 
+    def jobs_get(self) -> List[Dict]:
+        """Get a list of job data.
+
+        Returns:
+            A list of job data.
+        """
+        url = self.get_url('jobs')
+        return self.session.get(url).json()
+
+    def logout(self) -> None:
+        """Clear authorization cache."""
+        url = self.get_url('logout')
+        self.session.post(url)
+
 
 class Program(RestAdapterBase):
     """Rest adapter for program related endpoints."""
@@ -207,13 +222,9 @@ def get(self) -> Dict:
         """
         return self.session.get(self.get_url('self')).json()
 
-    def delete(self) -> Dict:
-        """Delete program job.
-
-        Returns:
-            JSON response.
-        """
-        return self.session.delete(self.get_url('self')).json()
+    def delete(self) -> None:
+        """Delete program job."""
+        self.session.delete(self.get_url('self'))
 
     def results(self) -> str:
         """Return program job results.
diff --git a/qiskit/providers/ibmq/runtime/__init__.py b/qiskit/providers/ibmq/runtime/__init__.py
index ef4db0437..56f91f479 100644
--- a/qiskit/providers/ibmq/runtime/__init__.py
+++ b/qiskit/providers/ibmq/runtime/__init__.py
@@ -19,14 +19,23 @@
 
 Modules related to Qiskit Runtime Service.
 
-.. caution::
+.. note::
+
+    The runtime service is not available to all providers. To check if a provider
+    has access::
+
+        from qiskit import IBMQ
 
-  This package is currently provided in beta form and heavy modifications to
-  both functionality and API are likely to occur.
+        IBMQ.load_account()
+        provider = IBMQ.get_provider(...)
+
+        can_use_runtime = provider.has_service('runtime')
 
 .. note::
 
-  The runtime service is not available to all accounts.
+    Not all backends support Runtime. Refer to documentation on
+    `Qiskit-Partners/qiskit-runtime
+    <https://github.com/Qiskit-Partners/qiskit-runtime>`_ for more information.
 
 The Qiskit Runtime Service allows authorized users to upload their quantum programs.
 A quantum program is a piece of code that takes certain inputs, performs
@@ -37,6 +46,10 @@
 runtime environment that significantly reduces waiting time during computational
 iterations.
 
+`Qiskit-Partners/qiskit-runtime <https://github.com/Qiskit-Partners/qiskit-runtime>`_
+contains detailed tutorials on how to use the runtime service.
+
+
 Listing runtime programs
 ------------------------
 
@@ -115,7 +128,7 @@
     from qiskit import IBMQ, QuantumCircuit
 
     provider = IBMQ.load_account()
-    backend = provider.backend.ibmq_qasm_simulator
+    backend = provider.backend.ibmq_montreal
 
     def interim_result_callback(job_id, interim_result):
         print(interim_result)
@@ -129,8 +142,56 @@ def interim_result_callback(job_id, interim_result):
 Uploading a program
 -------------------
 
+.. note::
+
+  Only authorized accounts can upload programs. Having access to the
+  runtime service doesn't imply access to upload programs.
+
+Each runtime program has both ``data`` and ``metadata``. Program data is
+the Python code to be executed. Program metadata provides usage information,
+such as program description, its inputs and outputs, and backend requirements.
+A detailed program metadata helps the consumers of the program to know what is
+needed to run the program.
+
+Each program data needs to have a ``main(backend, user_messenger, **kwargs)``
+method, which serves as the entry point to the program. The ``backend`` parameter
+is a :class:`ProgramBackend` instance whose :meth:`ProgramBackend.run` method
+can be used to submit circuits. The ``user_messenger`` is a :class:`UserMessenger`
+whose :meth:`UserMessenger.publish` method can be used to publish interim and
+final results. See :file:`program/program_template.py` for a program data
+template file.
+
+Each program metadata must include at least the program name, description, and
+maximum execution time. You can find description of each metadata field in
+the :meth:`IBMRuntimeService.upload_program` method. Instead of passing in
+the metadata fields individually, you can pass in a JSON file or a dictionary
+to :meth:`IBMRuntimeService.upload_program` via the ``metadata`` parameter.
+:file:`program/program_metadata_sample.json` is a sample file of program metadata.
+
+You can use the :meth:`IBMRuntimeService.upload_program` to upload a program.
+For example::
+
+    from qiskit import IBMQ
+
+    provider = IBMQ.load_account()
+    program_id = provider.runtime.upload_program(
+                    data="my_vqe.py",
+                    metadata="my_vqe_metadata.json",
+                    version=1.2
+                )
+
+In the example above, the file ``my_vqe.py`` contains the program data, and
+``my_vqe_metadata.json`` contains the program metadata. An additional
+parameter ``version`` is also specified, which will be used instead of the
+``version`` value in ``my_vqe_metadata.json``.
+
+Methods :meth:`IBMRuntimeService.update_program` and
+:meth:`IBMRuntimeService.delete_program` allow you to update and delete a
+program, respectively.
+
+Files related to writing a runtime program are in the
+``qiskit/providers/ibmq/runtime/program`` directory.
 
-TODO: Add doc about uploading a program
 
 Classes
 ==========================
diff --git a/qiskit/providers/ibmq/runtime/exceptions.py b/qiskit/providers/ibmq/runtime/exceptions.py
index c8b85e32c..56513696d 100644
--- a/qiskit/providers/ibmq/runtime/exceptions.py
+++ b/qiskit/providers/ibmq/runtime/exceptions.py
@@ -36,6 +36,11 @@ class RuntimeJobFailureError(QiskitRuntimeError):
     pass
 
 
+class RuntimeJobNotFound(QiskitRuntimeError):
+    """Error raised when a job is not found."""
+    pass
+
+
 class RuntimeInvalidStateError(QiskitRuntimeError):
     """Errors raised when the state is not valid for the operation."""
     pass
diff --git a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
index e40eedece..12af145a6 100644
--- a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
+++ b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
@@ -13,15 +13,17 @@
 """Qiskit runtime service."""
 
 import logging
-from typing import Dict, Callable, Optional, Union, List
+from typing import Dict, Callable, Optional, Union, List, Any
 import json
+import copy
 
 from qiskit.providers.ibmq import accountprovider  # pylint: disable=unused-import
 
 from .runtime_job import RuntimeJob
-from .runtime_program import RuntimeProgram
+from .runtime_program import RuntimeProgram, ProgramParameter, ProgramResult
 from .utils import RuntimeEncoder, RuntimeDecoder
-from .exceptions import QiskitRuntimeError, RuntimeDuplicateProgramError, RuntimeProgramNotFound
+from .exceptions import (QiskitRuntimeError, RuntimeDuplicateProgramError, RuntimeProgramNotFound,
+                         RuntimeJobNotFound)
 from ..api.clients.runtime import RuntimeClient
 from ..api.clients.runtime_ws import RuntimeWebsocketClient
 from ..api.exceptions import RequestsApiError
@@ -175,7 +177,8 @@ def run(
             program_id: Program ID.
             options: Runtime options. Currently the only available option is
                 ``backend_name``, which is required.
-            inputs: Program input parameters.
+            inputs: Program input parameters. These input values are passed
+                to the runtime program.
             callback: Callback function to be invoked for any interim results.
                 The callback function will receive 2 positional parameters:
 
@@ -208,25 +211,75 @@ def run(
 
     def upload_program(
             self,
-            name: str,
             data: Union[bytes, str],
-            max_execution_time: int
+            metadata: Optional[Union[Dict, str]] = None,
+            name: Optional[str] = None,
+            max_execution_time: Optional[int] = None,
+            description: Optional[str] = None,
+            version: Optional[float] = None,
+            backend_requirements: Optional[str] = None,
+            parameter_schema: Optional[str] = None,
+            parameters: Optional[List[ProgramParameter]] = None,
+            return_values: Optional[List[ProgramResult]] = None,
+            interim_results: Optional[List[ProgramResult]] = None
     ) -> str:
         """Upload a runtime program.
 
+        In addition to program data, the following program metadata are also
+        required:
+
+            - name
+            - max_execution_time
+            - description
+
+        Program metadata can be specified using the `metadata` parameter or
+        individual parameter (for example, `name` and `description`). If the
+        same metadata field is specified in both places, the individual parameter
+        takes precedence. For example, if you specify:
+
+            upload_program(metadata={"name": "name1"}, name="name2")
+
+        ``name2`` will be used as the program name.
+
         Args:
-            name: Name of the program.
             data: Name of the program file or program data to upload.
-            max_execution_time: Maximum execution time in seconds.
+            metadata: Name of the program metadata file or metadata dictionary.
+                A metadata file needs to be in the JSON format.
+                See :file:`program/program_metadata_sample.yaml` for an example.
+            name: Name of the program. Required if not specified via `metadata`.
+            max_execution_time: Maximum execution time in seconds. Required if
+                not specified via `metadata`.
+            description: Program description. Required if not specified via `metadata`.
+            version: Program version. The default is 1.0 if not specified.
+            backend_requirements: Backend requirements.
+            parameter_schema: Schema that can be used to validate the program
+                input parameters.
+            parameters: A list of program input parameters.
+            return_values: A list of program return values.
+            interim_results: A list of program interim results.
 
         Returns:
             Program ID.
 
         Raises:
+            IBMQInputValueError: If required metadata is missing.
             RuntimeDuplicateProgramError: If a program with the same name already exists.
             IBMQNotAuthorizedError: If you are not authorized to upload programs.
             QiskitRuntimeError: If the upload failed.
         """
+        program_metadata = self._merge_metadata(
+            initial={},
+            metadata=metadata,
+            name=name, max_execution_time=max_execution_time, description=description,
+            version=version, backend_requirements=backend_requirements,
+            parameter_schema=parameter_schema, parameters=parameters,
+            return_values=return_values, interim_results=interim_results)
+
+        # TODO uncomment when metada is supported on server
+        # for req in ['name', 'description', 'max_execution_time']:
+        #     if req not in program_metadata or not program_metadata[req]:
+        #         raise IBMQInputValueError(f"{req} is a required metadata field.")
+
         try:
             response = self._api_client.program_create(name, data, max_execution_time)
         except RequestsApiError as ex:
@@ -239,6 +292,98 @@ def upload_program(
             raise QiskitRuntimeError(f"Failed to create program: {ex}") from None
         return response['id']
 
+    def update_program(
+            self,
+            program_id: str,
+            data: Union[bytes, str] = None,
+            metadata: Optional[Union[Dict, str]] = None,
+            name: Optional[str] = None,
+            max_execution_time: Optional[int] = None,
+            description: Optional[str] = None,
+            version: Optional[float] = None,
+            backend_requirements: Optional[str] = None,
+            parameter_schema: Optional[str] = None,
+            parameters: Optional[List[ProgramParameter]] = None,
+            return_values: Optional[List[ProgramResult]] = None,
+            interim_results: Optional[List[ProgramResult]] = None
+    ) -> None:
+        """Update an existing runtime program.
+
+        Program metadata can be specified using the `metadata` parameter or
+        individual parameter (for example, `name` and `description`). If the
+        same metadata field is specified in both places, the individual parameter
+        takes precedence. For example, if you specify:
+
+            update_program(metadata={"name": "name1"}, name="name2")
+
+        ``name2`` will be used as the program name.
+
+        Args:
+            program_id: Program ID.
+            data: Name of the program file or program data to upload.
+            metadata: Name of the program metadata file or metadata dictionary.
+                A metadata file needs to be in the JSON format.
+                See :file:`program/program_metadata_sample.yaml` for an example.
+            name: Name of the program. Required if not specified via `metadata`.
+            max_execution_time: Maximum execution time in seconds. Required if
+                not specified via `metadata`.
+            description: Program description. Required if not specified via `metadata`.
+            version: Program version. The default is 1.0 if not specified.
+            backend_requirements: Backend requirements.
+            parameter_schema: Schema that can be used to validate the program
+                input parameters.
+            parameters: A list of program input parameters.
+            return_values: A list of program return values.
+            interim_results: A list of program interim results.
+
+        Raises:
+            RuntimeProgramNotFound: If the program doesn't exist.
+            IBMQNotAuthorizedError: If you are not authorized to upload programs.
+            QiskitRuntimeError: If the update failed.
+        """
+        program_metadata = self.program(program_id, refresh=True).to_dict()
+        program_metadata = self._merge_metadata(
+            initial=program_metadata,
+            metadata=metadata,
+            name=name, max_execution_time=max_execution_time, description=description,
+            version=version, backend_requirements=backend_requirements,
+            parameter_schema=parameter_schema, parameters=parameters,
+            return_values=return_values, interim_results=interim_results)
+
+        self.delete_program(program_id)
+        self.upload_program(data=data, metadata=program_metadata)
+
+    def _merge_metadata(
+            self,
+            initial: Dict,
+            metadata: Optional[Union[Dict, str]] = None,
+            **kwargs: Any
+    ) -> Dict:
+        """Merge multiple copies of metadata.
+
+        Args:
+            initial: The initial metadata. This may be mutated.
+            metadata: Name of the program metadata file or metadata dictionary.
+            **kwargs: Additional metadata fields to overwrite.
+
+        Returns:
+            Merged metadata.
+        """
+        upd_metadata = {}
+        if metadata is not None:
+            if isinstance(metadata, str):
+                with open(metadata, 'r') as file:
+                    upd_metadata = json.load(file)
+            else:
+                upd_metadata = copy.deepcopy(metadata)
+
+        initial.update(upd_metadata)
+
+        for key, val in kwargs.items():
+            if val is not None:
+                initial[key] = val
+        return initial
+
     def delete_program(self, program_id: str) -> None:
         """Delete a runtime program.
 
@@ -267,10 +412,60 @@ def job(self, job_id: str) -> RuntimeJob:
 
         Returns:
             Runtime job retrieved.
+
+        Raises:
+            RuntimeJobNotFound: If the job doesn't exist.
+            QiskitRuntimeError: If the request failed.
         """
-        response = self._api_client.job_get(job_id)
-        backend = self._provider.get_backend(response['backend'])
-        params = response.get('params', {})
+        try:
+            response = self._api_client.job_get(job_id)
+        except RequestsApiError as ex:
+            if ex.status_code == 404:
+                raise RuntimeJobNotFound(f"Job not found: {ex.message}") from None
+            raise QiskitRuntimeError(f"Failed to delete job: {ex}") from None
+        return self._decode_job(response)
+
+    def jobs(self) -> List[RuntimeJob]:
+        """Retrieve all runtime jobs.
+
+        Returns:
+            A list of runtime jobs.
+        """
+        response = self._api_client.jobs_get()
+        jobs = [self._decode_job(job) for job in response]
+        return jobs
+
+    def delete_job(self, job_id: str) -> None:
+        """Delete a runtime job.
+
+        Note that this operation cannot be reversed.
+
+        Args:
+            job_id: ID of the job to delete.
+
+        Raises:
+            RuntimeJobNotFound: If the job doesn't exist.
+            QiskitRuntimeError: If the request failed.
+        """
+        try:
+            self._api_client.job_delete(job_id)
+        except RequestsApiError as ex:
+            if ex.status_code == 404:
+                raise RuntimeJobNotFound(f"Job not found: {ex.message}") from None
+            raise QiskitRuntimeError(f"Failed to delete job: {ex}") from None
+
+    def _decode_job(self, raw_data: Dict) -> RuntimeJob:
+        """Decode job data received from the server.
+
+        Args:
+            raw_data: Raw job data received from the server.
+
+        Returns:
+            Decoded job data.
+        """
+        backend = self._provider.get_backend(raw_data['backend'])
+
+        params = raw_data.get('params', {})
         if isinstance(params, list):
             if len(params) > 0:
                 params = params[0]
@@ -283,6 +478,20 @@ def job(self, job_id: str) -> RuntimeJob:
         return RuntimeJob(backend=backend,
                           api_client=self._api_client,
                           ws_client=RuntimeWebsocketClient(self._ws_url, self._access_token),
-                          job_id=response['id'],
-                          program_id=response.get('program', {}).get('id', ""),
+                          job_id=raw_data['id'],
+                          program_id=raw_data.get('program', {}).get('id', ""),
                           params=decoded)
+
+    def logout(self) -> None:
+        """Clears authorization cache on the server.
+
+        For better performance, the runtime server caches each user's
+        authorization information. This method is used to force the server
+        to clear its cache.
+
+        Note:
+            Invoking this method ONLY when your access level to the runtime
+            service has changed - for example, the first time your account is
+            given the authority to upload a program.
+        """
+        self._api_client.logout()
diff --git a/qiskit/providers/ibmq/runtime/program/__init__.py b/qiskit/providers/ibmq/runtime/program/__init__.py
index 7a6e4637b..02273e6fe 100644
--- a/qiskit/providers/ibmq/runtime/program/__init__.py
+++ b/qiskit/providers/ibmq/runtime/program/__init__.py
@@ -12,5 +12,5 @@
 
 """Runtime program package.
 
-This package contains code used to write runtime programs.
+This package contains files to help you write runtime programs.
 """
diff --git a/qiskit/providers/ibmq/runtime/program/program_backend.py b/qiskit/providers/ibmq/runtime/program/program_backend.py
index 0333c55fa..211f911f8 100644
--- a/qiskit/providers/ibmq/runtime/program/program_backend.py
+++ b/qiskit/providers/ibmq/runtime/program/program_backend.py
@@ -29,8 +29,7 @@ class ProgramBackend(Backend, ABC):
     """Base class for a program backend.
 
     This is a :class:`~qiskit.providers.Backend` class for runtime programs to
-    use in place of :class:`~qiskit.providers.ibmq.IBMQBackend`.
-    This class can be used when writing a new runtime program.
+    submit circuits.
     """
 
     @abstractmethod
diff --git a/qiskit/providers/ibmq/runtime/program/program_metadata_sample.json b/qiskit/providers/ibmq/runtime/program/program_metadata_sample.json
new file mode 100644
index 000000000..52aa35f0e
--- /dev/null
+++ b/qiskit/providers/ibmq/runtime/program/program_metadata_sample.json
@@ -0,0 +1,18 @@
+{ "name": "Quantum-Kernel-Alignment",
+  "description": "Quantum kernel alignment algorithm that learns, on a given dataset, a quantum kernel maximizing the SVM classification margin.",
+  "max_execution_time": 3600,
+  "version": 1.0,
+  "backend_requirements": {"min_num_qubits":  5},
+  "parameters": [
+    {"name": "feature_map", "description": "An instance of FeatureMapACME used to map classical data into a quantum state space.", "type": "FeatureMapACME", "required": true},
+    {"name": "data", "description": "NxD array of training data, where N is the number of samples and D is the feature dimension.", "type": "numpy.ndarray", "required": true},
+    {"name": "labels", "description": "Nx1 array of +/-1 labels of the N training samples.", "type": "numpy.ndarray", "required": true}
+  ],
+  "return_values": [
+    {"name": "aligned_kernel_parameters", "description": "The optimized kernel parameters found from quantum kernel alignment.", "type": "numpy.ndarray"},
+    {"name": "aligned_kernel_matrix", "description": "The aligned quantum kernel matrix evaluated with the optimized kernel parameters on the training data.", "type": "numpy.ndarray"}
+  ],
+  "interim_results": [
+    {"name": "cost", "description": "Estimate of updated SVM objective function F using average of F(alpha_+, lambda_+) and F(alpha_-, lambda_-).", "type": "float"}
+  ]
+}
\ No newline at end of file
diff --git a/qiskit/providers/ibmq/runtime/program/program_template.py b/qiskit/providers/ibmq/runtime/program/program_template.py
index f696f766d..8a1461358 100644
--- a/qiskit/providers/ibmq/runtime/program/program_template.py
+++ b/qiskit/providers/ibmq/runtime/program/program_template.py
@@ -37,6 +37,9 @@ def program(backend: ProgramBackend, user_messenger: UserMessenger, **kwargs):
 def main(backend: ProgramBackend, user_messenger: UserMessenger, **kwargs):
     """This is the main entry point of a runtime program.
 
+    The name of this method must not change. It also must have ``backend``
+    and ``user_messenger`` as the first two positional arguments.
+
     Args:
         backend: Backend for the circuits to run on.
         user_messenger: Used to communicate with the program consumer.
diff --git a/qiskit/providers/ibmq/runtime/runtime_job.py b/qiskit/providers/ibmq/runtime/runtime_job.py
index 029338229..ebb23c393 100644
--- a/qiskit/providers/ibmq/runtime/runtime_job.py
+++ b/qiskit/providers/ibmq/runtime/runtime_job.py
@@ -31,6 +31,7 @@
 from .exceptions import RuntimeJobFailureError, RuntimeInvalidStateError
 from ..api.clients import RuntimeClient, RuntimeWebsocketClient
 from ..exceptions import IBMQError
+from ..api.exceptions import RequestsApiError
 
 logger = logging.getLogger(__name__)
 
@@ -68,7 +69,10 @@ class RuntimeJob:
     the results at a later time, but before the job finishes.
     """
 
-    _executor = futures.ThreadPoolExecutor()
+    POISON_PILL = "_poison_pill"
+    """Used to inform streaming to stop."""
+
+    _executor = futures.ThreadPoolExecutor(thread_name_prefix="runtime_job")
 
     def __init__(
             self,
@@ -112,7 +116,7 @@ def __init__(
     def result(
             self,
             timeout: Optional[float] = None,
-            wait: float = 5
+            wait: float = 5,
     ) -> Any:
         """Return the results of the job.
 
@@ -140,8 +144,17 @@ def result(
         return self._results
 
     def cancel(self) -> None:
-        """Cancel the job."""
-        self._api_client.job_cancel(self.job_id())
+        """Cancel the job.
+
+        Raises:
+            RuntimeInvalidStateError: If the job is in a state that cannot be cancelled.
+        """
+        try:
+            self._api_client.job_cancel(self.job_id())
+        except RequestsApiError as ex:
+            if ex.status_code == 409:
+                raise RuntimeInvalidStateError(f"Job cannot be cancelled: {ex}") from None
+            raise
         self._cancel_result_streaming()
         self._status = JobStatus.CANCELLED
 
@@ -198,11 +211,15 @@ def stream_results(self, callback: Callable) -> None:
                     2. Job interim result.
 
         Raises:
-            RuntimeInvalidStateError: If a callback function is already streaming results.
+            RuntimeInvalidStateError: If a callback function is already streaming results or
+                if the job already finished.
         """
         if self._streaming:
             raise RuntimeInvalidStateError("A callback function is already streaming results.")
 
+        if self._status in JOB_FINAL_STATES:
+            raise RuntimeInvalidStateError("Job already finished.")
+
         self._executor.submit(self._start_websocket_client,
                               result_queue=self._result_queue)
         self._executor.submit(self._stream_results,
@@ -240,6 +257,7 @@ def _start_websocket_client(
                 "An error occurred while streaming results "
                 "from the server for job %s:\n%s", self.job_id(), traceback.format_exc())
         finally:
+            self._result_queue.put_nowait(self.POISON_PILL)
             if self._streaming_loop is not None:
                 self._streaming_loop.run_until_complete(  # type: ignore[unreachable]
                     self._ws_client.disconnect())
@@ -256,7 +274,7 @@ def _stream_results(self, result_queue: queue.Queue, user_callback: Callable) ->
         while True:
             try:
                 response = result_queue.get()
-                if response == self._ws_client.POISON_PILL:
+                if response == self.POISON_PILL:
                     self._empty_result_queue(result_queue)
                     return
                 user_callback(self.job_id(), self._decode_data(response))
diff --git a/qiskit/providers/ibmq/runtime/runtime_program.py b/qiskit/providers/ibmq/runtime/runtime_program.py
index 38659a696..eac4cd26e 100644
--- a/qiskit/providers/ibmq/runtime/runtime_program.py
+++ b/qiskit/providers/ibmq/runtime/runtime_program.py
@@ -13,7 +13,7 @@
 """Qiskit runtime program."""
 
 import logging
-from typing import Optional, List, NamedTuple
+from typing import Optional, List, NamedTuple, Dict
 
 logger = logging.getLogger(__name__)
 
@@ -47,7 +47,10 @@ def __init__(
             parameters: Optional[List] = None,
             return_values: Optional[List] = None,
             interim_results: Optional[List] = None,
-            max_execution_time: int = 0
+            max_execution_time: int = 0,
+            version: float = 1.0,
+            backend_requirements: Optional[Dict] = None,
+            parameter_schema: str = ""
     ) -> None:
         """RuntimeProgram constructor.
 
@@ -59,11 +62,17 @@ def __init__(
             return_values: Documentation on program return values.
             interim_results: Documentation on program interim results.
             max_execution_time: Maximum execution time.
+            version: Program version.
+            backend_requirements: Backend requirements.
+            parameter_schema: Schema used to validate parameters.
         """
         self._name = program_name
         self._id = program_id
         self._description = description
         self._max_execution_time = max_execution_time
+        self._version = version
+        self._backend_requirements = backend_requirements or {}
+        self._parameter_schema = parameter_schema
         self._parameters = []
         self._return_values = []
         self._interim_results = []
@@ -119,6 +128,25 @@ def _format_common(items: List) -> None:
             formatted.append(" "*4 + "none")
         return '\n'.join(formatted)
 
+    def to_dict(self) -> Dict:
+        """Convert program metadata to dictionary format.
+
+        Returns:
+            Program metadata in dictionary format.
+        """
+        return {
+            "program_id": self.program_id,
+            "name": self.name,
+            "description": self.description,
+            "max_execution_time": self.max_execution_time,
+            "version": self.version,
+            "backend_requirements": self.backend_requirements,
+            "parameter_schema": self._parameter_schema,
+            "parameters": self.parameters,
+            "return_values": self.return_values,
+            "interim_results": self.interim_results
+        }
+
     @property
     def program_id(self) -> str:
         """Return program ID.
@@ -182,6 +210,33 @@ def max_execution_time(self) -> int:
         """
         return self._max_execution_time
 
+    @property
+    def version(self) -> float:
+        """Return program version.
+
+        Returns:
+            Program version.
+        """
+        return self._version
+
+    @property
+    def backend_requirements(self) -> Dict:
+        """Return backend requirements.
+
+        Returns:
+            Backend requirements for this program.
+        """
+        return self._backend_requirements
+
+    @property
+    def parameter_schema(self) -> str:
+        """Return program parameter schema.
+
+        Returns:
+            Program parameter schema.
+        """
+        return self._parameter_schema
+
 
 class ProgramParameter(NamedTuple):
     """Program parameter."""
diff --git a/test/ibmq/runtime/test_runtime_integration.py b/test/ibmq/runtime/test_runtime_integration.py
index acdbd337f..333a41a8d 100644
--- a/test/ibmq/runtime/test_runtime_integration.py
+++ b/test/ibmq/runtime/test_runtime_integration.py
@@ -22,7 +22,9 @@
 from qiskit.providers.ibmq.exceptions import IBMQNotAuthorizedError
 from qiskit.providers.ibmq.runtime.exceptions import (RuntimeDuplicateProgramError,
                                                       RuntimeProgramNotFound,
-                                                      RuntimeJobFailureError)
+                                                      RuntimeJobFailureError,
+                                                      RuntimeInvalidStateError,
+                                                      RuntimeJobNotFound)
 
 from ...ibmqtestcase import IBMQTestCase
 from ...decorators import requires_runtime_device
@@ -56,6 +58,11 @@ def main(backend, user_messenger, **kwargs):
     user_messenger.publish(final_result, final=True)
     """
 
+    RUNTIME_PROGRAM_METADATA = {
+        "max_execution_time": 600,
+        "description": "Qiskit test program"
+    }
+
     PROGRAM_PREFIX = 'qiskit-test'
 
     @classmethod
@@ -71,7 +78,7 @@ def setUpClass(cls, backend):
             cls.program_id = cls.provider.runtime.upload_program(
                 name=cls.PROGRAM_PREFIX,
                 data=cls.RUNTIME_PROGRAM.encode(),
-                max_execution_time=600)
+                metadata=cls.RUNTIME_PROGRAM_METADATA)
         except RuntimeDuplicateProgramError:
             pass
         except IBMQNotAuthorizedError:
@@ -102,13 +109,13 @@ def tearDown(self) -> None:
             except Exception:  # pylint: disable=broad-except
                 pass
 
-        # Cancel jobs.
+        # Cancel and delete jobs.
         for job in self.to_cancel:
-            if job.status() not in JOB_FINAL_STATES:
-                try:
-                    job.cancel()
-                except Exception:  # pylint: disable=broad-except
-                    pass
+            try:
+                job.cancel()
+                self.provider.runtime.delete_job(job.job_id())
+            except Exception:  # pylint: disable=broad-except
+                pass
 
     def test_runtime_service(self):
         """Test getting runtime service."""
@@ -219,6 +226,19 @@ def test_retrieve_job_done(self):
         self.assertEqual(job.job_id(), rjob.job_id())
         self.assertEqual(self.program_id, job.program_id)
 
+    def test_retrieve_all_jobs(self):
+        """Test retrieving all jobs."""
+        job = self._run_program()
+        rjobs = self.provider.runtime.jobs()
+        found = False
+        for rjob in rjobs:
+            if rjob.job_id() == job.job_id():
+                self.assertEqual(job.program_id, rjob.program_id)
+                self.assertEqual(job.inputs, rjob.inputs)
+                found = True
+                break
+        self.assertTrue(found, f"Job {job.job_id()} not returned.")
+
     def test_cancel_job_queued(self):
         """Test canceling a queued job."""
         _ = self._run_program(iterations=10)
@@ -230,7 +250,6 @@ def test_cancel_job_queued(self):
         rjob = self.provider.runtime.job(job.job_id())
         self.assertEqual(rjob.status(), JobStatus.CANCELLED)
 
-    @unittest.skip("Skip until fixed")
     def test_cancel_job_running(self):
         """Test canceling a running job."""
         job = self._run_program(iterations=3)
@@ -245,7 +264,23 @@ def test_cancel_job_done(self):
         """Test canceling a finished job."""
         job = self._run_program()
         job.wait_for_final_state()
-        job.cancel()
+        with self.assertRaises(RuntimeInvalidStateError):
+            job.cancel()
+
+    def test_delete_job(self):
+        """Test deleting a job."""
+        sub_tests = [JobStatus.QUEUED, JobStatus.RUNNING, JobStatus.DONE]
+        for status in sub_tests:
+            with self.subTest(status=status):
+                if status == JobStatus.QUEUED:
+                    _ = self._run_program(iterations=10)
+
+                job = self._run_program(iterations=2)
+                while job.status() != status:
+                    time.sleep(5)
+                self.provider.runtime.delete_job(job.job_id())
+                with self.assertRaises(RuntimeJobNotFound):
+                    self.provider.runtime.job(job.job_id())
 
     def test_interim_result_callback(self):
         """Test interim result callback."""
@@ -297,7 +332,6 @@ def result_callback(job_id, interim_result):
         self.assertFalse(callback_err)
         self.assertIsNone(job._ws_client._ws)
 
-    @unittest.skip("Skip until 267 is fixed")
     def test_stream_results_done(self):
         """Test streaming interim results after job is done."""
         def result_callback(job_id, interim_result):
@@ -308,8 +342,9 @@ def result_callback(job_id, interim_result):
         called_back = False
         job = self._run_program(interim_results="foobar")
         job.wait_for_final_state()
+        job._status = JobStatus.RUNNING  # Allow stream_results()
         job.stream_results(result_callback)
-        time.sleep(1)
+        time.sleep(2)
         self.assertFalse(called_back)
         self.assertIsNone(job._ws_client._ws)
 
@@ -333,7 +368,7 @@ def result_callback(job_id, interim_result):
         self.assertEqual(iterations-1, final_it)
         self.assertIsNone(job._ws_client._ws)
 
-    # @unittest.skip("Skip until 277 is fixed")
+    @unittest.skip("Skip until 277 is fixed")
     def test_callback_cancel_job(self):
         """Test canceling a running job while streaming results."""
         def result_callback(job_id, interim_result):
@@ -406,6 +441,13 @@ def test_wait_for_final_state(self):
         job.wait_for_final_state()
         self.assertEqual(JobStatus.DONE, job.status())
 
+    def test_logout(self):
+        """Test logout."""
+        self.provider.runtime.logout()
+        # Make sure we can still do things.
+        self._upload_program()
+        _ = self._run_program()
+
     def _validate_program(self, program):
         """Validate a program."""
         # TODO add more validation
@@ -421,6 +463,7 @@ def _upload_program(self, name=None, max_execution_time=300):
         program_id = self.provider.runtime.upload_program(
             name=name,
             data=self.RUNTIME_PROGRAM.encode(),
+            metadata=self.RUNTIME_PROGRAM_METADATA,
             max_execution_time=max_execution_time)
         self.to_delete.append(program_id)
         return program_id

From 8bdcc15c8e96a02970d0a56daac8f338ecbc13ed Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Thu, 29 Apr 2021 14:38:57 -0400
Subject: [PATCH 43/59] add result decoder

---
 .../ibmq/runtime/ibm_runtime_service.py       | 13 +++--
 .../providers/ibmq/runtime/result_decoder.py  | 37 +++++++++++++
 qiskit/providers/ibmq/runtime/runtime_job.py  | 53 ++++++++++---------
 test/ibmq/runtime/fake_runtime_client.py      | 17 +++++-
 test/ibmq/runtime/test_runtime.py             | 23 ++++++--
 test/ibmq/runtime/test_runtime_integration.py | 19 +++----
 test/ibmq/runtime/utils.py                    | 20 +++++++
 7 files changed, 138 insertions(+), 44 deletions(-)
 create mode 100644 qiskit/providers/ibmq/runtime/result_decoder.py

diff --git a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
index 12af145a6..5897e1bb8 100644
--- a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
+++ b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
@@ -13,7 +13,7 @@
 """Qiskit runtime service."""
 
 import logging
-from typing import Dict, Callable, Optional, Union, List, Any
+from typing import Dict, Callable, Optional, Union, List, Any, Type
 import json
 import copy
 
@@ -21,6 +21,7 @@
 
 from .runtime_job import RuntimeJob
 from .runtime_program import RuntimeProgram, ProgramParameter, ProgramResult
+from .result_decoder import ResultDecoder
 from .utils import RuntimeEncoder, RuntimeDecoder
 from .exceptions import (QiskitRuntimeError, RuntimeDuplicateProgramError, RuntimeProgramNotFound,
                          RuntimeJobNotFound)
@@ -169,7 +170,8 @@ def run(
             program_id: str,
             options: Dict,
             inputs: Dict,
-            callback: Optional[Callable] = None
+            callback: Optional[Callable] = None,
+            result_decoder: Optional[Type[ResultDecoder]] = None
     ) -> RuntimeJob:
         """Execute the runtime program.
 
@@ -185,6 +187,9 @@ def run(
                     1. Job ID
                     2. Job interim result.
 
+            result_decoder: A :class:`ResultDecoder` subclass used to decode job results.
+                ``ResultDecoder`` is used if not specified.
+
         Returns:
             A ``RuntimeJob`` instance representing the execution.
 
@@ -196,6 +201,7 @@ def run(
 
         backend_name = options['backend_name']
         params_str = json.dumps(inputs, cls=RuntimeEncoder)
+        result_decoder = result_decoder or ResultDecoder
         response = self._api_client.program_run(program_id=program_id,
                                                 credentials=self._provider.credentials,
                                                 backend_name=backend_name,
@@ -206,7 +212,8 @@ def run(
                          api_client=self._api_client,
                          ws_client=RuntimeWebsocketClient(self._ws_url, self._access_token),
                          job_id=response['id'], program_id=program_id, params=inputs,
-                         user_callback=callback)
+                         user_callback=callback,
+                         result_decoder=result_decoder)
         return job
 
     def upload_program(
diff --git a/qiskit/providers/ibmq/runtime/result_decoder.py b/qiskit/providers/ibmq/runtime/result_decoder.py
new file mode 100644
index 000000000..de5f29c14
--- /dev/null
+++ b/qiskit/providers/ibmq/runtime/result_decoder.py
@@ -0,0 +1,37 @@
+# This code is part of Qiskit.
+#
+# (C) Copyright IBM 2021.
+#
+# This code is licensed under the Apache License, Version 2.0. You may
+# obtain a copy of this license in the LICENSE.txt file in the root directory
+# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Any modifications or derivative works of this code must retain this
+# copyright notice, and modified files need to carry a notice indicating
+# that they have been altered from the originals.
+
+"""Qiskit runtime job result decoder."""
+
+import json
+from typing import Any
+
+from .utils import RuntimeDecoder
+
+
+class ResultDecoder:
+    """Runtime job result decoder."""
+
+    @classmethod
+    def decode(cls, data: str) -> Any:
+        """Decode the result data.
+
+        Args:
+            data: Result data to be decoded.
+
+        Returns:
+            Decoded result data.
+        """
+        try:
+            return json.loads(data, cls=RuntimeDecoder)
+        except json.JSONDecodeError:
+            return data
diff --git a/qiskit/providers/ibmq/runtime/runtime_job.py b/qiskit/providers/ibmq/runtime/runtime_job.py
index ebb23c393..0e9212cfd 100644
--- a/qiskit/providers/ibmq/runtime/runtime_job.py
+++ b/qiskit/providers/ibmq/runtime/runtime_job.py
@@ -12,10 +12,9 @@
 
 """Qiskit runtime job."""
 
-from typing import Any, Optional, Callable, Dict
+from typing import Any, Optional, Callable, Dict, Type
 import time
 import logging
-import json
 import asyncio
 from concurrent import futures
 import traceback
@@ -26,9 +25,9 @@
 from qiskit.providers.jobstatus import JobStatus, JOB_FINAL_STATES
 from qiskit.providers.ibmq import ibmqbackend  # pylint: disable=unused-import
 
-from .utils import RuntimeDecoder
 from .constants import API_TO_JOB_STATUS
 from .exceptions import RuntimeJobFailureError, RuntimeInvalidStateError
+from .result_decoder import ResultDecoder
 from ..api.clients import RuntimeClient, RuntimeWebsocketClient
 from ..exceptions import IBMQError
 from ..api.exceptions import RequestsApiError
@@ -82,7 +81,8 @@ def __init__(
             job_id: str,
             program_id: str,
             params: Optional[Dict] = None,
-            user_callback: Optional[Callable] = None
+            user_callback: Optional[Callable] = None,
+            result_decoder: Type[ResultDecoder] = ResultDecoder
     ) -> None:
         """RuntimeJob constructor.
 
@@ -94,6 +94,7 @@ def __init__(
             program_id: ID of the program this job is for.
             params: Job parameters.
             user_callback: User callback function.
+            result_decoder: A :class:`ResultDecoder` subclass used to decode job results.
         """
         self._job_id = job_id
         self._backend = backend
@@ -103,6 +104,7 @@ def __init__(
         self._params = params or {}
         self._program_id = program_id
         self._status = JobStatus.INITIALIZING
+        self._result_decoder = result_decoder
 
         # Used for streaming
         self._streaming = False
@@ -117,6 +119,7 @@ def result(
             self,
             timeout: Optional[float] = None,
             wait: float = 5,
+            decoder: Optional[Type[ResultDecoder]] = None
     ) -> Any:
         """Return the results of the job.
 
@@ -127,6 +130,7 @@ def result(
         Args:
             timeout: Number of seconds to wait for job.
             wait: Seconds between queries.
+            decoder: A :class:`ResultDecoder` subclass used to decode job results.
 
         Returns:
             Runtime job result.
@@ -134,13 +138,14 @@ def result(
         Raises:
             RuntimeJobFailureError: If the job failed.
         """
-        if not self._results:
+        _decoder = decoder or self._result_decoder
+        if not self._results or _decoder != self._result_decoder:
             self.wait_for_final_state(timeout=timeout, wait=wait)
             result_raw = self._api_client.job_results(job_id=self.job_id())
             if self._status == JobStatus.ERROR:
                 raise RuntimeJobFailureError(f"Unable to retrieve result for job {self.job_id()}. "
                                              f"Job has failed:\n{result_raw}")
-            self._results = self._decode_data(result_raw)
+            self._results = _decoder.decode(result_raw)
         return self._results
 
     def cancel(self) -> None:
@@ -200,7 +205,11 @@ def wait_for_final_state(
             time.sleep(wait)
             status = self.status()
 
-    def stream_results(self, callback: Callable) -> None:
+    def stream_results(
+            self,
+            callback: Callable,
+            decoder: Optional[Type[ResultDecoder]] = None
+    ) -> None:
         """Start streaming job results.
 
         Args:
@@ -210,6 +219,8 @@ def stream_results(self, callback: Callable) -> None:
                     1. Job ID
                     2. Job interim result.
 
+            decoder: A :class:`ResultDecoder` subclass used to decode job results.
+
         Raises:
             RuntimeInvalidStateError: If a callback function is already streaming results or
                 if the job already finished.
@@ -223,7 +234,8 @@ def stream_results(self, callback: Callable) -> None:
         self._executor.submit(self._start_websocket_client,
                               result_queue=self._result_queue)
         self._executor.submit(self._stream_results,
-                              result_queue=self._result_queue, user_callback=callback)
+                              result_queue=self._result_queue, user_callback=callback,
+                              decoder=decoder)
 
     def _cancel_result_streaming(self) -> None:
         """Cancel result streaming."""
@@ -263,21 +275,28 @@ def _start_websocket_client(
                     self._ws_client.disconnect())
             self._streaming = False
 
-    def _stream_results(self, result_queue: queue.Queue, user_callback: Callable) -> None:
+    def _stream_results(
+            self,
+            result_queue: queue.Queue,
+            user_callback: Callable,
+            decoder: Optional[Type[ResultDecoder]] = None
+    ) -> None:
         """Stream interim results.
 
         Args:
             result_queue: Queue used to pass websocket messages.
             user_callback: User callback function.
+            decoder: A :class:`ResultDecoder` (sub)class used to decode job results.
         """
         logger.debug("Start result streaming for job %s", self.job_id())
+        _decoder = decoder or self._result_decoder
         while True:
             try:
                 response = result_queue.get()
                 if response == self.POISON_PILL:
                     self._empty_result_queue(result_queue)
                     return
-                user_callback(self.job_id(), self._decode_data(response))
+                user_callback(self.job_id(), _decoder.decode(response))
             except Exception:  # pylint: disable=broad-except
                 logger.warning(
                     "An error occurred while streaming results "
@@ -295,20 +314,6 @@ def _empty_result_queue(self, result_queue: queue.Queue) -> None:
         except queue.Empty:
             pass
 
-    def _decode_data(self, data: Any) -> Any:
-        """Attempt to decode data using default decoder.
-
-        Args:
-            data: Data to be decoded.
-
-        Returns:
-            Decoded data, or the original data if decoding failed.
-        """
-        try:
-            return json.loads(data, cls=RuntimeDecoder)
-        except json.JSONDecodeError:
-            return data
-
     def job_id(self) -> str:
         """Return a unique id identifying the job.
 
diff --git a/test/ibmq/runtime/fake_runtime_client.py b/test/ibmq/runtime/fake_runtime_client.py
index 4459c675a..d5d3a4df0 100644
--- a/test/ibmq/runtime/fake_runtime_client.py
+++ b/test/ibmq/runtime/fake_runtime_client.py
@@ -14,10 +14,12 @@
 
 import time
 import uuid
+import json
 from concurrent.futures import ThreadPoolExecutor
 
 from qiskit.providers.ibmq.credentials import Credentials
 from qiskit.providers.ibmq.api.exceptions import RequestsApiError
+from qiskit.providers.ibmq.runtime.utils import RuntimeEncoder
 
 
 class BaseFakeProgram:
@@ -71,7 +73,7 @@ def _auto_progress(self):
             self._status = status
 
         if self._status == "SUCCEEDED":
-            self._result = "foo"
+            self._result = json.dumps("foo")
 
     def to_dict(self):
         """Convert to dictionary format."""
@@ -120,6 +122,19 @@ def cancel(self):
         self._status = "CANCELLED"
 
 
+class CustomResultRuntimeJob(BaseFakeRuntimeJob):
+    """Class for using custom job result."""
+
+    custom_result = "bar"
+
+    def _auto_progress(self):
+        """Automatically update job status."""
+        super()._auto_progress()
+
+        if self._status == "SUCCEEDED":
+            self._result = json.dumps(self.custom_result, cls=RuntimeEncoder)
+
+
 class BaseFakeRuntimeClient:
     """Base class for faking the runtime client."""
 
diff --git a/test/ibmq/runtime/test_runtime.py b/test/ibmq/runtime/test_runtime.py
index cdafde03f..c91335a83 100644
--- a/test/ibmq/runtime/test_runtime.py
+++ b/test/ibmq/runtime/test_runtime.py
@@ -31,8 +31,10 @@
 
 
 from ...ibmqtestcase import IBMQTestCase
-from .fake_runtime_client import BaseFakeRuntimeClient, FailedRuntimeJob, CancelableRuntimeJob
-from .utils import SerializableClass, UnserializableClass
+from .fake_runtime_client import (BaseFakeRuntimeClient, FailedRuntimeJob, CancelableRuntimeJob,
+                                  CustomResultRuntimeJob)
+from .utils import (SerializableClass, UnserializableClass, SerializableClassDecoder,
+                    get_complex_types)
 
 
 class TestRuntime(IBMQTestCase):
@@ -194,6 +196,19 @@ def test_wait_for_final_state(self):
         job.wait_for_final_state()
         self.assertEqual(JobStatus.DONE, job.status())
 
+    def test_result_decoder(self):
+        """Test result decoder."""
+        custom_result = get_complex_types()
+        job_cls = CustomResultRuntimeJob
+        job_cls.custom_result = custom_result
+
+        sub_tests = [(SerializableClassDecoder, None), (None, SerializableClassDecoder)]
+        for result_decoder, decoder in sub_tests:
+            with self.subTest(decoder=decoder):
+                job = self._run_program(job_classes=job_cls, decoder=result_decoder)
+                result = job.result(decoder=decoder)
+                self.assertIsInstance(result['serializable_class'], SerializableClass)
+
     def _upload_program(self, name=None, max_execution_time=300):
         """Upload a new program."""
         name = name or uuid.uuid4().hex
@@ -204,7 +219,7 @@ def _upload_program(self, name=None, max_execution_time=300):
             max_execution_time=max_execution_time)
         return program_id
 
-    def _run_program(self, program_id=None, inputs=None, job_classes=None):
+    def _run_program(self, program_id=None, inputs=None, job_classes=None, decoder=None):
         """Run a program."""
         options = {'backend_name': "some_backend"}
         if job_classes:
@@ -212,7 +227,7 @@ def _run_program(self, program_id=None, inputs=None, job_classes=None):
         if program_id is None:
             program_id = self._upload_program()
         job = self.runtime.run(program_id=program_id, inputs=inputs,
-                               options=options)
+                               options=options, result_decoder=decoder)
         return job
 
     # TODO add websocket tests
diff --git a/test/ibmq/runtime/test_runtime_integration.py b/test/ibmq/runtime/test_runtime_integration.py
index 333a41a8d..2cb2c0ab4 100644
--- a/test/ibmq/runtime/test_runtime_integration.py
+++ b/test/ibmq/runtime/test_runtime_integration.py
@@ -18,7 +18,7 @@
 import time
 import random
 
-from qiskit.providers.jobstatus import JobStatus, JOB_FINAL_STATES
+from qiskit.providers.jobstatus import JobStatus
 from qiskit.providers.ibmq.exceptions import IBMQNotAuthorizedError
 from qiskit.providers.ibmq.runtime.exceptions import (RuntimeDuplicateProgramError,
                                                       RuntimeProgramNotFound,
@@ -28,7 +28,7 @@
 
 from ...ibmqtestcase import IBMQTestCase
 from ...decorators import requires_runtime_device
-from .utils import SerializableClass
+from .utils import SerializableClass, SerializableClassDecoder, get_complex_types
 
 
 @unittest.skipIf(not os.environ.get('USE_STAGING_CREDENTIALS', ''), "Only runs on staging")
@@ -396,7 +396,7 @@ def result_callback(job_id, interim_result):
 
     def test_final_result(self):
         """Test getting final result."""
-        final_result = self._get_complex_types()
+        final_result = get_complex_types()
         job = self._run_program(final_result=final_result)
         result = job.result()
         self._assert_complex_types_equal(final_result, result)
@@ -412,7 +412,7 @@ def test_job_status(self):
 
     def test_job_inputs(self):
         """Test job inputs."""
-        interim_results = self._get_complex_types()
+        interim_results = get_complex_types()
         inputs = {'iterations': 1,
                   'interim_results': interim_results}
         options = {'backend_name': self.backend.name()}
@@ -472,16 +472,11 @@ def _get_program_name(self):
         """Return a unique program name."""
         return self.PROGRAM_PREFIX + "_" + uuid.uuid4().hex
 
-    def _get_complex_types(self):
-        return {"string": "foo",
-                "float": 1.5,
-                "complex": 2+3j,
-                "class": SerializableClass("foo")}
-
     def _assert_complex_types_equal(self, expected, received):
         """Verify the received data in complex types is expected."""
-        if 'class' in received:
-            received['class'] = SerializableClass.from_json(received['class'])
+        if 'serializable_class' in received:
+            received['serializable_class'] = \
+                SerializableClass.from_json(received['serializable_class'])
         self.assertEqual(expected, received)
 
     def _run_program(self, program_id=None, iterations=1,
diff --git a/test/ibmq/runtime/utils.py b/test/ibmq/runtime/utils.py
index 93c1bf796..caed4f6bd 100644
--- a/test/ibmq/runtime/utils.py
+++ b/test/ibmq/runtime/utils.py
@@ -14,6 +14,15 @@
 
 import json
 
+from qiskit.providers.ibmq.runtime.result_decoder import ResultDecoder
+
+
+def get_complex_types():
+    return {"string": "foo",
+            "float": 1.5,
+            "complex": 2+3j,
+            "serializable_class": SerializableClass("foo")}
+
 
 class SerializableClass:
     """Custom class with serialization methods."""
@@ -34,6 +43,17 @@ def __eq__(self, other):
         return self.value == other.value
 
 
+class SerializableClassDecoder(ResultDecoder):
+
+    @classmethod
+    def decode(cls, data):
+        decoded = super().decode(data)
+        if 'serializable_class' in decoded:
+            decoded['serializable_class'] = \
+                SerializableClass.from_json(decoded['serializable_class'])
+        return decoded
+
+
 class UnserializableClass:
     """Custom class without serialization methods."""
 

From 8fb76ba97bfd0a8310b963bddc1955b5ef9cc6d6 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Tue, 4 May 2021 17:12:05 -0400
Subject: [PATCH 44/59] add metadata

---
 qiskit/providers/ibmq/api/clients/runtime.py  | 38 +++++++--
 qiskit/providers/ibmq/api/rest/runtime.py     | 58 +++++++++++---
 qiskit/providers/ibmq/runtime/__init__.py     | 15 ++--
 .../ibmq/runtime/ibm_runtime_service.py       | 68 +++++++++++-----
 .../program/program_metadata_sample.json      | 17 ++--
 .../ibmq/runtime/program/program_template.py  | 19 +----
 .../ibmq/runtime/program/user_messenger.py    | 11 ++-
 qiskit/providers/ibmq/runtime/runtime_job.py  |  5 +-
 .../providers/ibmq/runtime/runtime_program.py | 19 +++--
 test/ibmq/runtime/fake_runtime_client.py      | 60 +++++++++++---
 test/ibmq/runtime/test_runtime.py             | 79 ++++++++++++++++++-
 test/ibmq/runtime/test_runtime_integration.py | 28 ++++---
 12 files changed, 298 insertions(+), 119 deletions(-)

diff --git a/qiskit/providers/ibmq/api/clients/runtime.py b/qiskit/providers/ibmq/api/clients/runtime.py
index f29535c47..700ea9095 100644
--- a/qiskit/providers/ibmq/api/clients/runtime.py
+++ b/qiskit/providers/ibmq/api/clients/runtime.py
@@ -13,7 +13,7 @@
 """Client for accessing IBM Quantum runtime service."""
 
 import logging
-from typing import List, Dict, Union
+from typing import List, Dict, Union, Optional
 
 from qiskit.providers.ibmq.credentials import Credentials
 from qiskit.providers.ibmq.api.session import RetrySession
@@ -49,22 +49,40 @@ def list_programs(self) -> List[Dict]:
 
     def program_create(
             self,
-            program_name: str,
             program_data: Union[bytes, str],
-            max_execution_time: int
+            name: str,
+            description: str,
+            max_execution_time: int,
+            version: Optional[str] = None,
+            backend_requirements: Optional[Dict] = None,
+            parameters: Optional[Dict] = None,
+            return_values: Optional[List] = None,
+            interim_results: Optional[List] = None
     ) -> Dict:
         """Create a new program.
 
         Args:
-            program_name: Name of the program.
+            name: Name of the program.
             program_data: Program data.
+            description: Program description.
             max_execution_time: Maximum execution time.
+            version: Program version.
+            backend_requirements: Backend requirements.
+            parameters: Program parameters.
+            return_values: Program return values.
+            interim_results: Program interim results.
 
         Returns:
             Server response.
         """
-        return self.api.create_program(program_name=program_name, program_data=program_data,
-                                       max_execution_time=max_execution_time)
+        return self.api.create_program(
+            program_data=program_data,
+            name=name,
+            description=description, max_execution_time=max_execution_time,
+            version=version, backend_requirements=backend_requirements,
+            parameters=parameters, return_values=return_values,
+            interim_results=interim_results
+        )
 
     def program_get(self, program_id: str) -> Dict:
         """Return a specific program.
@@ -131,13 +149,17 @@ def job_get(self, job_id: str) -> Dict:
         logger.debug("Runtime job get response: %s", response)
         return response
 
-    def jobs_get(self) -> List:
+    def jobs_get(self, limit: int = None, skip: int = None) -> List:
         """Get job data for all jobs.
 
+        Args:
+            limit: Number of results to return.
+            skip: Number of results to skip.
+
         Returns:
             A list of job data.
         """
-        return self.api.jobs_get()
+        return self.api.jobs_get(limit=limit, skip=skip)
 
     def job_results(self, job_id: str) -> str:
         """Get the results of a program job.
diff --git a/qiskit/providers/ibmq/api/rest/runtime.py b/qiskit/providers/ibmq/api/rest/runtime.py
index 001594819..10cc4f58d 100644
--- a/qiskit/providers/ibmq/api/rest/runtime.py
+++ b/qiskit/providers/ibmq/api/rest/runtime.py
@@ -13,7 +13,7 @@
 """Random REST adapter."""
 
 import logging
-from typing import Dict, List, Any, Union
+from typing import Dict, List, Any, Union, Optional
 import json
 from concurrent import futures
 
@@ -65,30 +65,55 @@ def list_programs(self) -> List[Dict]:
 
     def create_program(
             self,
-            program_name: str,
             program_data: Union[bytes, str],
-            max_execution_time: int
+            name: str,
+            description: str,
+            max_execution_time: int,
+            version: Optional[str] = None,
+            backend_requirements: Optional[Dict] = None,
+            parameters: Optional[Dict] = None,
+            return_values: Optional[List] = None,
+            interim_results: Optional[List] = None
     ) -> Dict:
         """Upload a new program.
 
         Args:
-            program_name: Name of the program.
             program_data: Program data.
+            name: Name of the program.
+            description: Program description.
             max_execution_time: Maximum execution time.
+            version: Program version.
+            backend_requirements: Backend requirements.
+            parameters: Program parameters.
+            return_values: Program return values.
+            interim_results: Program interim results.
 
         Returns:
             JSON response.
         """
         url = self.get_url('programs')
-        data = {'name': (None, program_name),
-                'cost': (None, str(max_execution_time))}
+        data = {'name': name,
+                'cost': str(max_execution_time),
+                'description': description.encode(),
+                'max_execution_time': max_execution_time}
+        if version is not None:
+            data['version'] = version
+        if backend_requirements:
+            data['backendRequirements'] = json.dumps(backend_requirements)
+        if parameters:
+            data['parameters'] = json.dumps({"doc": parameters})
+        if return_values:
+            data['returnValues'] = json.dumps(return_values)
+        if interim_results:
+            data['interimResults'] = json.dumps(interim_results)
+
         if isinstance(program_data, str):
             with open(program_data, 'rb') as file:
-                data['program'] = (program_name, file)
-                response = self.session.post(url, files=data).json()
+                files = {'program': (name, file)}
+                response = self.session.post(url, data=data, files=files).json()
         else:
-            data['program'] = (program_name, program_data)
-            response = self.session.post(url, files=data).json()
+            files = {'program': (name, program_data)}
+            response = self.session.post(url, data=data, files=files).json()
         return response
 
     def program_run(
@@ -125,14 +150,23 @@ def program_run(
         data = json.dumps(payload)
         return self.session.post(url, data=data).json()
 
-    def jobs_get(self) -> List[Dict]:
+    def jobs_get(self, limit: int = None, skip: int = None) -> List[Dict]:
         """Get a list of job data.
 
+        Args:
+            limit: Number of results to return.
+            skip: Number of results to skip.
+
         Returns:
             A list of job data.
         """
         url = self.get_url('jobs')
-        return self.session.get(url).json()
+        payload = {}
+        if limit:
+            payload['limit'] = limit
+        if skip:
+            payload['offset'] = skip
+        return self.session.get(url, json=payload).json()
 
     def logout(self) -> None:
         """Clear authorization cache."""
diff --git a/qiskit/providers/ibmq/runtime/__init__.py b/qiskit/providers/ibmq/runtime/__init__.py
index 56f91f479..1108f7cb1 100644
--- a/qiskit/providers/ibmq/runtime/__init__.py
+++ b/qiskit/providers/ibmq/runtime/__init__.py
@@ -37,12 +37,12 @@
     `Qiskit-Partners/qiskit-runtime
     <https://github.com/Qiskit-Partners/qiskit-runtime>`_ for more information.
 
-The Qiskit Runtime Service allows authorized users to upload their quantum programs.
-A quantum program is a piece of code that takes certain inputs, performs
-quantum and classical processing, and returns the results. Other
+The Qiskit Runtime Service allows authorized users to upload their Qiskit quantum programs.
+A Qiskit quantum program is a piece of Python code that takes certain inputs, performs
+quantum and classical processing, and returns the results. The same or other
 authorized users can invoke these quantum programs by simply passing in parameters.
 
-These quantum programs, sometimes called runtime programs, run in a special
+These Qiskit quantum programs, sometimes called runtime programs, run in a special
 runtime environment that significantly reduces waiting time during computational
 iterations.
 
@@ -70,9 +70,9 @@
 
 In the example above, ``provider.runtime`` points to the runtime service class
 :class:`IBMRuntimeService`, which is the main entry
-point for using this service. The example prints the program definitions of all
+point for using this service. The example prints the program metadata of all
 available runtime programs and of just the ``circuit-runner`` program. A program
-definition consists of a program's ID, name, description, input parameters,
+metadata consists of a program's ID, name, description, input parameters,
 return values, interim results, and other information that helps you to know
 more about the program.
 
@@ -203,6 +203,7 @@ def interim_result_callback(job_id, interim_result):
    RuntimeProgram
    UserMessenger
    ProgramBackend
+   ResultDecoder
 """
 
 from .ibm_runtime_service import IBMRuntimeService
@@ -210,3 +211,5 @@ def interim_result_callback(job_id, interim_result):
 from .runtime_program import RuntimeProgram
 from .program.user_messenger import UserMessenger
 from .program.program_backend import ProgramBackend
+from .result_decoder import ResultDecoder
+from .utils import RuntimeEncoder, RuntimeDecoder
diff --git a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
index 5897e1bb8..6b2fee324 100644
--- a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
+++ b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
@@ -158,12 +158,21 @@ def _to_program(self, response: Dict) -> RuntimeProgram:
         Returns:
             A ``RuntimeProgram`` instance.
         """
+        backend_req = json.loads(response.get('backendRequirements', '{}'))
+        params = json.loads(response.get('parameters', '{}')).get("doc", [])
+        ret_vals = json.loads(response.get('returnValues', '{}'))
+        interim_results = json.loads(response.get('interimResults', '{}'))
+
         return RuntimeProgram(program_name=response['name'],
                               program_id=response['id'],
                               description=response.get('description', ""),
-                              parameters=response.get('parameters', None),
-                              return_values=response.get('return_values', None),
-                              max_execution_time=response.get('cost', 0))
+                              parameters=params,
+                              return_values=ret_vals,
+                              interim_results=interim_results,
+                              max_execution_time=response.get('cost', 0),
+                              creation_date=response.get('creationDate', ""),
+                              version=response.get('version', "0"),
+                              backend_requirements=backend_req)
 
     def run(
             self,
@@ -177,8 +186,8 @@ def run(
 
         Args:
             program_id: Program ID.
-            options: Runtime options. Currently the only available option is
-                ``backend_name``, which is required.
+            options: Runtime options that control the execution environment.
+                Currently the only available option is ``backend_name``, which is required.
             inputs: Program input parameters. These input values are passed
                 to the runtime program.
             callback: Callback function to be invoked for any interim results.
@@ -225,7 +234,6 @@ def upload_program(
             description: Optional[str] = None,
             version: Optional[float] = None,
             backend_requirements: Optional[str] = None,
-            parameter_schema: Optional[str] = None,
             parameters: Optional[List[ProgramParameter]] = None,
             return_values: Optional[List[ProgramResult]] = None,
             interim_results: Optional[List[ProgramResult]] = None
@@ -259,8 +267,6 @@ def upload_program(
             description: Program description. Required if not specified via `metadata`.
             version: Program version. The default is 1.0 if not specified.
             backend_requirements: Backend requirements.
-            parameter_schema: Schema that can be used to validate the program
-                input parameters.
             parameters: A list of program input parameters.
             return_values: A list of program return values.
             interim_results: A list of program interim results.
@@ -279,16 +285,15 @@ def upload_program(
             metadata=metadata,
             name=name, max_execution_time=max_execution_time, description=description,
             version=version, backend_requirements=backend_requirements,
-            parameter_schema=parameter_schema, parameters=parameters,
+            parameters=parameters,
             return_values=return_values, interim_results=interim_results)
 
-        # TODO uncomment when metada is supported on server
-        # for req in ['name', 'description', 'max_execution_time']:
-        #     if req not in program_metadata or not program_metadata[req]:
-        #         raise IBMQInputValueError(f"{req} is a required metadata field.")
+        for req in ['name', 'description', 'max_execution_time']:
+            if req not in program_metadata or not program_metadata[req]:
+                raise IBMQInputValueError(f"{req} is a required metadata field.")
 
         try:
-            response = self._api_client.program_create(name, data, max_execution_time)
+            response = self._api_client.program_create(program_data=data, **program_metadata)
         except RequestsApiError as ex:
             if ex.status_code == 409:
                 raise RuntimeDuplicateProgramError(
@@ -309,7 +314,6 @@ def update_program(
             description: Optional[str] = None,
             version: Optional[float] = None,
             backend_requirements: Optional[str] = None,
-            parameter_schema: Optional[str] = None,
             parameters: Optional[List[ProgramParameter]] = None,
             return_values: Optional[List[ProgramResult]] = None,
             interim_results: Optional[List[ProgramResult]] = None
@@ -337,8 +341,6 @@ def update_program(
             description: Program description. Required if not specified via `metadata`.
             version: Program version. The default is 1.0 if not specified.
             backend_requirements: Backend requirements.
-            parameter_schema: Schema that can be used to validate the program
-                input parameters.
             parameters: A list of program input parameters.
             return_values: A list of program return values.
             interim_results: A list of program interim results.
@@ -354,7 +356,7 @@ def update_program(
             metadata=metadata,
             name=name, max_execution_time=max_execution_time, description=description,
             version=version, backend_requirements=backend_requirements,
-            parameter_schema=parameter_schema, parameters=parameters,
+            parameters=parameters,
             return_values=return_values, interim_results=interim_results)
 
         self.delete_program(program_id)
@@ -384,12 +386,31 @@ def _merge_metadata(
             else:
                 upd_metadata = copy.deepcopy(metadata)
 
+        self._tuple_to_dict(initial)
         initial.update(upd_metadata)
 
+        self._tuple_to_dict(kwargs)
         for key, val in kwargs.items():
             if val is not None:
                 initial[key] = val
-        return initial
+
+        # TODO validate metadata format
+        metadata_keys = ['name', 'max_execution_time', 'description', 'version',
+                         'backend_requirements', 'parameters', 'return_values',
+                         'interim_results']
+        return {key: val for key, val in initial.items() if key in metadata_keys}
+
+    def _tuple_to_dict(self, metadata: Dict) -> None:
+        """Convert fields in metadata from named tuples to dictionaries.
+
+        Args:
+            metadata: Metadata to be converted.
+        """
+        for key in ['parameters', 'return_values', 'interim_results']:
+            doc_list = metadata.pop(key, None)
+            if not doc_list or isinstance(doc_list[0], dict):
+                continue
+            metadata[key] = [dict(elem._asdict()) for elem in doc_list]
 
     def delete_program(self, program_id: str) -> None:
         """Delete a runtime program.
@@ -432,13 +453,17 @@ def job(self, job_id: str) -> RuntimeJob:
             raise QiskitRuntimeError(f"Failed to delete job: {ex}") from None
         return self._decode_job(response)
 
-    def jobs(self) -> List[RuntimeJob]:
+    def jobs(self, limit: int = 10, skip: int = 0) -> List[RuntimeJob]:
         """Retrieve all runtime jobs.
 
+        Args:
+            limit: Number of jobs to retrieve.
+            skip: Starting index for the job retrieval.
+
         Returns:
             A list of runtime jobs.
         """
-        response = self._api_client.jobs_get()
+        response = self._api_client.jobs_get(limit=limit, skip=skip)
         jobs = [self._decode_job(job) for job in response]
         return jobs
 
@@ -470,6 +495,7 @@ def _decode_job(self, raw_data: Dict) -> RuntimeJob:
         Returns:
             Decoded job data.
         """
+        print(f">>>>>> raw_data is {raw_data}")
         backend = self._provider.get_backend(raw_data['backend'])
 
         params = raw_data.get('params', {})
diff --git a/qiskit/providers/ibmq/runtime/program/program_metadata_sample.json b/qiskit/providers/ibmq/runtime/program/program_metadata_sample.json
index 52aa35f0e..a38c18fd6 100644
--- a/qiskit/providers/ibmq/runtime/program/program_metadata_sample.json
+++ b/qiskit/providers/ibmq/runtime/program/program_metadata_sample.json
@@ -1,18 +1,17 @@
-{ "name": "Quantum-Kernel-Alignment",
-  "description": "Quantum kernel alignment algorithm that learns, on a given dataset, a quantum kernel maximizing the SVM classification margin.",
-  "max_execution_time": 3600,
+{
+  "name": "runtime-simple",
+  "description": "Simple runtime program used for testing.",
+  "max_execution_time": 300,
   "version": 1.0,
   "backend_requirements": {"min_num_qubits":  5},
   "parameters": [
-    {"name": "feature_map", "description": "An instance of FeatureMapACME used to map classical data into a quantum state space.", "type": "FeatureMapACME", "required": true},
-    {"name": "data", "description": "NxD array of training data, where N is the number of samples and D is the feature dimension.", "type": "numpy.ndarray", "required": true},
-    {"name": "labels", "description": "Nx1 array of +/-1 labels of the N training samples.", "type": "numpy.ndarray", "required": true}
+    {"name": "iterations", "description": "Number of iterations to run. Each iteration generates a runs a random circuit.", "type": "integer", "required": true}
   ],
   "return_values": [
-    {"name": "aligned_kernel_parameters", "description": "The optimized kernel parameters found from quantum kernel alignment.", "type": "numpy.ndarray"},
-    {"name": "aligned_kernel_matrix", "description": "The aligned quantum kernel matrix evaluated with the optimized kernel parameters on the training data.", "type": "numpy.ndarray"}
+    {"name": "-", "description": "A string that says 'All done!'.", "type": "string"}
   ],
   "interim_results": [
-    {"name": "cost", "description": "Estimate of updated SVM objective function F using average of F(alpha_+, lambda_+) and F(alpha_-, lambda_-).", "type": "float"}
+    {"name": "iteration", "description": "Iteration number.", "type": "int"},
+    {"name": "counts", "description": "Histogram data of the circuit result.", "type": "dict"}
   ]
 }
\ No newline at end of file
diff --git a/qiskit/providers/ibmq/runtime/program/program_template.py b/qiskit/providers/ibmq/runtime/program/program_template.py
index 8a1461358..e5c328e96 100644
--- a/qiskit/providers/ibmq/runtime/program/program_template.py
+++ b/qiskit/providers/ibmq/runtime/program/program_template.py
@@ -19,12 +19,7 @@
 send circuits to the backend and messages to the user, respectively.
 """
 
-import sys
-import json
-
-from qiskit import Aer
 from qiskit.providers.ibmq.runtime import UserMessenger, ProgramBackend
-from qiskit.providers.ibmq.runtime.utils import RuntimeDecoder
 
 
 def program(backend: ProgramBackend, user_messenger: UserMessenger, **kwargs):
@@ -42,20 +37,10 @@ def main(backend: ProgramBackend, user_messenger: UserMessenger, **kwargs):
 
     Args:
         backend: Backend for the circuits to run on.
-        user_messenger: Used to communicate with the program consumer.
+        user_messenger: Used to communicate with the program user.
         kwargs: User inputs.
     """
     # Massage the input if necessary.
     result = program(backend, user_messenger, **kwargs)
     # UserMessenger can be used to publish final results.
-    user_messenger.publish(result, final_result=True)
-
-
-if __name__ == '__main__':
-    # This is used for testing locally using Aer simulator.
-    sim_backend = Aer.get_backend('qasm_simulator')
-    user_params = {}
-    if len(sys.argv) > 1:
-        # If there are user parameters.
-        user_params = json.loads(sys.argv[1], cls=RuntimeDecoder)
-    main(sim_backend, UserMessenger(), **user_params)
+    user_messenger.publish(result, final=True)
diff --git a/qiskit/providers/ibmq/runtime/program/user_messenger.py b/qiskit/providers/ibmq/runtime/program/user_messenger.py
index e14802a40..421f61ad7 100644
--- a/qiskit/providers/ibmq/runtime/program/user_messenger.py
+++ b/qiskit/providers/ibmq/runtime/program/user_messenger.py
@@ -10,31 +10,30 @@
 # copyright notice, and modified files need to carry a notice indicating
 # that they have been altered from the originals.
 
-"""Base class for handling communication with program consumers."""
+"""Base class for handling communication with program users."""
 
 import json
-from typing import Any
+from typing import Any, Type
 
 from ..utils import RuntimeEncoder
 
 
 class UserMessenger:
-    """Base class for handling communication with program consumers.
+    """Base class for handling communication with program users.
 
-    A program consumer is the user that executes the runtime program.
     This class can be used when writing a new runtime program.
     """
 
     def publish(
             self,
             message: Any,
-            encoder: json.JSONEncoder = RuntimeEncoder,  # type: ignore[assignment]
+            encoder: Type[json.JSONEncoder] = RuntimeEncoder,
             final: bool = False
     ) -> None:
         """Publish message.
 
         You can use this method to publish messages, such as interim and final results,
-        to the program consumer. The messages will be made immediately available to the consumer,
+        to the program user. The messages will be made immediately available to the user,
         but they may choose not to receive the messages.
 
         The `final` parameter is used to indicate whether the message is
diff --git a/qiskit/providers/ibmq/runtime/runtime_job.py b/qiskit/providers/ibmq/runtime/runtime_job.py
index 0e9212cfd..c3824a4aa 100644
--- a/qiskit/providers/ibmq/runtime/runtime_job.py
+++ b/qiskit/providers/ibmq/runtime/runtime_job.py
@@ -160,7 +160,7 @@ def cancel(self) -> None:
             if ex.status_code == 409:
                 raise RuntimeInvalidStateError(f"Job cannot be cancelled: {ex}") from None
             raise
-        self._cancel_result_streaming()
+        self.cancel_result_streaming()
         self._status = JobStatus.CANCELLED
 
     def status(self) -> JobStatus:
@@ -237,9 +237,8 @@ def stream_results(
                               result_queue=self._result_queue, user_callback=callback,
                               decoder=decoder)
 
-    def _cancel_result_streaming(self) -> None:
+    def cancel_result_streaming(self) -> None:
         """Cancel result streaming."""
-        # TODO - consider making this public
         if not self._streaming:
             return
         self._streaming_loop.call_soon_threadsafe(self._streaming_task.cancel())
diff --git a/qiskit/providers/ibmq/runtime/runtime_program.py b/qiskit/providers/ibmq/runtime/runtime_program.py
index eac4cd26e..a65996bc2 100644
--- a/qiskit/providers/ibmq/runtime/runtime_program.py
+++ b/qiskit/providers/ibmq/runtime/runtime_program.py
@@ -48,9 +48,9 @@ def __init__(
             return_values: Optional[List] = None,
             interim_results: Optional[List] = None,
             max_execution_time: int = 0,
-            version: float = 1.0,
+            version: str = "0",
             backend_requirements: Optional[Dict] = None,
-            parameter_schema: str = ""
+            creation_date: str = ""
     ) -> None:
         """RuntimeProgram constructor.
 
@@ -64,7 +64,7 @@ def __init__(
             max_execution_time: Maximum execution time.
             version: Program version.
             backend_requirements: Backend requirements.
-            parameter_schema: Schema used to validate parameters.
+            creation_date: Program creation date.
         """
         self._name = program_name
         self._id = program_id
@@ -72,10 +72,10 @@ def __init__(
         self._max_execution_time = max_execution_time
         self._version = version
         self._backend_requirements = backend_requirements or {}
-        self._parameter_schema = parameter_schema
         self._parameters = []
         self._return_values = []
         self._interim_results = []
+        self._creation_date = creation_date
 
         if parameters:
             for param in parameters:
@@ -141,7 +141,6 @@ def to_dict(self) -> Dict:
             "max_execution_time": self.max_execution_time,
             "version": self.version,
             "backend_requirements": self.backend_requirements,
-            "parameter_schema": self._parameter_schema,
             "parameters": self.parameters,
             "return_values": self.return_values,
             "interim_results": self.interim_results
@@ -211,7 +210,7 @@ def max_execution_time(self) -> int:
         return self._max_execution_time
 
     @property
-    def version(self) -> float:
+    def version(self) -> str:
         """Return program version.
 
         Returns:
@@ -229,13 +228,13 @@ def backend_requirements(self) -> Dict:
         return self._backend_requirements
 
     @property
-    def parameter_schema(self) -> str:
-        """Return program parameter schema.
+    def creation_date(self) -> str:
+        """Return program creation date.
 
         Returns:
-            Program parameter schema.
+            Program creation date.
         """
-        return self._parameter_schema
+        return self._creation_date
 
 
 class ProgramParameter(NamedTuple):
diff --git a/test/ibmq/runtime/fake_runtime_client.py b/test/ibmq/runtime/fake_runtime_client.py
index d5d3a4df0..c0220414a 100644
--- a/test/ibmq/runtime/fake_runtime_client.py
+++ b/test/ibmq/runtime/fake_runtime_client.py
@@ -25,20 +25,39 @@
 class BaseFakeProgram:
     """Base class for faking a program."""
 
-    def __init__(self, program_id, name, data, cost=600):
+    def __init__(self, program_id, name, data, cost, description, version="1.0",
+                 backend_requirements=None, parameters=None, return_values=None,
+                 interim_results=None):
         """Initialize a fake program."""
         self._id = program_id
         self._name = name
         self._data = data
         self._cost = cost
+        self._description = description
+        self._version = version
+        self._backend_requirements = backend_requirements
+        self._parameters = parameters
+        self._return_values = return_values
+        self._interim_results = interim_results
 
     def to_dict(self, include_data=False):
         """Convert this program to a dictionary format."""
         out = {'id': self._id,
                'name': self._name,
-               'cost': self._cost}
+               'cost': self._cost,
+               'description': self._description,
+               'version': self._version}
         if include_data:
             out['data'] = self._data
+        if self._backend_requirements:
+            out['backendRequirements'] = json.dumps(self._backend_requirements)
+        if self._parameters:
+            out['parameters'] = json.dumps({"doc": self._parameters})
+        if self._return_values:
+            out['returnValues'] = json.dumps(self._return_values)
+        if self._interim_results:
+            out['interimResults'] = json.dumps(self._interim_results)
+
         return out
 
 
@@ -157,14 +176,20 @@ def list_programs(self):
             programs.append(prog.to_dict())
         return programs
 
-    def program_create(self, program_name, program_data, max_execution_time):
+    def program_create(self, program_data, name, description, max_execution_time, version="1.0",
+                       backend_requirements=None, parameters=None, return_values=None,
+                       interim_results=None):
         """Create a program."""
         if isinstance(program_data, str):
             with open(program_data, 'rb') as file:
                 program_data = file.read()
-        program_id = uuid.uuid4().hex
-        self._programs[program_id] = BaseFakeProgram(program_id, program_name, program_data,
-                                                     max_execution_time)
+        program_id = name
+        if program_id in self._programs:
+            raise RequestsApiError("Program already exists.", status_code=409)
+        self._programs[program_id] = BaseFakeProgram(
+            program_id=program_id, name=name, data=program_data, cost=max_execution_time,
+            description=description, version=version, backend_requirements=backend_requirements,
+            parameters=parameters, return_values=return_values, interim_results=interim_results)
         return {'id': program_id}
 
     def program_get(self, program_id: str):
@@ -202,14 +227,27 @@ def program_delete(self, program_id: str) -> None:
 
     def job_get(self, job_id):
         """Get the specific job."""
-        if job_id not in self._jobs:
-            raise RequestsApiError("Job not found", status_code=404)
-        return self._jobs[job_id].to_dict()
+        return self._get_job(job_id).to_dict()
+
+    def jobs_get(self):
+        """Get all jobs."""
+        return [job.to_dict() for job in self._jobs]
 
     def job_results(self, job_id):
         """Get the results of a program job."""
-        return self._jobs[job_id].result()
+        return self._get_job(job_id).result()
 
     def job_cancel(self, job_id):
         """Cancel the job."""
-        self._jobs[job_id].cancel()
+        self._get_job(job_id).cancel()
+
+    def job_delete(self, job_id):
+        """Delete the job."""
+        self._get_job(job_id)
+        del self._jobs[job_id]
+
+    def _get_job(self, job_id):
+        """Get job."""
+        if job_id not in self._jobs:
+            raise RequestsApiError("Job not found", status_code=404)
+        return self._jobs[job_id]
diff --git a/test/ibmq/runtime/test_runtime.py b/test/ibmq/runtime/test_runtime.py
index c91335a83..a39f7ee6d 100644
--- a/test/ibmq/runtime/test_runtime.py
+++ b/test/ibmq/runtime/test_runtime.py
@@ -13,6 +13,7 @@
 """Tests for runtime service."""
 
 import json
+import os
 from io import StringIO
 from unittest.mock import patch
 from unittest import mock
@@ -28,6 +29,7 @@
 from qiskit.providers.ibmq.runtime import IBMRuntimeService, RuntimeJob
 from qiskit.providers.ibmq.runtime.exceptions import (RuntimeProgramNotFound,
                                                       RuntimeJobFailureError)
+from qiskit.providers.ibmq.runtime.runtime_program import ProgramParameter, ProgramResult
 
 
 from ...ibmqtestcase import IBMQTestCase
@@ -40,6 +42,24 @@
 class TestRuntime(IBMQTestCase):
     """Class for testing runtime modules."""
 
+    DEFAULT_METADATA = {
+        "name": "qiskit-test",
+        "description": "Test program.",
+        "max_execution_time": 300,
+        "version": "0.1",
+        "backend_requirements": {"min_num_qubits":  5},
+        "parameters": [
+            {"name": "param1", "description": "Some parameter.",
+             "type": "integer", "required": True}
+        ],
+        "return_values": [
+            {"name": "ret_val", "description": "Some return value.", "type": "string"}
+        ],
+        "interim_results": [
+            {"name": "int_res", "description": "Some interim result", "type": "string"},
+        ]
+    }
+
     def setUp(self):
         """Initial test setup."""
         super().setUp()
@@ -103,7 +123,6 @@ def test_print_programs(self):
             for prog in programs:
                 self.assertIn(prog.program_id, stdout)
                 self.assertIn(prog.name, stdout)
-                # self.assertIn(prog.description, stdout)  TODO - add when enabled
 
     def test_upload_program(self):
         """Test uploading a program."""
@@ -209,14 +228,68 @@ def test_result_decoder(self):
                 result = job.result(decoder=decoder)
                 self.assertIsInstance(result['serializable_class'], SerializableClass)
 
+    def test_program_metadata(self):
+        """Test program metadata."""
+        fn = "test_metadata.json"
+        with open(fn, 'w') as file:
+            json.dump(self.DEFAULT_METADATA, file)
+        self.addCleanup(os.remove, fn)
+
+        sub_tests = [fn, self.DEFAULT_METADATA]
+
+        for metadata in sub_tests:
+            with self.subTest(metadata_type=type(metadata)):
+                program_id = self.runtime.upload_program(data="foo".encode(), metadata=metadata)
+                program = self.runtime.program(program_id)
+                self.runtime.delete_program(program_id)
+                self.assertEqual(self.DEFAULT_METADATA['name'], program.name)
+                self.assertEqual(self.DEFAULT_METADATA['description'], program.description)
+                self.assertEqual(self.DEFAULT_METADATA['max_execution_time'],
+                                 program.max_execution_time)
+                self.assertEqual(self.DEFAULT_METADATA["version"], program.version)
+                self.assertEqual(self.DEFAULT_METADATA['backend_requirements'],
+                                 program.backend_requirements)
+                self.assertEqual([ProgramParameter(**param) for param in
+                                  self.DEFAULT_METADATA['parameters']],
+                                 program.parameters)
+                self.assertEqual([ProgramResult(**ret) for ret in
+                                  self.DEFAULT_METADATA['return_values']],
+                                 program.return_values)
+                self.assertEqual([ProgramResult(**ret) for ret in
+                                  self.DEFAULT_METADATA['interim_results']],
+                                 program.interim_results)
+
+    def test_metadata_combined(self):
+        """Test combining metadata"""
+        update_metadata = {"version": "1.2", "max_execution_time": 600}
+        program_id = self.runtime.upload_program(
+            data="foo".encode(), metadata=self.DEFAULT_METADATA, **update_metadata)
+        program = self.runtime.program(program_id)
+        self.assertEqual(update_metadata['max_execution_time'], program.max_execution_time)
+        self.assertEqual(update_metadata["version"], program.version)
+
+    def test_update_program(self):
+        """Test updating program."""
+        update_metadata = {"version": "1.2", "max_execution_time": 600}
+        final_version = "1.3"
+        program_data = "foo".encode()
+        program_id = self.runtime.upload_program(
+            data=program_data, metadata=self.DEFAULT_METADATA)
+        self.runtime.update_program(program_id, data=program_data,
+                                    metadata=update_metadata, version=final_version)
+        program = self.runtime.program(program_id)
+        self.assertEqual(update_metadata['max_execution_time'], program.max_execution_time)
+        self.assertEqual(final_version, program.version)
+
     def _upload_program(self, name=None, max_execution_time=300):
         """Upload a new program."""
         name = name or uuid.uuid4().hex
-        data = "A fancy program"
+        data = "def main() {}"
         program_id = self.runtime.upload_program(
             name=name,
             data=data.encode(),
-            max_execution_time=max_execution_time)
+            max_execution_time=max_execution_time,
+            description="A test program")
         return program_id
 
     def _run_program(self, program_id=None, inputs=None, job_classes=None, decoder=None):
diff --git a/test/ibmq/runtime/test_runtime_integration.py b/test/ibmq/runtime/test_runtime_integration.py
index 2cb2c0ab4..208c7e3d6 100644
--- a/test/ibmq/runtime/test_runtime_integration.py
+++ b/test/ibmq/runtime/test_runtime_integration.py
@@ -17,6 +17,7 @@
 import uuid
 import time
 import random
+from contextlib import suppress
 
 from qiskit.providers.jobstatus import JobStatus
 from qiskit.providers.ibmq.exceptions import IBMQNotAuthorizedError
@@ -31,6 +32,8 @@
 from .utils import SerializableClass, SerializableClassDecoder, get_complex_types
 
 
+os.environ['USE_STAGING_CREDENTIALS'] = "true"
+
 @unittest.skipIf(not os.environ.get('USE_STAGING_CREDENTIALS', ''), "Only runs on staging")
 class TestRuntimeIntegration(IBMQTestCase):
     """Integration tests for runtime modules."""
@@ -104,18 +107,15 @@ def tearDown(self) -> None:
         super().tearDown()
         # Delete programs
         for prog in self.to_delete:
-            try:
+            with suppress(Exception):
                 self.provider.runtime.delete_program(prog)
-            except Exception:  # pylint: disable=broad-except
-                pass
 
         # Cancel and delete jobs.
         for job in self.to_cancel:
-            try:
+            with suppress(Exception):
                 job.cancel()
+            with suppress(Exception):
                 self.provider.runtime.delete_job(job.job_id())
-            except Exception:  # pylint: disable=broad-except
-                pass
 
     def test_runtime_service(self):
         """Test getting runtime service."""
@@ -398,11 +398,13 @@ def test_final_result(self):
         """Test getting final result."""
         final_result = get_complex_types()
         job = self._run_program(final_result=final_result)
-        result = job.result()
-        self._assert_complex_types_equal(final_result, result)
+        result = job.result(decoder=SerializableClassDecoder)
+        # self._assert_complex_types_equal(final_result, result)
+        self.assertEqual(final_result, result)
 
-        rresults = self.provider.runtime.job(job.job_id()).result()
-        self._assert_complex_types_equal(final_result, rresults)
+        rresults = self.provider.runtime.job(job.job_id()).result(decoder=SerializableClassDecoder)
+        self.assertEqual(final_result, rresults)
+        # self._assert_complex_types_equal(final_result, rresults)
 
     def test_job_status(self):
         """Test job status."""
@@ -450,11 +452,10 @@ def test_logout(self):
 
     def _validate_program(self, program):
         """Validate a program."""
-        # TODO add more validation
         self.assertTrue(program)
         self.assertTrue(program.name)
         self.assertTrue(program.program_id)
-        # self.assertTrue(program.description)
+        self.assertTrue(program.description)
         self.assertTrue(program.max_execution_time)
 
     def _upload_program(self, name=None, max_execution_time=300):
@@ -464,7 +465,8 @@ def _upload_program(self, name=None, max_execution_time=300):
             name=name,
             data=self.RUNTIME_PROGRAM.encode(),
             metadata=self.RUNTIME_PROGRAM_METADATA,
-            max_execution_time=max_execution_time)
+            max_execution_time=max_execution_time,
+            description="Qiskit test program")
         self.to_delete.append(program_id)
         return program_id
 

From 20a9bb943a5a089364c7627fcdeb103961029c20 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Tue, 4 May 2021 21:40:33 -0400
Subject: [PATCH 45/59] add pagination

---
 qiskit/providers/ibmq/api/clients/runtime.py  |  4 +-
 qiskit/providers/ibmq/api/rest/runtime.py     |  6 +--
 .../ibmq/runtime/ibm_runtime_service.py       | 25 +++++++++--
 .../providers/ibmq/runtime/runtime_program.py |  3 ++
 test/ibmq/runtime/fake_runtime_client.py      | 31 +++++++++++---
 test/ibmq/runtime/test_runtime.py             | 41 +++++++++++++++++++
 test/ibmq/runtime/test_runtime_integration.py | 15 ++++++-
 7 files changed, 110 insertions(+), 15 deletions(-)

diff --git a/qiskit/providers/ibmq/api/clients/runtime.py b/qiskit/providers/ibmq/api/clients/runtime.py
index 700ea9095..fc4b58384 100644
--- a/qiskit/providers/ibmq/api/clients/runtime.py
+++ b/qiskit/providers/ibmq/api/clients/runtime.py
@@ -149,7 +149,7 @@ def job_get(self, job_id: str) -> Dict:
         logger.debug("Runtime job get response: %s", response)
         return response
 
-    def jobs_get(self, limit: int = None, skip: int = None) -> List:
+    def jobs_get(self, limit: int = None, skip: int = None) -> Dict:
         """Get job data for all jobs.
 
         Args:
@@ -157,7 +157,7 @@ def jobs_get(self, limit: int = None, skip: int = None) -> List:
             skip: Number of results to skip.
 
         Returns:
-            A list of job data.
+            JSON response.
         """
         return self.api.jobs_get(limit=limit, skip=skip)
 
diff --git a/qiskit/providers/ibmq/api/rest/runtime.py b/qiskit/providers/ibmq/api/rest/runtime.py
index 10cc4f58d..2ececf8c7 100644
--- a/qiskit/providers/ibmq/api/rest/runtime.py
+++ b/qiskit/providers/ibmq/api/rest/runtime.py
@@ -150,7 +150,7 @@ def program_run(
         data = json.dumps(payload)
         return self.session.post(url, data=data).json()
 
-    def jobs_get(self, limit: int = None, skip: int = None) -> List[Dict]:
+    def jobs_get(self, limit: int = None, skip: int = None) -> Dict:
         """Get a list of job data.
 
         Args:
@@ -158,7 +158,7 @@ def jobs_get(self, limit: int = None, skip: int = None) -> List[Dict]:
             skip: Number of results to skip.
 
         Returns:
-            A list of job data.
+            JSON response.
         """
         url = self.get_url('jobs')
         payload = {}
@@ -166,7 +166,7 @@ def jobs_get(self, limit: int = None, skip: int = None) -> List[Dict]:
             payload['limit'] = limit
         if skip:
             payload['offset'] = skip
-        return self.session.get(url, json=payload).json()
+        return self.session.get(url, params=payload).json()
 
     def logout(self) -> None:
         """Clear authorization cache."""
diff --git a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
index 6b2fee324..99d185b0b 100644
--- a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
+++ b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
@@ -463,9 +463,27 @@ def jobs(self, limit: int = 10, skip: int = 0) -> List[RuntimeJob]:
         Returns:
             A list of runtime jobs.
         """
-        response = self._api_client.jobs_get(limit=limit, skip=skip)
-        jobs = [self._decode_job(job) for job in response]
-        return jobs
+        job_responses = []  # type: List[Dict[str, Any]]
+        current_page_limit = limit or 20
+
+        while True:
+            job_page = self._api_client.jobs_get(limit=current_page_limit, skip=skip)["jobs"]
+            if not job_page:
+                # Stop if there are no more jobs returned by the server.
+                break
+
+            job_responses += job_page
+            if limit:
+                if len(job_responses) >= limit:
+                    # Stop if we have reached the limit.
+                    break
+                current_page_limit = limit - len(job_responses)
+            else:
+                current_page_limit = 20
+
+            skip += len(job_page)
+
+        return [self._decode_job(job) for job in job_responses]
 
     def delete_job(self, job_id: str) -> None:
         """Delete a runtime job.
@@ -495,7 +513,6 @@ def _decode_job(self, raw_data: Dict) -> RuntimeJob:
         Returns:
             Decoded job data.
         """
-        print(f">>>>>> raw_data is {raw_data}")
         backend = self._provider.get_backend(raw_data['backend'])
 
         params = raw_data.get('params', {})
diff --git a/qiskit/providers/ibmq/runtime/runtime_program.py b/qiskit/providers/ibmq/runtime/runtime_program.py
index a65996bc2..70bc9c1c5 100644
--- a/qiskit/providers/ibmq/runtime/runtime_program.py
+++ b/qiskit/providers/ibmq/runtime/runtime_program.py
@@ -108,6 +108,9 @@ def _format_common(items: List) -> None:
         formatted = [f'{self.program_id}:',
                      f"  Name: {self.name}",
                      f"  Description: {self.description}",
+                     f"  Version: {self.version}",
+                     f"  Creation date: {self.creation_date}",
+                     f"  Max execution time: {self.max_execution_time}",
                      f"  Parameters:"]
 
         if self._parameters:
diff --git a/test/ibmq/runtime/fake_runtime_client.py b/test/ibmq/runtime/fake_runtime_client.py
index c0220414a..8565828fd 100644
--- a/test/ibmq/runtime/fake_runtime_client.py
+++ b/test/ibmq/runtime/fake_runtime_client.py
@@ -154,14 +154,31 @@ def _auto_progress(self):
             self._result = json.dumps(self.custom_result, cls=RuntimeEncoder)
 
 
+class TimedRuntimeJob(BaseFakeRuntimeJob):
+    """Class for a job that runs for the input seconds."""
+
+    def __init__(self, **kwargs):
+        self._runtime = kwargs.pop('run_time')
+        super().__init__(**kwargs)
+
+    def _auto_progress(self):
+        self._status = "RUNNING"
+        time.sleep(self._runtime)
+        self._status = "SUCCEEDED"
+
+        if self._status == "SUCCEEDED":
+            self._result = json.dumps("foo")
+
+
 class BaseFakeRuntimeClient:
     """Base class for faking the runtime client."""
 
-    def __init__(self):
+    def __init__(self, job_classes=None, job_kwargs=None):
         """Initialize a fake runtime client."""
         self._programs = {}
         self._jobs = {}
-        self._job_classes = []
+        self._job_classes = job_classes or []
+        self._job_kwargs = job_kwargs or {}
 
     def set_job_classes(self, classes):
         """Set job classes to use."""
@@ -215,7 +232,7 @@ def program_run(
         job = job_cls(job_id=job_id, program_id=program_id,
                       hub=credentials.hub, group=credentials.group,
                       project=credentials.project, backend_name=backend_name,
-                      params=params)
+                      params=params, **self._job_kwargs)
         self._jobs[job_id] = job
         return {'id': job_id}
 
@@ -229,9 +246,13 @@ def job_get(self, job_id):
         """Get the specific job."""
         return self._get_job(job_id).to_dict()
 
-    def jobs_get(self):
+    def jobs_get(self, limit=None, skip=None):
         """Get all jobs."""
-        return [job.to_dict() for job in self._jobs]
+        limit = limit or len(self._jobs)
+        skip = skip or 0
+        jobs = list(self._jobs.values())[skip:limit+skip]
+        return {"jobs": [job.to_dict() for job in jobs],
+                "count": len(self._jobs)}
 
     def job_results(self, job_id):
         """Get the results of a program job."""
diff --git a/test/ibmq/runtime/test_runtime.py b/test/ibmq/runtime/test_runtime.py
index a39f7ee6d..63b2491a9 100644
--- a/test/ibmq/runtime/test_runtime.py
+++ b/test/ibmq/runtime/test_runtime.py
@@ -176,6 +176,47 @@ def test_retrieve_job(self):
         self.assertEqual(job.job_id(), rjob.job_id())
         self.assertEqual(program_id, rjob.program_id)
 
+    def test_jobs_no_limit(self):
+        """Test retrieving jobs without limit."""
+        jobs = []
+        program_id = self._upload_program()
+        for _ in range(25):
+            jobs.append(self._run_program(program_id))
+        rjobs = self.runtime.jobs(limit=None)
+        self.assertEqual(25, len(rjobs))
+
+    def test_jobs_limit(self):
+        """Test retrieving jobs with limit."""
+        jobs = []
+        job_count = 25
+        program_id = self._upload_program()
+        for _ in range(job_count):
+            jobs.append(self._run_program(program_id))
+
+        limits = [21, 30]
+        for limit in limits:
+            with self.subTest(limit=limit):
+                rjobs = self.runtime.jobs(limit=limit)
+                self.assertEqual(min(limit, job_count), len(rjobs))
+
+    def test_jobs_skip(self):
+        """Test retrieving jobs with skip."""
+        jobs = []
+        program_id = self._upload_program()
+        for _ in range(5):
+            jobs.append(self._run_program(program_id))
+        rjobs = self.runtime.jobs(skip=4)
+        self.assertEqual(1, len(rjobs))
+
+    def test_jobs_skip_limit(self):
+        """Test retrieving jobs with skip and limit."""
+        jobs = []
+        program_id = self._upload_program()
+        for _ in range(10):
+            jobs.append(self._run_program(program_id))
+        rjobs = self.runtime.jobs(skip=4, limit=2)
+        self.assertEqual(2, len(rjobs))
+
     def test_cancel_job(self):
         """Test canceling a job."""
         job = self._run_program(job_classes=CancelableRuntimeJob)
diff --git a/test/ibmq/runtime/test_runtime_integration.py b/test/ibmq/runtime/test_runtime_integration.py
index 208c7e3d6..6fb0ab295 100644
--- a/test/ibmq/runtime/test_runtime_integration.py
+++ b/test/ibmq/runtime/test_runtime_integration.py
@@ -239,6 +239,18 @@ def test_retrieve_all_jobs(self):
                 break
         self.assertTrue(found, f"Job {job.job_id()} not returned.")
 
+    def test_retrieve_jobs_limit(self):
+        """Test retrieving jobs with limit."""
+        jobs = []
+        for _ in range(3):
+            jobs.append(self._run_program())
+
+        rjobs = self.provider.runtime.jobs(limit=2)
+        self.assertEqual(len(rjobs), 2)
+        job_ids = {job.job_id() for job in jobs}
+        rjob_ids = {rjob.job_id() for rjob in rjobs}
+        self.assertTrue(rjob_ids.issubset(job_ids))
+
     def test_cancel_job_queued(self):
         """Test canceling a queued job."""
         _ = self._run_program(iterations=10)
@@ -368,7 +380,6 @@ def result_callback(job_id, interim_result):
         self.assertEqual(iterations-1, final_it)
         self.assertIsNone(job._ws_client._ws)
 
-    @unittest.skip("Skip until 277 is fixed")
     def test_callback_cancel_job(self):
         """Test canceling a running job while streaming results."""
         def result_callback(job_id, interim_result):
@@ -457,6 +468,8 @@ def _validate_program(self, program):
         self.assertTrue(program.program_id)
         self.assertTrue(program.description)
         self.assertTrue(program.max_execution_time)
+        self.assertTrue(program.creation_date)
+        self.assertTrue(program.version)
 
     def _upload_program(self, name=None, max_execution_time=300):
         """Upload a new program."""

From 6ed5394bcc6b1f5948e14ac113eb73127b975b83 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Wed, 5 May 2021 09:42:52 -0400
Subject: [PATCH 46/59] use qobj

---
 qiskit/providers/ibmq/runtime/utils.py |  10 ++
 test/ibmq/runtime/test_runtime_ws.py   | 139 +++++++++++++++++++++++++
 test/ibmq/runtime/websocket_server.py  |  75 +++++++++++++
 3 files changed, 224 insertions(+)
 create mode 100644 test/ibmq/runtime/test_runtime_ws.py
 create mode 100644 test/ibmq/runtime/websocket_server.py

diff --git a/qiskit/providers/ibmq/runtime/utils.py b/qiskit/providers/ibmq/runtime/utils.py
index 5e1cecc5d..f8370c5d6 100644
--- a/qiskit/providers/ibmq/runtime/utils.py
+++ b/qiskit/providers/ibmq/runtime/utils.py
@@ -16,11 +16,14 @@
 import json
 from typing import Any
 import base64
+from contextlib import suppress
 
 import dill
 import numpy as np
 
 from qiskit.result import Result
+from qiskit import assemble, QuantumCircuit
+from qiskit.assembler.disassemble import disassemble
 
 
 class RuntimeEncoder(json.JSONEncoder):
@@ -31,6 +34,10 @@ def default(self, obj: Any) -> Any:  # pylint: disable=arguments-differ
             return {'__type__': 'array', '__value__': obj.tolist()}
         if isinstance(obj, complex):
             return {'__type__': 'complex', '__value__': [obj.real, obj.imag]}
+        if isinstance(obj, QuantumCircuit):
+            with suppress(Exception):  # Cannot assemble circuits with unbound params.
+                qobj = assemble(obj)
+                return {'__type__': 'qobj', '__value__': qobj.to_dict()}
         if isinstance(obj, Result):
             return {'__type__': 'result', '__value__': obj.to_dict()}
         if hasattr(obj, 'to_json'):
@@ -56,6 +63,9 @@ def object_hook(self, obj: Any) -> Any:
                 return val[0] + 1j * val[1]
             if obj['__type__'] == 'array':
                 return np.array(obj['__value__'])
+            if obj['__type__'] == ['qobj']:
+                circuits, _, _ = disassemble(obj['__value__'])
+                return circuits
             if obj['__type__'] == 'result':
                 return Result.from_dict(obj['__value__'])
             if obj['__type__'] == 'to_json':
diff --git a/test/ibmq/runtime/test_runtime_ws.py b/test/ibmq/runtime/test_runtime_ws.py
new file mode 100644
index 000000000..507197316
--- /dev/null
+++ b/test/ibmq/runtime/test_runtime_ws.py
@@ -0,0 +1,139 @@
+# This code is part of Qiskit.
+#
+# (C) Copyright IBM 2018, 2019.
+#
+# This code is licensed under the Apache License, Version 2.0. You may
+# obtain a copy of this license in the LICENSE.txt file in the root directory
+# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Any modifications or derivative works of this code must retain this
+# copyright notice, and modified files need to carry a notice indicating
+# that they have been altered from the originals.
+
+"""Test for the Websocket client."""
+
+import sys
+import asyncio
+import warnings
+from contextlib import suppress
+import queue
+import time
+from concurrent.futures import ThreadPoolExecutor
+import threading
+
+import websockets
+
+from qiskit.providers.ibmq.api.clients.runtime_ws import RuntimeWebsocketClient
+from qiskit.providers.ibmq.runtime import RuntimeJob
+from qiskit.test.mock.fake_qasm_simulator import FakeQasmSimulator
+
+from ...ibmqtestcase import IBMQTestCase
+from .websocket_server import (websocket_handler, JOB_ID_PROGRESS_DONE, JOB_ID_ALREADY_DONE,
+                               JOB_ID_RETRY_SUCCESS, JOB_ID_RETRY_FAILURE, JOB_ID_RANDOM_CODE,
+                               JOB_PROGRESS_RESULT_COUNT)
+from .fake_runtime_client import BaseFakeRuntimeClient, TimedRuntimeJob
+
+
+class TestRuntimeWebsocketClient(IBMQTestCase):
+    """Tests for the the websocket client against a mock server."""
+
+    TEST_IP_ADDRESS = '127.0.0.1'
+    INVALID_PORT = 9876
+    VALID_PORT = 8765
+    VALID_URL = f"ws://{TEST_IP_ADDRESS}:{VALID_PORT}"
+
+    _executor = ThreadPoolExecutor()
+
+    @classmethod
+    def setUpClass(cls):
+        """Initial class level setup."""
+        super().setUpClass()
+
+        # Launch the mock server.
+        # start_server = websockets.serve(websocket_handler, cls.TEST_IP_ADDRESS, cls.VALID_PORT)
+        # cls.server = asyncio.get_event_loop().run_until_complete(start_server)
+        cls._ws_event = threading.Event()
+
+    @classmethod
+    def _ws_server(cls):
+        loop = asyncio.get_event_loop()
+        start_server = websockets.serve(websocket_handler, cls.TEST_IP_ADDRESS, cls.VALID_PORT)
+        cls.server = asyncio.get_event_loop().run_until_complete(start_server)
+        cls._ws_event.wait(timeout=10)
+
+
+    @classmethod
+    def tearDownClass(cls):
+        """Class level cleanup."""
+        super().tearDownClass()
+
+        # Close the mock server.
+        loop = asyncio.get_event_loop()
+        cls.server.close()
+        loop.run_until_complete(cls.server.wait_closed())
+
+        with warnings.catch_warnings():
+            # Suppress websockets deprecation warning
+            warnings.filterwarnings("ignore", category=PendingDeprecationWarning)
+            warnings.filterwarnings("ignore", category=DeprecationWarning)
+            # Manually cancel any pending asyncio tasks.
+            if sys.version_info[0:2] < (3, 9):
+                pending = asyncio.Task.all_tasks()
+            else:
+                pending = asyncio.all_tasks(loop)
+        for task in pending:
+            task.cancel()
+            try:
+                with suppress(asyncio.CancelledError):
+                    loop.run_until_complete(task)
+            except Exception as err:  # pylint: disable=broad-except
+                cls.log.error("An error %s occurred canceling task %s. "
+                              "Traceback:", str(err), str(task))
+                task.print_stack()
+
+    def test_interim_result_callback(self):
+        """Test interim result callback."""
+        def result_callback(job_id, interim_result):
+            print(f">>>>> callback called")
+            nonlocal results
+            results.append(interim_result)
+            self.assertEqual(JOB_ID_PROGRESS_DONE, job_id)
+
+        results = []
+        # ws = RuntimeWebsocketClient(self.VALID_URL, "my_token")
+        ws = RuntimeWebsocketClient('ws://{}:{}'.format(
+            self.TEST_IP_ADDRESS, self.VALID_PORT), "foo")
+        # api = BaseFakeRuntimeClient(job_classes=TimedRuntimeJob,
+        #                             job_kwargs={"run_time": JOB_PROGRESS_RESULT_COUNT+2})
+        job = RuntimeJob(backend=FakeQasmSimulator(),
+                         api_client=BaseFakeRuntimeClient(),
+                         ws_client=ws,
+                         job_id=JOB_ID_PROGRESS_DONE,
+                         program_id="my-program",
+                         user_callback=result_callback)
+        time.sleep(JOB_PROGRESS_RESULT_COUNT+2)
+        self.assertEqual(JOB_PROGRESS_RESULT_COUNT, len(results))
+        self.assertIsNone(job._ws_client._ws)
+
+    def test_stream_results(self):
+        pass
+
+    def test_cancel_streaming(self):
+        pass
+
+    def test_completed_job(self):
+        pass
+
+    def test_websocket_retry_success(self):
+        pass
+
+    def test_websocket_retry_failure(self):
+        pass
+
+    def test_job_interim_results(self):
+        """Test retrieving a job already in final status."""
+        client = RuntimeWebsocketClient('ws://{}:{}'.format(
+            self.TEST_IP_ADDRESS, self.VALID_PORT), "foo")
+
+        asyncio.get_event_loop().run_until_complete(
+            client.job_results(JOB_ID_PROGRESS_DONE, queue.Queue()))
diff --git a/test/ibmq/runtime/websocket_server.py b/test/ibmq/runtime/websocket_server.py
new file mode 100644
index 000000000..f3bfc9f36
--- /dev/null
+++ b/test/ibmq/runtime/websocket_server.py
@@ -0,0 +1,75 @@
+# This code is part of Qiskit.
+#
+# (C) Copyright IBM 2021.
+#
+# This code is licensed under the Apache License, Version 2.0. You may
+# obtain a copy of this license in the LICENSE.txt file in the root directory
+# of this source tree or at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# Any modifications or derivative works of this code must retain this
+# copyright notice, and modified files need to carry a notice indicating
+# that they have been altered from the originals.
+
+"""Websocket server for testing purposes."""
+
+import asyncio
+
+JOB_ID_PROGRESS_DONE = 'JOB_ID_PROGRESS_DONE'
+JOB_ID_ALREADY_DONE = 'JOB_ID_ALREADY_DONE'
+JOB_ID_RETRY_SUCCESS = 'JOB_ID_RETRY_SUCCESS'
+JOB_ID_RETRY_FAILURE = 'JOB_ID_RETRY_FAILURE'
+JOB_ID_RANDOM_CODE = 'JOB_ID_RANDOM_CODE'
+JOB_PROGRESS_RESULT_COUNT = 5
+
+
+async def websocket_handler(websocket, path):
+    """Entry point for the websocket mock server."""
+    print(">>>>> handler called")
+    request = path.split('/')[-1]
+    await websocket.send("ACK")
+
+    if request == JOB_ID_PROGRESS_DONE:
+        await handle_job_progress_done(websocket)
+    elif request == JOB_ID_ALREADY_DONE:
+        await handle_job_already_done(websocket)
+    elif request == JOB_ID_RETRY_SUCCESS:
+        await handle_token_retry_success(websocket)
+    elif request == JOB_ID_RETRY_FAILURE:
+        await handle_token_retry_failure(websocket)
+    elif request == JOB_ID_RANDOM_CODE:
+        await handle_close_random_code(websocket)
+    else:
+        raise ValueError(f"Unknown request {request}")
+
+
+async def handle_job_progress_done(websocket):
+    """Send a few results then close with 1000."""
+    for idx in range(JOB_PROGRESS_RESULT_COUNT):
+        print(f">>>>> adding result")
+        await websocket.send(f"foo{idx}".encode())
+        await asyncio.sleep(1)
+    await websocket.close(code=1000)
+
+
+async def handle_job_already_done(websocket):
+    """Close immediately with 1000."""
+    await websocket.close(code=1000)
+
+
+async def handle_token_retry_success(websocket):
+    """Close the socket once and force a retry."""
+    if not hasattr(handle_token_retry_success, 'retry_attempt'):
+        setattr(handle_token_retry_success, 'retry_attempt', True)
+        await handle_token_retry_failure(websocket)
+    else:
+        await handle_job_progress_done(websocket)
+
+
+async def handle_token_retry_failure(websocket):
+    """Continually close the socket, until both the first attempt and retry fail."""
+    await websocket.close()
+
+
+async def handle_close_random_code(websocket):
+    """Close with a random code."""
+    await websocket.close(code=1234)

From c9a5035a4b16777e5d970c8d640cdf5b8dae5472 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Wed, 5 May 2021 09:44:36 -0400
Subject: [PATCH 47/59] remove unused code

---
 test/ibmq/runtime/test_runtime_ws.py  | 8 +-------
 test/ibmq/runtime/websocket_server.py | 2 --
 2 files changed, 1 insertion(+), 9 deletions(-)

diff --git a/test/ibmq/runtime/test_runtime_ws.py b/test/ibmq/runtime/test_runtime_ws.py
index 507197316..93f1b27f4 100644
--- a/test/ibmq/runtime/test_runtime_ws.py
+++ b/test/ibmq/runtime/test_runtime_ws.py
@@ -61,7 +61,6 @@ def _ws_server(cls):
         cls.server = asyncio.get_event_loop().run_until_complete(start_server)
         cls._ws_event.wait(timeout=10)
 
-
     @classmethod
     def tearDownClass(cls):
         """Class level cleanup."""
@@ -94,17 +93,12 @@ def tearDownClass(cls):
     def test_interim_result_callback(self):
         """Test interim result callback."""
         def result_callback(job_id, interim_result):
-            print(f">>>>> callback called")
             nonlocal results
             results.append(interim_result)
             self.assertEqual(JOB_ID_PROGRESS_DONE, job_id)
 
         results = []
-        # ws = RuntimeWebsocketClient(self.VALID_URL, "my_token")
-        ws = RuntimeWebsocketClient('ws://{}:{}'.format(
-            self.TEST_IP_ADDRESS, self.VALID_PORT), "foo")
-        # api = BaseFakeRuntimeClient(job_classes=TimedRuntimeJob,
-        #                             job_kwargs={"run_time": JOB_PROGRESS_RESULT_COUNT+2})
+        ws = RuntimeWebsocketClient(self.VALID_URL, "my_token")
         job = RuntimeJob(backend=FakeQasmSimulator(),
                          api_client=BaseFakeRuntimeClient(),
                          ws_client=ws,
diff --git a/test/ibmq/runtime/websocket_server.py b/test/ibmq/runtime/websocket_server.py
index f3bfc9f36..89da4ebcb 100644
--- a/test/ibmq/runtime/websocket_server.py
+++ b/test/ibmq/runtime/websocket_server.py
@@ -24,7 +24,6 @@
 
 async def websocket_handler(websocket, path):
     """Entry point for the websocket mock server."""
-    print(">>>>> handler called")
     request = path.split('/')[-1]
     await websocket.send("ACK")
 
@@ -45,7 +44,6 @@ async def websocket_handler(websocket, path):
 async def handle_job_progress_done(websocket):
     """Send a few results then close with 1000."""
     for idx in range(JOB_PROGRESS_RESULT_COUNT):
-        print(f">>>>> adding result")
         await websocket.send(f"foo{idx}".encode())
         await asyncio.sleep(1)
     await websocket.close(code=1000)

From e6150a8c216715d35f461921edeb3508f522d7db Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Wed, 5 May 2021 12:31:12 -0400
Subject: [PATCH 48/59] fix typo

---
 qiskit/providers/ibmq/runtime/utils.py        | 6 ++++--
 test/ibmq/runtime/test_runtime_integration.py | 2 --
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/qiskit/providers/ibmq/runtime/utils.py b/qiskit/providers/ibmq/runtime/utils.py
index f8370c5d6..5f24a747c 100644
--- a/qiskit/providers/ibmq/runtime/utils.py
+++ b/qiskit/providers/ibmq/runtime/utils.py
@@ -24,6 +24,7 @@
 from qiskit.result import Result
 from qiskit import assemble, QuantumCircuit
 from qiskit.assembler.disassemble import disassemble
+from qiskit.qobj import QasmQobj
 
 
 class RuntimeEncoder(json.JSONEncoder):
@@ -63,8 +64,9 @@ def object_hook(self, obj: Any) -> Any:
                 return val[0] + 1j * val[1]
             if obj['__type__'] == 'array':
                 return np.array(obj['__value__'])
-            if obj['__type__'] == ['qobj']:
-                circuits, _, _ = disassemble(obj['__value__'])
+            if obj['__type__'] == 'qobj':
+                qobj = QasmQobj.from_dict(obj['__value__'])
+                circuits, _, _ = disassemble(qobj)
                 return circuits
             if obj['__type__'] == 'result':
                 return Result.from_dict(obj['__value__'])
diff --git a/test/ibmq/runtime/test_runtime_integration.py b/test/ibmq/runtime/test_runtime_integration.py
index 6fb0ab295..f0bcd801b 100644
--- a/test/ibmq/runtime/test_runtime_integration.py
+++ b/test/ibmq/runtime/test_runtime_integration.py
@@ -32,8 +32,6 @@
 from .utils import SerializableClass, SerializableClassDecoder, get_complex_types
 
 
-os.environ['USE_STAGING_CREDENTIALS'] = "true"
-
 @unittest.skipIf(not os.environ.get('USE_STAGING_CREDENTIALS', ''), "Only runs on staging")
 class TestRuntimeIntegration(IBMQTestCase):
     """Integration tests for runtime modules."""

From aa4a1943c96c0c8b4b91db2a78577b5a1d447207 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Wed, 5 May 2021 14:06:56 -0400
Subject: [PATCH 49/59] remove qobj

---
 qiskit/providers/ibmq/runtime/utils.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/qiskit/providers/ibmq/runtime/utils.py b/qiskit/providers/ibmq/runtime/utils.py
index 5f24a747c..9fba5396e 100644
--- a/qiskit/providers/ibmq/runtime/utils.py
+++ b/qiskit/providers/ibmq/runtime/utils.py
@@ -35,10 +35,6 @@ def default(self, obj: Any) -> Any:  # pylint: disable=arguments-differ
             return {'__type__': 'array', '__value__': obj.tolist()}
         if isinstance(obj, complex):
             return {'__type__': 'complex', '__value__': [obj.real, obj.imag]}
-        if isinstance(obj, QuantumCircuit):
-            with suppress(Exception):  # Cannot assemble circuits with unbound params.
-                qobj = assemble(obj)
-                return {'__type__': 'qobj', '__value__': qobj.to_dict()}
         if isinstance(obj, Result):
             return {'__type__': 'result', '__value__': obj.to_dict()}
         if hasattr(obj, 'to_json'):
@@ -64,10 +60,6 @@ def object_hook(self, obj: Any) -> Any:
                 return val[0] + 1j * val[1]
             if obj['__type__'] == 'array':
                 return np.array(obj['__value__'])
-            if obj['__type__'] == 'qobj':
-                qobj = QasmQobj.from_dict(obj['__value__'])
-                circuits, _, _ = disassemble(qobj)
-                return circuits
             if obj['__type__'] == 'result':
                 return Result.from_dict(obj['__value__'])
             if obj['__type__'] == 'to_json':

From 9616b25f78e6cb7bdd6a00a11d475a6f09854c89 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Thu, 6 May 2021 10:23:50 -0400
Subject: [PATCH 50/59] add ws tests

---
 qiskit/providers/ibmq/api/rest/runtime.py     |   2 +-
 qiskit/providers/ibmq/runtime/__init__.py     |  19 +-
 qiskit/providers/ibmq/runtime/constants.py    |   1 -
 qiskit/providers/ibmq/runtime/runtime_job.py  |   7 +-
 qiskit/providers/ibmq/runtime/utils.py        |   4 -
 .../notes/runtime-f9a57a8286fa6197.yaml       |  32 ++--
 test/ibmq/runtime/test_runtime.py             |  10 +-
 test/ibmq/runtime/test_runtime_ws.py          | 165 ++++++++++++++----
 test/ibmq/runtime/utils.py                    |   3 +
 test/ibmq/runtime/websocket_server.py         |  12 +-
 10 files changed, 182 insertions(+), 73 deletions(-)

diff --git a/qiskit/providers/ibmq/api/rest/runtime.py b/qiskit/providers/ibmq/api/rest/runtime.py
index 2ececf8c7..f46c93635 100644
--- a/qiskit/providers/ibmq/api/rest/runtime.py
+++ b/qiskit/providers/ibmq/api/rest/runtime.py
@@ -112,7 +112,7 @@ def create_program(
                 files = {'program': (name, file)}
                 response = self.session.post(url, data=data, files=files).json()
         else:
-            files = {'program': (name, program_data)}
+            files = {'program': (name, program_data)}  # type: ignore[dict-item]
             response = self.session.post(url, data=data, files=files).json()
         return response
 
diff --git a/qiskit/providers/ibmq/runtime/__init__.py b/qiskit/providers/ibmq/runtime/__init__.py
index 1108f7cb1..f5f82452b 100644
--- a/qiskit/providers/ibmq/runtime/__init__.py
+++ b/qiskit/providers/ibmq/runtime/__init__.py
@@ -37,15 +37,24 @@
     `Qiskit-Partners/qiskit-runtime
     <https://github.com/Qiskit-Partners/qiskit-runtime>`_ for more information.
 
+.. caution::
+
+  This package is currently provided in beta form and heavy modifications to
+  both functionality and API are likely to occur. Backward compatibility is not
+  always guaranteed.
+
+Qiskit Runtime is a new architecture offered by IBM Quantum that
+significantly reduces waiting time during computational iterations.
+You can execute your experiments near the quantum hardware, without
+the interactions of multiple layers of classical and quantum hardware
+slowing it down.
+
 The Qiskit Runtime Service allows authorized users to upload their Qiskit quantum programs.
-A Qiskit quantum program is a piece of Python code that takes certain inputs, performs
+A Qiskit quantum program, also called a runtime program, is a piece of Python
+code that takes certain inputs, performs
 quantum and classical processing, and returns the results. The same or other
 authorized users can invoke these quantum programs by simply passing in parameters.
 
-These Qiskit quantum programs, sometimes called runtime programs, run in a special
-runtime environment that significantly reduces waiting time during computational
-iterations.
-
 `Qiskit-Partners/qiskit-runtime <https://github.com/Qiskit-Partners/qiskit-runtime>`_
 contains detailed tutorials on how to use the runtime service.
 
diff --git a/qiskit/providers/ibmq/runtime/constants.py b/qiskit/providers/ibmq/runtime/constants.py
index 43c825dc9..6cea93480 100644
--- a/qiskit/providers/ibmq/runtime/constants.py
+++ b/qiskit/providers/ibmq/runtime/constants.py
@@ -19,7 +19,6 @@
     'QUEUED': JobStatus.QUEUED,
     'RUNNING': JobStatus.RUNNING,
     'COMPLETED': JobStatus.DONE,
-    'SUCCEEDED': JobStatus.DONE,    # TODO remove when no longer used
     'FAILED': JobStatus.ERROR,
     'CANCELLED': JobStatus.CANCELLED
 }
diff --git a/qiskit/providers/ibmq/runtime/runtime_job.py b/qiskit/providers/ibmq/runtime/runtime_job.py
index c3824a4aa..a16de95b6 100644
--- a/qiskit/providers/ibmq/runtime/runtime_job.py
+++ b/qiskit/providers/ibmq/runtime/runtime_job.py
@@ -26,7 +26,7 @@
 from qiskit.providers.ibmq import ibmqbackend  # pylint: disable=unused-import
 
 from .constants import API_TO_JOB_STATUS
-from .exceptions import RuntimeJobFailureError, RuntimeInvalidStateError
+from .exceptions import RuntimeJobFailureError, RuntimeInvalidStateError, QiskitRuntimeError
 from .result_decoder import ResultDecoder
 from ..api.clients import RuntimeClient, RuntimeWebsocketClient
 from ..exceptions import IBMQError
@@ -139,7 +139,7 @@ def result(
             RuntimeJobFailureError: If the job failed.
         """
         _decoder = decoder or self._result_decoder
-        if not self._results or _decoder != self._result_decoder:
+        if not self._results or (_decoder != self._result_decoder):  # type: ignore[unreachable]
             self.wait_for_final_state(timeout=timeout, wait=wait)
             result_raw = self._api_client.job_results(job_id=self.job_id())
             if self._status == JobStatus.ERROR:
@@ -153,13 +153,14 @@ def cancel(self) -> None:
 
         Raises:
             RuntimeInvalidStateError: If the job is in a state that cannot be cancelled.
+            QiskitRuntimeError: If unable to cancel job.
         """
         try:
             self._api_client.job_cancel(self.job_id())
         except RequestsApiError as ex:
             if ex.status_code == 409:
                 raise RuntimeInvalidStateError(f"Job cannot be cancelled: {ex}") from None
-            raise
+            raise QiskitRuntimeError(f"Failed to cancel job: {ex}") from None
         self.cancel_result_streaming()
         self._status = JobStatus.CANCELLED
 
diff --git a/qiskit/providers/ibmq/runtime/utils.py b/qiskit/providers/ibmq/runtime/utils.py
index 9fba5396e..5e1cecc5d 100644
--- a/qiskit/providers/ibmq/runtime/utils.py
+++ b/qiskit/providers/ibmq/runtime/utils.py
@@ -16,15 +16,11 @@
 import json
 from typing import Any
 import base64
-from contextlib import suppress
 
 import dill
 import numpy as np
 
 from qiskit.result import Result
-from qiskit import assemble, QuantumCircuit
-from qiskit.assembler.disassemble import disassemble
-from qiskit.qobj import QasmQobj
 
 
 class RuntimeEncoder(json.JSONEncoder):
diff --git a/releasenotes/notes/runtime-f9a57a8286fa6197.yaml b/releasenotes/notes/runtime-f9a57a8286fa6197.yaml
index cf6320168..266bd065c 100644
--- a/releasenotes/notes/runtime-f9a57a8286fa6197.yaml
+++ b/releasenotes/notes/runtime-f9a57a8286fa6197.yaml
@@ -1,18 +1,30 @@
 ---
 prelude: >
     This release introduces a new feature ``Qiskit Runtime Service``.
-    This new service allows authorized users to upload quantum programs that
-    can be invoked by others. These quantum programs run in a special
-    environment that significantly reduces waiting time during computational
-    iterations.
+    Qiskit Runtime is a new architecture offered by IBM Quantum that
+    significantly reduces waiting time during computational iterations.
+    You can execute your experiments near the quantum hardware, without
+    the interactions of multiple layers of classical and quantum hardware
+    slowing it down.
+
+    Qiskit Runtime allows authorized users to upload their Qiskit quantum
+    programs, which are Python code that takes
+    certain inputs, performs quantum and maybe classical computation, and returns
+    the processing results. The same or other authorized users can then invoke
+    these quantum programs by simply passing in the required input parameters.
+
+    Qiskit Runtime is currently in private beta for select account but will be
+    released to the public in the near future.
 features:
   - |
-    This release introduces a new feature ``Qiskit Runtime Service``. This
-    new service allows authorized users to upload quantum programs. Other
-    authorized users can invoke these quantum programs by simply passing
-    in parameters. These quantum programs run in a speical runtime
-    environment that significantly reduces waiting time during computational
-    iterations.
+    This release introduces a new feature ``Qiskit Runtime Service``.
+    Qiskit Runtime is a new architecture that
+    significantly reduces waiting time during computational iterations.
+    This new service allows authorized users to upload their Qiskit quantum
+    programs, which are Python code that takes
+    certain inputs, performs quantum and maybe classical computation, and returns
+    the processing results. The same or other authorized users can then invoke
+    these quantum programs by simply passing in the required input parameters.
 
     An example of using this new service::
 
diff --git a/test/ibmq/runtime/test_runtime.py b/test/ibmq/runtime/test_runtime.py
index 63b2491a9..063de83f9 100644
--- a/test/ibmq/runtime/test_runtime.py
+++ b/test/ibmq/runtime/test_runtime.py
@@ -271,12 +271,12 @@ def test_result_decoder(self):
 
     def test_program_metadata(self):
         """Test program metadata."""
-        fn = "test_metadata.json"
-        with open(fn, 'w') as file:
+        file_name = "test_metadata.json"
+        with open(file_name, 'w') as file:
             json.dump(self.DEFAULT_METADATA, file)
-        self.addCleanup(os.remove, fn)
+        self.addCleanup(os.remove, file_name)
 
-        sub_tests = [fn, self.DEFAULT_METADATA]
+        sub_tests = [file_name, self.DEFAULT_METADATA]
 
         for metadata in sub_tests:
             with self.subTest(metadata_type=type(metadata)):
@@ -343,5 +343,3 @@ def _run_program(self, program_id=None, inputs=None, job_classes=None, decoder=N
         job = self.runtime.run(program_id=program_id, inputs=inputs,
                                options=options, result_decoder=decoder)
         return job
-
-    # TODO add websocket tests
diff --git a/test/ibmq/runtime/test_runtime_ws.py b/test/ibmq/runtime/test_runtime_ws.py
index 93f1b27f4..126adfb21 100644
--- a/test/ibmq/runtime/test_runtime_ws.py
+++ b/test/ibmq/runtime/test_runtime_ws.py
@@ -16,7 +16,6 @@
 import asyncio
 import warnings
 from contextlib import suppress
-import queue
 import time
 from concurrent.futures import ThreadPoolExecutor
 import threading
@@ -25,13 +24,14 @@
 
 from qiskit.providers.ibmq.api.clients.runtime_ws import RuntimeWebsocketClient
 from qiskit.providers.ibmq.runtime import RuntimeJob
+from qiskit.providers.ibmq.runtime.exceptions import RuntimeInvalidStateError
 from qiskit.test.mock.fake_qasm_simulator import FakeQasmSimulator
 
 from ...ibmqtestcase import IBMQTestCase
 from .websocket_server import (websocket_handler, JOB_ID_PROGRESS_DONE, JOB_ID_ALREADY_DONE,
-                               JOB_ID_RETRY_SUCCESS, JOB_ID_RETRY_FAILURE, JOB_ID_RANDOM_CODE,
+                               JOB_ID_RETRY_SUCCESS, JOB_ID_RETRY_FAILURE,
                                JOB_PROGRESS_RESULT_COUNT)
-from .fake_runtime_client import BaseFakeRuntimeClient, TimedRuntimeJob
+from .fake_runtime_client import BaseFakeRuntimeClient
 
 
 class TestRuntimeWebsocketClient(IBMQTestCase):
@@ -50,24 +50,25 @@ def setUpClass(cls):
         super().setUpClass()
 
         # Launch the mock server.
-        # start_server = websockets.serve(websocket_handler, cls.TEST_IP_ADDRESS, cls.VALID_PORT)
-        # cls.server = asyncio.get_event_loop().run_until_complete(start_server)
-        cls._ws_event = threading.Event()
+        cls._ws_stop_event = threading.Event()
+        cls._ws_start_event = threading.Event()
+        cls._future = cls._executor.submit(cls._ws_server)
+        cls._ws_start_event.wait(5)
 
     @classmethod
     def _ws_server(cls):
-        loop = asyncio.get_event_loop()
+        loop = asyncio.new_event_loop()
+        asyncio.set_event_loop(loop)
+        # Launch the mock server.
         start_server = websockets.serve(websocket_handler, cls.TEST_IP_ADDRESS, cls.VALID_PORT)
-        cls.server = asyncio.get_event_loop().run_until_complete(start_server)
-        cls._ws_event.wait(timeout=10)
+        cls.server = loop.run_until_complete(start_server)
+        cls._ws_start_event.set()
 
-    @classmethod
-    def tearDownClass(cls):
-        """Class level cleanup."""
-        super().tearDownClass()
+        # A bit hacky but we need to keep the loop running to serve the request.
+        # An there's no easy way to interrupt a run_forever.
+        while not cls._ws_stop_event.is_set():
+            loop.run_until_complete(asyncio.sleep(1))
 
-        # Close the mock server.
-        loop = asyncio.get_event_loop()
         cls.server.close()
         loop.run_until_complete(cls.server.wait_closed())
 
@@ -90,6 +91,15 @@ def tearDownClass(cls):
                               "Traceback:", str(err), str(task))
                 task.print_stack()
 
+    @classmethod
+    def tearDownClass(cls):
+        """Class level cleanup."""
+        super().tearDownClass()
+
+        # Close the mock server.
+        cls._ws_stop_event.set()
+        cls._future.result()
+
     def test_interim_result_callback(self):
         """Test interim result callback."""
         def result_callback(job_id, interim_result):
@@ -98,36 +108,125 @@ def result_callback(job_id, interim_result):
             self.assertEqual(JOB_ID_PROGRESS_DONE, job_id)
 
         results = []
-        ws = RuntimeWebsocketClient(self.VALID_URL, "my_token")
-        job = RuntimeJob(backend=FakeQasmSimulator(),
-                         api_client=BaseFakeRuntimeClient(),
-                         ws_client=ws,
-                         job_id=JOB_ID_PROGRESS_DONE,
-                         program_id="my-program",
-                         user_callback=result_callback)
+        job = self._get_job(callback=result_callback)
         time.sleep(JOB_PROGRESS_RESULT_COUNT+2)
         self.assertEqual(JOB_PROGRESS_RESULT_COUNT, len(results))
         self.assertIsNone(job._ws_client._ws)
 
     def test_stream_results(self):
-        pass
+        """Test streaming results."""
+        def result_callback(job_id, interim_result):
+            nonlocal results
+            results.append(interim_result)
+            self.assertEqual(JOB_ID_PROGRESS_DONE, job_id)
+
+        results = []
+        job = self._get_job()
+        job.stream_results(callback=result_callback)
+        time.sleep(JOB_PROGRESS_RESULT_COUNT+2)
+        self.assertEqual(JOB_PROGRESS_RESULT_COUNT, len(results))
+        self.assertIsNone(job._ws_client._ws)
+
+    def test_duplicate_streaming(self):
+        """Testing duplicate streaming."""
+        def result_callback(job_id, interim_result):
+            nonlocal results
+            results.append(interim_result)
+            self.assertEqual(JOB_ID_PROGRESS_DONE, job_id)
+
+        results = []
+        job = self._get_job(callback=result_callback)
+        time.sleep(1)
+        with self.assertRaises(RuntimeInvalidStateError):
+            job.stream_results(callback=result_callback)
 
     def test_cancel_streaming(self):
-        pass
+        """Test canceling streaming."""
+        def result_callback(job_id, interim_result):
+            nonlocal results
+            results.append(interim_result)
+            self.assertEqual(JOB_ID_PROGRESS_DONE, job_id)
+
+        results = []
+        job = self._get_job(callback=result_callback)
+        time.sleep(1)
+        job.cancel_result_streaming()
+        time.sleep(1)
+        self.assertIsNone(job._ws_client._ws)
+
+    def test_cancel_closed_streaming(self):
+        """Test canceling streaming that's already closed."""
+        def result_callback(job_id, interim_result):
+            nonlocal results
+            results.append(interim_result)
+            self.assertEqual(JOB_ID_ALREADY_DONE, job_id)
+
+        results = []
+        job = self._get_job(callback=result_callback, job_id=JOB_ID_ALREADY_DONE)
+        time.sleep(2)
+        job.cancel_result_streaming()
+        self.assertIsNone(job._ws_client._ws)
 
     def test_completed_job(self):
-        pass
+        """Test callback from completed job."""
+        def result_callback(job_id, interim_result):
+            nonlocal results
+            results.append(interim_result)
+            self.assertEqual(JOB_ID_ALREADY_DONE, job_id)
+
+        results = []
+        job = self._get_job(callback=result_callback, job_id=JOB_ID_ALREADY_DONE)
+        time.sleep(2)
+        self.assertEqual(0, len(results))
+        self.assertIsNone(job._ws_client._ws)
+
+    def test_completed_job_stream(self):
+        """Test streaming from completed job."""
+        def result_callback(job_id, interim_result):
+            nonlocal results
+            results.append(interim_result)
+            self.assertEqual(JOB_ID_ALREADY_DONE, job_id)
+
+        results = []
+        job = self._get_job(job_id=JOB_ID_ALREADY_DONE)
+        job.stream_results(callback=result_callback)
+        time.sleep(2)
+        self.assertEqual(0, len(results))
+        self.assertIsNone(job._ws_client._ws)
 
     def test_websocket_retry_success(self):
-        pass
+        """Test successful retry."""
+        def result_callback(job_id, interim_result):
+            nonlocal results
+            results.append(interim_result)
+            self.assertEqual(JOB_ID_RETRY_SUCCESS, job_id)
+
+        results = []
+        job = self._get_job(job_id=JOB_ID_RETRY_SUCCESS, callback=result_callback)
+        time.sleep(JOB_PROGRESS_RESULT_COUNT+2)
+        self.assertEqual(JOB_PROGRESS_RESULT_COUNT, len(results))
+        self.assertIsNone(job._ws_client._ws)
 
     def test_websocket_retry_failure(self):
-        pass
+        """Test failed retry."""
+        def result_callback(job_id, interim_result):
+            nonlocal results
+            results.append(interim_result)
+            self.assertEqual(JOB_ID_RETRY_FAILURE, job_id)
 
-    def test_job_interim_results(self):
-        """Test retrieving a job already in final status."""
-        client = RuntimeWebsocketClient('ws://{}:{}'.format(
-            self.TEST_IP_ADDRESS, self.VALID_PORT), "foo")
+        results = []
+        job = self._get_job(job_id=JOB_ID_RETRY_FAILURE, callback=result_callback)
+        time.sleep(20)  # Need to wait for all retries.
+        self.assertEqual(0, len(results))
+        self.assertIsNone(job._ws_client._ws)
 
-        asyncio.get_event_loop().run_until_complete(
-            client.job_results(JOB_ID_PROGRESS_DONE, queue.Queue()))
+    def _get_job(self, callback=None, job_id=JOB_ID_PROGRESS_DONE):
+        """Get a runtime job."""
+        ws_client = RuntimeWebsocketClient(self.VALID_URL, "my_token")
+        job = RuntimeJob(backend=FakeQasmSimulator(),
+                         api_client=BaseFakeRuntimeClient(),
+                         ws_client=ws_client,
+                         job_id=job_id,
+                         program_id="my-program",
+                         user_callback=callback)
+        return job
diff --git a/test/ibmq/runtime/utils.py b/test/ibmq/runtime/utils.py
index caed4f6bd..46f8ce913 100644
--- a/test/ibmq/runtime/utils.py
+++ b/test/ibmq/runtime/utils.py
@@ -18,6 +18,7 @@
 
 
 def get_complex_types():
+    """Return a dictionary with values of more complicated types."""
     return {"string": "foo",
             "float": 1.5,
             "complex": 2+3j,
@@ -44,9 +45,11 @@ def __eq__(self, other):
 
 
 class SerializableClassDecoder(ResultDecoder):
+    """Decoder used for decode SerializableClass in job result."""
 
     @classmethod
     def decode(cls, data):
+        """Decode input data."""
         decoded = super().decode(data)
         if 'serializable_class' in decoded:
             decoded['serializable_class'] = \
diff --git a/test/ibmq/runtime/websocket_server.py b/test/ibmq/runtime/websocket_server.py
index 89da4ebcb..2cc6bb68b 100644
--- a/test/ibmq/runtime/websocket_server.py
+++ b/test/ibmq/runtime/websocket_server.py
@@ -18,7 +18,6 @@
 JOB_ID_ALREADY_DONE = 'JOB_ID_ALREADY_DONE'
 JOB_ID_RETRY_SUCCESS = 'JOB_ID_RETRY_SUCCESS'
 JOB_ID_RETRY_FAILURE = 'JOB_ID_RETRY_FAILURE'
-JOB_ID_RANDOM_CODE = 'JOB_ID_RANDOM_CODE'
 JOB_PROGRESS_RESULT_COUNT = 5
 
 
@@ -35,8 +34,6 @@ async def websocket_handler(websocket, path):
         await handle_token_retry_success(websocket)
     elif request == JOB_ID_RETRY_FAILURE:
         await handle_token_retry_failure(websocket)
-    elif request == JOB_ID_RANDOM_CODE:
-        await handle_close_random_code(websocket)
     else:
         raise ValueError(f"Unknown request {request}")
 
@@ -44,7 +41,7 @@ async def websocket_handler(websocket, path):
 async def handle_job_progress_done(websocket):
     """Send a few results then close with 1000."""
     for idx in range(JOB_PROGRESS_RESULT_COUNT):
-        await websocket.send(f"foo{idx}".encode())
+        await websocket.send(f"foo{idx}")
         await asyncio.sleep(1)
     await websocket.close(code=1000)
 
@@ -65,9 +62,4 @@ async def handle_token_retry_success(websocket):
 
 async def handle_token_retry_failure(websocket):
     """Continually close the socket, until both the first attempt and retry fail."""
-    await websocket.close()
-
-
-async def handle_close_random_code(websocket):
-    """Close with a random code."""
-    await websocket.close(code=1234)
+    await websocket.close(code=1011)

From 6415f89a59d5f056bdd5682c977fe540351e1eb9 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Thu, 6 May 2021 11:20:25 -0400
Subject: [PATCH 51/59] change program output format

---
 qiskit/providers/ibmq/runtime/runtime_program.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qiskit/providers/ibmq/runtime/runtime_program.py b/qiskit/providers/ibmq/runtime/runtime_program.py
index 70bc9c1c5..37c7a7f28 100644
--- a/qiskit/providers/ibmq/runtime/runtime_program.py
+++ b/qiskit/providers/ibmq/runtime/runtime_program.py
@@ -111,7 +111,7 @@ def _format_common(items: List) -> None:
                      f"  Version: {self.version}",
                      f"  Creation date: {self.creation_date}",
                      f"  Max execution time: {self.max_execution_time}",
-                     f"  Parameters:"]
+                     f"  Input parameters:"]
 
         if self._parameters:
             _format_common(self._parameters)

From d5b77cc05c51a6fc9294d0765bcadd1ea1469cf5 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Thu, 6 May 2021 14:19:06 -0400
Subject: [PATCH 52/59] fix test job status

---
 test/ibmq/runtime/fake_runtime_client.py | 10 +++++-----
 test/ibmq/runtime/test_runtime_ws.py     |  1 +
 test/ibmq/test_account_client.py         |  2 +-
 test/ibmq/test_random.py                 |  2 +-
 4 files changed, 8 insertions(+), 7 deletions(-)

diff --git a/test/ibmq/runtime/fake_runtime_client.py b/test/ibmq/runtime/fake_runtime_client.py
index 8565828fd..a7897c43b 100644
--- a/test/ibmq/runtime/fake_runtime_client.py
+++ b/test/ibmq/runtime/fake_runtime_client.py
@@ -67,7 +67,7 @@ class BaseFakeRuntimeJob:
     _job_progress = [
         "QUEUED",
         "RUNNING",
-        "SUCCEEDED"
+        "COMPLETED"
     ]
 
     _executor = ThreadPoolExecutor()
@@ -91,7 +91,7 @@ def _auto_progress(self):
             time.sleep(0.5)
             self._status = status
 
-        if self._status == "SUCCEEDED":
+        if self._status == "COMPLETED":
             self._result = json.dumps("foo")
 
     def to_dict(self):
@@ -150,7 +150,7 @@ def _auto_progress(self):
         """Automatically update job status."""
         super()._auto_progress()
 
-        if self._status == "SUCCEEDED":
+        if self._status == "COMPLETED":
             self._result = json.dumps(self.custom_result, cls=RuntimeEncoder)
 
 
@@ -164,9 +164,9 @@ def __init__(self, **kwargs):
     def _auto_progress(self):
         self._status = "RUNNING"
         time.sleep(self._runtime)
-        self._status = "SUCCEEDED"
+        self._status = "COMPLETED"
 
-        if self._status == "SUCCEEDED":
+        if self._status == "COMPLETED":
             self._result = json.dumps("foo")
 
 
diff --git a/test/ibmq/runtime/test_runtime_ws.py b/test/ibmq/runtime/test_runtime_ws.py
index 126adfb21..28b7369e1 100644
--- a/test/ibmq/runtime/test_runtime_ws.py
+++ b/test/ibmq/runtime/test_runtime_ws.py
@@ -60,6 +60,7 @@ def _ws_server(cls):
         loop = asyncio.new_event_loop()
         asyncio.set_event_loop(loop)
         # Launch the mock server.
+        # pylint: disable=no-member
         start_server = websockets.serve(websocket_handler, cls.TEST_IP_ADDRESS, cls.VALID_PORT)
         cls.server = loop.run_until_complete(start_server)
         cls._ws_start_event.set()
diff --git a/test/ibmq/test_account_client.py b/test/ibmq/test_account_client.py
index e18e50c27..8df28a530 100644
--- a/test/ibmq/test_account_client.py
+++ b/test/ibmq/test_account_client.py
@@ -69,7 +69,7 @@ def tearDown(self) -> None:
 
     def _get_client(self):
         """Helper for instantiating an AccountClient."""
-        return AccountClient(self.provider.credentials)
+        return AccountClient(self.provider.credentials)  # pylint: disable=no-value-for-parameter
 
     def test_exception_message(self):
         """Check exception has proper message."""
diff --git a/test/ibmq/test_random.py b/test/ibmq/test_random.py
index a74b6a5df..9d3dd34e5 100644
--- a/test/ibmq/test_random.py
+++ b/test/ibmq/test_random.py
@@ -103,7 +103,7 @@ def setUpClass(cls, provider):
         # pylint: disable=arguments-differ
         super().setUpClass()
         cls.provider = provider
-        random_service = IBMQRandomService(provider)
+        random_service = IBMQRandomService(provider)  # pylint: disable=no-value-for-parameter
         random_service._random_client = FakeRandomClient()
         random_service._initialized = False
         cls.provider._random = random_service

From 7e97927db9d39b4907cea6b1f6bc4dc103e1a472 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Thu, 6 May 2021 18:11:59 -0400
Subject: [PATCH 53/59] remove update program

---
 .../ibmq/runtime/ibm_runtime_service.py       | 58 -------------------
 test/ibmq/runtime/test_runtime.py             | 13 -----
 test/ibmq/runtime/test_runtime_integration.py | 12 ----
 3 files changed, 83 deletions(-)

diff --git a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
index 99d185b0b..be124fb42 100644
--- a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
+++ b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
@@ -304,64 +304,6 @@ def upload_program(
             raise QiskitRuntimeError(f"Failed to create program: {ex}") from None
         return response['id']
 
-    def update_program(
-            self,
-            program_id: str,
-            data: Union[bytes, str] = None,
-            metadata: Optional[Union[Dict, str]] = None,
-            name: Optional[str] = None,
-            max_execution_time: Optional[int] = None,
-            description: Optional[str] = None,
-            version: Optional[float] = None,
-            backend_requirements: Optional[str] = None,
-            parameters: Optional[List[ProgramParameter]] = None,
-            return_values: Optional[List[ProgramResult]] = None,
-            interim_results: Optional[List[ProgramResult]] = None
-    ) -> None:
-        """Update an existing runtime program.
-
-        Program metadata can be specified using the `metadata` parameter or
-        individual parameter (for example, `name` and `description`). If the
-        same metadata field is specified in both places, the individual parameter
-        takes precedence. For example, if you specify:
-
-            update_program(metadata={"name": "name1"}, name="name2")
-
-        ``name2`` will be used as the program name.
-
-        Args:
-            program_id: Program ID.
-            data: Name of the program file or program data to upload.
-            metadata: Name of the program metadata file or metadata dictionary.
-                A metadata file needs to be in the JSON format.
-                See :file:`program/program_metadata_sample.yaml` for an example.
-            name: Name of the program. Required if not specified via `metadata`.
-            max_execution_time: Maximum execution time in seconds. Required if
-                not specified via `metadata`.
-            description: Program description. Required if not specified via `metadata`.
-            version: Program version. The default is 1.0 if not specified.
-            backend_requirements: Backend requirements.
-            parameters: A list of program input parameters.
-            return_values: A list of program return values.
-            interim_results: A list of program interim results.
-
-        Raises:
-            RuntimeProgramNotFound: If the program doesn't exist.
-            IBMQNotAuthorizedError: If you are not authorized to upload programs.
-            QiskitRuntimeError: If the update failed.
-        """
-        program_metadata = self.program(program_id, refresh=True).to_dict()
-        program_metadata = self._merge_metadata(
-            initial=program_metadata,
-            metadata=metadata,
-            name=name, max_execution_time=max_execution_time, description=description,
-            version=version, backend_requirements=backend_requirements,
-            parameters=parameters,
-            return_values=return_values, interim_results=interim_results)
-
-        self.delete_program(program_id)
-        self.upload_program(data=data, metadata=program_metadata)
-
     def _merge_metadata(
             self,
             initial: Dict,
diff --git a/test/ibmq/runtime/test_runtime.py b/test/ibmq/runtime/test_runtime.py
index 063de83f9..1919a7ce0 100644
--- a/test/ibmq/runtime/test_runtime.py
+++ b/test/ibmq/runtime/test_runtime.py
@@ -309,19 +309,6 @@ def test_metadata_combined(self):
         self.assertEqual(update_metadata['max_execution_time'], program.max_execution_time)
         self.assertEqual(update_metadata["version"], program.version)
 
-    def test_update_program(self):
-        """Test updating program."""
-        update_metadata = {"version": "1.2", "max_execution_time": 600}
-        final_version = "1.3"
-        program_data = "foo".encode()
-        program_id = self.runtime.upload_program(
-            data=program_data, metadata=self.DEFAULT_METADATA)
-        self.runtime.update_program(program_id, data=program_data,
-                                    metadata=update_metadata, version=final_version)
-        program = self.runtime.program(program_id)
-        self.assertEqual(update_metadata['max_execution_time'], program.max_execution_time)
-        self.assertEqual(final_version, program.version)
-
     def _upload_program(self, name=None, max_execution_time=300):
         """Upload a new program."""
         name = name or uuid.uuid4().hex
diff --git a/test/ibmq/runtime/test_runtime_integration.py b/test/ibmq/runtime/test_runtime_integration.py
index f0bcd801b..0b61946ae 100644
--- a/test/ibmq/runtime/test_runtime_integration.py
+++ b/test/ibmq/runtime/test_runtime_integration.py
@@ -152,18 +152,6 @@ def test_upload_program_conflict(self):
         with self.assertRaises(RuntimeDuplicateProgramError):
             self._upload_program(name=name)
 
-    def test_update_program(self):
-        """Test updating a program."""
-        program_id = self._upload_program()
-        program = self.provider.runtime.program(program_id)
-
-        self.provider.runtime.delete_program(program_id)
-        new_cost = program.max_execution_time + 1000
-        new_id = self._upload_program(name=program.name, max_execution_time=new_cost)
-        updated = self.provider.runtime.program(new_id, refresh=True)
-        self.assertEqual(new_cost, updated.max_execution_time,
-                         f"Program {new_id} does not have the expected cost.")
-
     def test_delete_program(self):
         """Test deleting program."""
         program_id = self._upload_program()

From 68f32a6a0a4ce5686d34714a8832147e93f98cfc Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Thu, 6 May 2021 20:36:26 -0400
Subject: [PATCH 54/59] fix run_circuits parameters

---
 qiskit/providers/ibmq/accountprovider.py      | 82 ++++++-------------
 test/ibmq/runtime/test_runtime_integration.py |  7 ++
 2 files changed, 33 insertions(+), 56 deletions(-)

diff --git a/qiskit/providers/ibmq/accountprovider.py b/qiskit/providers/ibmq/accountprovider.py
index ba55e12f7..e57626b6d 100644
--- a/qiskit/providers/ibmq/accountprovider.py
+++ b/qiskit/providers/ibmq/accountprovider.py
@@ -20,8 +20,7 @@
 from qiskit.providers import ProviderV1 as Provider  # type: ignore[attr-defined]
 from qiskit.providers.models import (QasmBackendConfiguration,
                                      PulseBackendConfiguration)
-from qiskit.circuit import QuantumCircuit, Parameter
-from qiskit.pulse.instruction_schedule_map import InstructionScheduleMap
+from qiskit.circuit import QuantumCircuit
 from qiskit.providers.backend import BackendV1 as Backend
 from qiskit.providers.basebackend import BaseBackend
 from qiskit.transpiler import Layout
@@ -197,23 +196,18 @@ def _discover_remote_backends(self, timeout: Optional[float] = None) -> Dict[str
     def run_circuits(
             self,
             circuits: Union[QuantumCircuit, List[QuantumCircuit]],
-            backend: Optional[Union[Backend, BaseBackend]] = None,
+            backend: Union[Backend, BaseBackend],
+            shots: Optional[int] = None,
             initial_layout: Optional[Union[Layout, Dict, List]] = None,
+            layout_method: Optional[str] = None,
+            routing_method: Optional[str] = None,
+            translation_method: Optional[str] = None,
             seed_transpiler: Optional[int] = None,
             optimization_level: Optional[int] = None,
-            transpiler_options: Optional[dict] = None,
-            scheduling_method: Optional[str] = None,
-            shots: Optional[int] = None,
-            memory: Optional[bool] = None,
-            memory_slots: Optional[int] = None,
-            memory_slot_size: Optional[int] = None,
-            rep_time: Optional[int] = None,
+            init_qubits: Optional[bool] = True,
             rep_delay: Optional[float] = None,
-            parameter_binds: Optional[List[Dict[Parameter, float]]] = None,
-            schedule_circuit: bool = False,
-            inst_map: InstructionScheduleMap = None,
-            meas_map: List[List[int]] = None,
-            init_qubits: Optional[bool] = None,
+            transpiler_options: Optional[dict] = None,
+            measurement_error_mitigation: Optional[bool] = False,
             **run_config: Dict
     ) -> 'runtime_job.RuntimeJob':
         """Execute the input circuit(s) on a backend using the runtime service.
@@ -231,6 +225,15 @@ def run_circuits(
 
             initial_layout: Initial position of virtual qubits on physical qubits.
 
+            layout_method: Name of layout selection pass ('trivial', 'dense',
+                'noise_adaptive', 'sabre').
+                Sometimes a perfect layout can be available in which case the layout_method
+                may not run.
+
+            routing_method: Name of routing pass ('basic', 'lookahead', 'stochastic', 'sabre')
+
+            translation_method: Name of translation pass ('unroller', 'translator', 'synthesis')
+
             seed_transpiler: Sets random seed for the stochastic parts of the transpiler.
 
             optimization_level: How much optimization to perform on the circuits.
@@ -238,47 +241,19 @@ def run_circuits(
                 transpilation time.
                 If None, level 1 will be chosen as default.
 
-            transpiler_options: Additional transpiler options.
-
-            scheduling_method: Scheduling method.
-
             shots: Number of repetitions of each circuit, for sampling. Default: 1024.
 
-            memory: If True, per-shot measurement bitstrings are returned as well
-                (provided the backend supports it). Default: False
-
-            memory_slots: Number of classical memory slots used in this job.
-
-            memory_slot_size: Size of each memory slot if the output is Level 0.
-
-            rep_time: Time per program execution in seconds. Must be from the list provided
-                by the backend configuration (``backend.configuration().rep_times``).
-                Defaults to the first entry.
-
             rep_delay: Delay between programs in seconds. Only supported on certain
                 backends (``backend.configuration().dynamic_reprate_enabled`` ). If supported,
                 ``rep_delay`` will be used instead of ``rep_time`` and must be from the
                 range supplied by the backend (``backend.configuration().rep_delay_range``).
                 Default is given by ``backend.configuration().default_rep_delay``.
 
-            parameter_binds: List of Parameter bindings over which the set of
-                experiments will be executed. Each list element (bind) should be of the form
-                ``{Parameter1: value1, Parameter2: value2, ...}``. All binds will be
-                executed across all experiments, e.g. if parameter_binds is a
-                length-n list, and there are m experiments, a total of :math:`m x n`
-                experiments will be run (one for each experiment/bind pair).
-
-            schedule_circuit: If ``True``, ``circuits`` will be converted to
-                :class:`qiskit.pulse.Schedule` objects prior to execution.
-
-            inst_map: Mapping of circuit operations to pulse schedules. If None, defaults to the
-                ``instruction_schedule_map`` of ``backend``.
+            init_qubits: Whether to reset the qubits to the ground state for each shot.
 
-            meas_map: List of sets of qubits that must be measured together. If None,
-                defaults to the ``meas_map`` of ``backend``.
+            transpiler_options: Additional transpiler options.
 
-            init_qubits: Whether to reset the qubits to the ground state for each shot.
-                Default: ``True``.
+            measurement_error_mitigation: Whether to apply measurement error mitigation.
 
             **run_config: Extra arguments used to configure the circuit execution.
 
@@ -290,19 +265,14 @@ def run_circuits(
             'initial_layout': initial_layout,
             'seed_transpiler': seed_transpiler,
             'optimization_level': optimization_level,
-            'scheduling_method': scheduling_method,
+            'layout_method': layout_method,
             'shots': shots,
-            'memory': memory,
-            'memory_slots': memory_slots,
-            'memory_slot_size': memory_slot_size,
-            'rep_time': rep_time,
+            'routing_method': routing_method,
+            'translation_method': translation_method,
             'rep_delay': rep_delay,
-            'parameter_binds': parameter_binds,
-            'schedule_circuit': schedule_circuit,
-            'inst_map': inst_map,
-            'meas_map': meas_map,
             'init_qubits': init_qubits,
-            'transpiler_options': transpiler_options
+            'transpiler_options': transpiler_options,
+            'measurement_error_mitigation': measurement_error_mitigation
         }
         inputs.update(run_config)
         options = {'backend_name': backend.name()}
diff --git a/test/ibmq/runtime/test_runtime_integration.py b/test/ibmq/runtime/test_runtime_integration.py
index 0b61946ae..8a8cc246b 100644
--- a/test/ibmq/runtime/test_runtime_integration.py
+++ b/test/ibmq/runtime/test_runtime_integration.py
@@ -20,6 +20,7 @@
 from contextlib import suppress
 
 from qiskit.providers.jobstatus import JobStatus
+from qiskit.test.reference_circuits import ReferenceCircuits
 from qiskit.providers.ibmq.exceptions import IBMQNotAuthorizedError
 from qiskit.providers.ibmq.runtime.exceptions import (RuntimeDuplicateProgramError,
                                                       RuntimeProgramNotFound,
@@ -447,6 +448,12 @@ def test_logout(self):
         self._upload_program()
         _ = self._run_program()
 
+    def test_run_circuit(self):
+        """Test run_circuit"""
+        job = self.provider.run_circuits(
+            ReferenceCircuits.bell(), backend=self.backend, shots=100)
+        job.result()
+
     def _validate_program(self, program):
         """Validate a program."""
         self.assertTrue(program)

From 49e957ee12c75458d84d0c263d8bca6295df1110 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Thu, 6 May 2021 21:36:29 -0400
Subject: [PATCH 55/59] doc update

---
 qiskit/providers/ibmq/runtime/__init__.py     | 43 ++++++++++---------
 .../ibmq/runtime/ibm_runtime_service.py       | 27 +++++++-----
 .../ibmq/runtime/program/__init__.py          |  4 +-
 .../runtime/{ => program}/result_decoder.py   | 20 ++++++++-
 .../ibmq/runtime/program/user_messenger.py    |  2 +-
 qiskit/providers/ibmq/runtime/runtime_job.py  | 20 ++++-----
 .../providers/ibmq/runtime/runtime_program.py | 22 +++++-----
 test/ibmq/runtime/utils.py                    |  2 +-
 8 files changed, 80 insertions(+), 60 deletions(-)
 rename qiskit/providers/ibmq/runtime/{ => program}/result_decoder.py (56%)

diff --git a/qiskit/providers/ibmq/runtime/__init__.py b/qiskit/providers/ibmq/runtime/__init__.py
index f5f82452b..ddc03ce0f 100644
--- a/qiskit/providers/ibmq/runtime/__init__.py
+++ b/qiskit/providers/ibmq/runtime/__init__.py
@@ -11,9 +11,9 @@
 # that they have been altered from the originals.
 
 """
-======================================================
-Runtime Service (:mod:`qiskit.providers.ibmq.runtime`)
-======================================================
+==============================================
+Runtime (:mod:`qiskit.providers.ibmq.runtime`)
+==============================================
 
 .. currentmodule:: qiskit.providers.ibmq.runtime
 
@@ -21,7 +21,7 @@
 
 .. note::
 
-    The runtime service is not available to all providers. To check if a provider
+    The Qiskit Runtime service is not available to all providers. To check if your provider
     has access::
 
         from qiskit import IBMQ
@@ -33,7 +33,7 @@
 
 .. note::
 
-    Not all backends support Runtime. Refer to documentation on
+    Not all backends support Qiskit Runtime. Refer to documentation in
     `Qiskit-Partners/qiskit-runtime
     <https://github.com/Qiskit-Partners/qiskit-runtime>`_ for more information.
 
@@ -52,11 +52,11 @@
 The Qiskit Runtime Service allows authorized users to upload their Qiskit quantum programs.
 A Qiskit quantum program, also called a runtime program, is a piece of Python
 code that takes certain inputs, performs
-quantum and classical processing, and returns the results. The same or other
+quantum and maybe classical processing, and returns the results. The same or other
 authorized users can invoke these quantum programs by simply passing in parameters.
 
 `Qiskit-Partners/qiskit-runtime <https://github.com/Qiskit-Partners/qiskit-runtime>`_
-contains detailed tutorials on how to use the runtime service.
+contains detailed tutorials on how to use Qiskit Runtime.
 
 
 Listing runtime programs
@@ -74,14 +74,14 @@
     # Get a single program.
     program = provider.runtime.program('circuit-runner')
 
-    # Print program definition.
+    # Print program metadata.
     print(program)
 
 In the example above, ``provider.runtime`` points to the runtime service class
 :class:`IBMRuntimeService`, which is the main entry
 point for using this service. The example prints the program metadata of all
 available runtime programs and of just the ``circuit-runner`` program. A program
-metadata consists of a program's ID, name, description, input parameters,
+metadata consists of the program's ID, name, description, input parameters,
 return values, interim results, and other information that helps you to know
 more about the program.
 
@@ -92,6 +92,7 @@
 For example::
 
     from qiskit import IBMQ, QuantumCircuit
+    from qiskit_runtime.circuit_runner import RunnerResult
 
     provider = IBMQ.load_account()
     backend = provider.backend.ibmq_montreal
@@ -110,14 +111,14 @@
                                inputs=runtime_inputs)
 
     # Get runtime job result.
-    result = job.result()
+    result = job.result(decoder=RunnerResult)
 
 The example above invokes the ``circuit-runner`` program,
 which compiles, executes, and optionally applies measurement error mitigation to
 the circuit result.
 
-Runtime Job
------------
+Runtime Jobs
+------------
 
 When you use the :meth:`IBMRuntimeService.run` method to invoke a runtime
 program, a
@@ -166,8 +167,9 @@ def interim_result_callback(job_id, interim_result):
 method, which serves as the entry point to the program. The ``backend`` parameter
 is a :class:`ProgramBackend` instance whose :meth:`ProgramBackend.run` method
 can be used to submit circuits. The ``user_messenger`` is a :class:`UserMessenger`
-whose :meth:`UserMessenger.publish` method can be used to publish interim and
-final results. See :file:`program/program_template.py` for a program data
+instance whose :meth:`UserMessenger.publish` method can be used to publish interim and
+final results.
+See :file:`qiskit.providers.ibmq.runtime.program.program_template.py` for a program data
 template file.
 
 Each program metadata must include at least the program name, description, and
@@ -186,17 +188,16 @@ def interim_result_callback(job_id, interim_result):
     program_id = provider.runtime.upload_program(
                     data="my_vqe.py",
                     metadata="my_vqe_metadata.json",
-                    version=1.2
+                    version="1.2"
                 )
 
 In the example above, the file ``my_vqe.py`` contains the program data, and
 ``my_vqe_metadata.json`` contains the program metadata. An additional
-parameter ``version`` is also specified, which will be used instead of the
-``version`` value in ``my_vqe_metadata.json``.
+parameter ``version`` is also specified, which takes precedence over any
+``version`` value specified in ``my_vqe_metadata.json``.
 
-Methods :meth:`IBMRuntimeService.update_program` and
-:meth:`IBMRuntimeService.delete_program` allow you to update and delete a
-program, respectively.
+Method :meth:`IBMRuntimeService.delete_program` allows you to delete a
+program.
 
 Files related to writing a runtime program are in the
 ``qiskit/providers/ibmq/runtime/program`` directory.
@@ -220,5 +221,5 @@ def interim_result_callback(job_id, interim_result):
 from .runtime_program import RuntimeProgram
 from .program.user_messenger import UserMessenger
 from .program.program_backend import ProgramBackend
-from .result_decoder import ResultDecoder
+from .program.result_decoder import ResultDecoder
 from .utils import RuntimeEncoder, RuntimeDecoder
diff --git a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
index be124fb42..42be027f3 100644
--- a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
+++ b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
@@ -21,10 +21,10 @@
 
 from .runtime_job import RuntimeJob
 from .runtime_program import RuntimeProgram, ProgramParameter, ProgramResult
-from .result_decoder import ResultDecoder
 from .utils import RuntimeEncoder, RuntimeDecoder
 from .exceptions import (QiskitRuntimeError, RuntimeDuplicateProgramError, RuntimeProgramNotFound,
                          RuntimeJobNotFound)
+from .program.result_decoder import ResultDecoder
 from ..api.clients.runtime import RuntimeClient
 from ..api.clients.runtime_ws import RuntimeWebsocketClient
 from ..api.exceptions import RequestsApiError
@@ -34,21 +34,22 @@
 
 
 class IBMRuntimeService:
-    """Class for interacting with the Qiskit runtime service.
+    """Class for interacting with the Qiskit Runtime service.
 
-    The Qiskit Runtime Service allows authorized users to upload their quantum programs
-    that can be invoked by other users. A quantum program is a piece of code that takes
-    certain inputs, performs quantum and classical processing, and returns the
-    results. Quantum programs, also known as runtime programs, run in a special
+    The Qiskit Runtime service allows authorized users to upload their quantum programs
+    that can be invoked by themselves and other users. A quantum program is a piece of code that takes
+    certain inputs, performs quantum and maybe classical processing, and returns the
+    results. Quantum programs, also known as Qiskit runtime programs, run in a special
     runtime environment that significantly reduces waiting time during computational
     iterations.
 
     A sample workflow of using the runtime service::
 
         from qiskit import IBMQ, QuantumCircuit
+        from qiskit_runtime.circuit_runner import RunnerResult
 
         provider = IBMQ.load_account()
-        backend = provider.backend.ibmq_qasm_simulator
+        backend = provider.backend.ibmq_montreal
 
         # List all available programs.
         provider.runtime.pprint_programs()
@@ -67,11 +68,11 @@ class IBMRuntimeService:
                                    inputs=runtime_inputs)
 
         # Get runtime job result.
-        result = job.result()
+        result = job.result(decoder=RunnerResult)
 
     If the program has any interim results, you can use the ``callback``
     parameter of the :meth:`run` method to stream the interim results.
-    Alternatively, you can use the :meth:`stream_results` method to stream
+    Alternatively, you can use the :meth:`RuntimeJob.stream_results` method to stream
     the results at a later time, but before the job finishes.
 
     The :meth:`run` method returns a
@@ -107,6 +108,8 @@ def pprint_programs(self, refresh: bool = False) -> None:
     def programs(self, refresh: bool = False) -> List[RuntimeProgram]:
         """Return available runtime programs.
 
+        Currently only program metadata is returned.
+
         Args:
             refresh: If ``True``, re-query the server for the programs. Otherwise
                 return the cached value.
@@ -125,6 +128,8 @@ def programs(self, refresh: bool = False) -> List[RuntimeProgram]:
     def program(self, program_id: str, refresh: bool = False) -> RuntimeProgram:
         """Retrieve a runtime program.
 
+        Currently only program metadata is returned.
+
         Args:
             program_id: Program ID.
             refresh: If ``True``, re-query the server for the program. Otherwise
@@ -240,7 +245,7 @@ def upload_program(
     ) -> str:
         """Upload a runtime program.
 
-        In addition to program data, the following program metadata are also
+        In addition to program data, the following program metadata is also
         required:
 
             - name
@@ -250,7 +255,7 @@ def upload_program(
         Program metadata can be specified using the `metadata` parameter or
         individual parameter (for example, `name` and `description`). If the
         same metadata field is specified in both places, the individual parameter
-        takes precedence. For example, if you specify:
+        takes precedence. For example, if you specify::
 
             upload_program(metadata={"name": "name1"}, name="name2")
 
diff --git a/qiskit/providers/ibmq/runtime/program/__init__.py b/qiskit/providers/ibmq/runtime/program/__init__.py
index 02273e6fe..16919434b 100644
--- a/qiskit/providers/ibmq/runtime/program/__init__.py
+++ b/qiskit/providers/ibmq/runtime/program/__init__.py
@@ -10,7 +10,7 @@
 # copyright notice, and modified files need to carry a notice indicating
 # that they have been altered from the originals.
 
-"""Runtime program package.
+"""Qiskit Runtime program package.
 
-This package contains files to help you write runtime programs.
+This package contains files to help you write Qiskit Runtime programs.
 """
diff --git a/qiskit/providers/ibmq/runtime/result_decoder.py b/qiskit/providers/ibmq/runtime/program/result_decoder.py
similarity index 56%
rename from qiskit/providers/ibmq/runtime/result_decoder.py
rename to qiskit/providers/ibmq/runtime/program/result_decoder.py
index de5f29c14..1ba634e3e 100644
--- a/qiskit/providers/ibmq/runtime/result_decoder.py
+++ b/qiskit/providers/ibmq/runtime/program/result_decoder.py
@@ -15,11 +15,27 @@
 import json
 from typing import Any
 
-from .utils import RuntimeDecoder
+from qiskit.providers.ibmq.runtime.utils import RuntimeDecoder
 
 
 class ResultDecoder:
-    """Runtime job result decoder."""
+    """Runtime job result decoder.
+
+    You can subclass this class and overwrite the :meth:`decode` method
+    to create a custom result decoder for the
+    results of your runtime program. For example::
+
+        class MyResultDecoder(ResultDecoder):
+
+            @classmethod
+            def decode(cls, data):
+                decoded = super().decode(data)
+                custom_processing(decoded)  # perform custom processing
+
+    Users of your program will need to pass in the subclass when invoking
+    :meth:`qiskit.providers.ibmq.runtime.RuntimeJob.result` or
+    :meth:`qiskit.providers.ibmq.runtime.IBMRuntimeService.run`.
+    """
 
     @classmethod
     def decode(cls, data: str) -> Any:
diff --git a/qiskit/providers/ibmq/runtime/program/user_messenger.py b/qiskit/providers/ibmq/runtime/program/user_messenger.py
index 421f61ad7..36240c9fa 100644
--- a/qiskit/providers/ibmq/runtime/program/user_messenger.py
+++ b/qiskit/providers/ibmq/runtime/program/user_messenger.py
@@ -21,7 +21,7 @@
 class UserMessenger:
     """Base class for handling communication with program users.
 
-    This class can be used when writing a new runtime program.
+    This class can be used when writing a new Qiskit Runtime program.
     """
 
     def publish(
diff --git a/qiskit/providers/ibmq/runtime/runtime_job.py b/qiskit/providers/ibmq/runtime/runtime_job.py
index a16de95b6..220668817 100644
--- a/qiskit/providers/ibmq/runtime/runtime_job.py
+++ b/qiskit/providers/ibmq/runtime/runtime_job.py
@@ -27,7 +27,7 @@
 
 from .constants import API_TO_JOB_STATUS
 from .exceptions import RuntimeJobFailureError, RuntimeInvalidStateError, QiskitRuntimeError
-from .result_decoder import ResultDecoder
+from .program.result_decoder import ResultDecoder
 from ..api.clients import RuntimeClient, RuntimeWebsocketClient
 from ..exceptions import IBMQError
 from ..api.exceptions import RequestsApiError
@@ -39,9 +39,9 @@ class RuntimeJob:
     """Representation of a runtime program execution.
 
     A new ``RuntimeJob`` instance is returned when you call
-    :meth:`qiskit.providers.ibmq.runtime.IBMRuntimeService.run`
+    :meth:`IBMRuntimeService.run<qiskit.providers.ibmq.runtime.IBMRuntimeService.run>`
     to execute a runtime program, and when you call
-    :meth:`qiskit.providers.ibmq.runtime.IBMRuntimeService.job`
+    :meth:`IBMRuntimeService.job<qiskit.providers.ibmq.runtime.IBMRuntimeService.job>`
     to retrieve a previously executed job.
 
     If the program execution is successful, you can inspect the job's status by
@@ -68,7 +68,7 @@ class RuntimeJob:
     the results at a later time, but before the job finishes.
     """
 
-    POISON_PILL = "_poison_pill"
+    _POISON_PILL = "_poison_pill"
     """Used to inform streaming to stop."""
 
     _executor = futures.ThreadPoolExecutor(thread_name_prefix="runtime_job")
@@ -123,10 +123,6 @@ def result(
     ) -> Any:
         """Return the results of the job.
 
-        If ``include_interim=True`` is specified, this method will return a
-        list that includes both interim and final results in the order they
-        were published by the program.
-
         Args:
             timeout: Number of seconds to wait for job.
             wait: Seconds between queries.
@@ -269,7 +265,7 @@ def _start_websocket_client(
                 "An error occurred while streaming results "
                 "from the server for job %s:\n%s", self.job_id(), traceback.format_exc())
         finally:
-            self._result_queue.put_nowait(self.POISON_PILL)
+            self._result_queue.put_nowait(self._POISON_PILL)
             if self._streaming_loop is not None:
                 self._streaming_loop.run_until_complete(  # type: ignore[unreachable]
                     self._ws_client.disconnect())
@@ -293,7 +289,7 @@ def _stream_results(
         while True:
             try:
                 response = result_queue.get()
-                if response == self.POISON_PILL:
+                if response == self._POISON_PILL:
                     self._empty_result_queue(result_queue)
                     return
                 user_callback(self.job_id(), _decoder.decode(response))
@@ -315,7 +311,7 @@ def _empty_result_queue(self, result_queue: queue.Queue) -> None:
             pass
 
     def job_id(self) -> str:
-        """Return a unique id identifying the job.
+        """Return a unique ID identifying the job.
 
         Returns:
             Job ID.
@@ -341,7 +337,7 @@ def inputs(self) -> Dict:
 
     @property
     def program_id(self) -> str:
-        """Returns program ID.
+        """Program ID.
 
         Returns:
             ID of the program this job is for.
diff --git a/qiskit/providers/ibmq/runtime/runtime_program.py b/qiskit/providers/ibmq/runtime/runtime_program.py
index 37c7a7f28..5d547bd55 100644
--- a/qiskit/providers/ibmq/runtime/runtime_program.py
+++ b/qiskit/providers/ibmq/runtime/runtime_program.py
@@ -151,7 +151,7 @@ def to_dict(self) -> Dict:
 
     @property
     def program_id(self) -> str:
-        """Return program ID.
+        """Program ID.
 
         Returns:
             Program ID.
@@ -160,7 +160,7 @@ def program_id(self) -> str:
 
     @property
     def name(self) -> str:
-        """Return program name.
+        """Program name.
 
         Returns:
             Program name.
@@ -169,7 +169,7 @@ def name(self) -> str:
 
     @property
     def description(self) -> str:
-        """Return program description.
+        """Program description.
 
         Returns:
             Program description.
@@ -178,7 +178,7 @@ def description(self) -> str:
 
     @property
     def parameters(self) -> List['ProgramParameter']:
-        """Return program parameter definitions.
+        """Program parameter definitions.
 
         Returns:
             Parameter definitions for this program.
@@ -187,7 +187,7 @@ def parameters(self) -> List['ProgramParameter']:
 
     @property
     def return_values(self) -> List['ProgramResult']:
-        """Return program return value definitions.
+        """Program return value definitions.
 
         Returns:
             Return value definitions for this program.
@@ -196,7 +196,7 @@ def return_values(self) -> List['ProgramResult']:
 
     @property
     def interim_results(self) -> List['ProgramResult']:
-        """Return program interim result definitions.
+        """Program interim result definitions.
 
         Returns:
             Interim result definitions for this program.
@@ -205,7 +205,9 @@ def interim_results(self) -> List['ProgramResult']:
 
     @property
     def max_execution_time(self) -> int:
-        """Return maximum execution time.
+        """Maximum execution time in seconds.
+
+        A program execution exceeding this time will be forcibly terminated.
 
         Returns:
             Maximum execution time.
@@ -214,7 +216,7 @@ def max_execution_time(self) -> int:
 
     @property
     def version(self) -> str:
-        """Return program version.
+        """Program version.
 
         Returns:
             Program version.
@@ -223,7 +225,7 @@ def version(self) -> str:
 
     @property
     def backend_requirements(self) -> Dict:
-        """Return backend requirements.
+        """Backend requirements.
 
         Returns:
             Backend requirements for this program.
@@ -232,7 +234,7 @@ def backend_requirements(self) -> Dict:
 
     @property
     def creation_date(self) -> str:
-        """Return program creation date.
+        """Program creation date.
 
         Returns:
             Program creation date.
diff --git a/test/ibmq/runtime/utils.py b/test/ibmq/runtime/utils.py
index 46f8ce913..97c7ba766 100644
--- a/test/ibmq/runtime/utils.py
+++ b/test/ibmq/runtime/utils.py
@@ -14,7 +14,7 @@
 
 import json
 
-from qiskit.providers.ibmq.runtime.result_decoder import ResultDecoder
+from qiskit.providers.ibmq.runtime import ResultDecoder
 
 
 def get_complex_types():

From b26a7850c2d7d1be84829db99a2efb449c50c1a8 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Thu, 6 May 2021 21:49:20 -0400
Subject: [PATCH 56/59] fix lint

---
 qiskit/providers/ibmq/runtime/ibm_runtime_service.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
index 42be027f3..444b75aff 100644
--- a/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
+++ b/qiskit/providers/ibmq/runtime/ibm_runtime_service.py
@@ -37,11 +37,11 @@ class IBMRuntimeService:
     """Class for interacting with the Qiskit Runtime service.
 
     The Qiskit Runtime service allows authorized users to upload their quantum programs
-    that can be invoked by themselves and other users. A quantum program is a piece of code that takes
-    certain inputs, performs quantum and maybe classical processing, and returns the
-    results. Quantum programs, also known as Qiskit runtime programs, run in a special
-    runtime environment that significantly reduces waiting time during computational
-    iterations.
+    that can be invoked by themselves and other users. A quantum program is a
+    piece of code that takes certain inputs, performs quantum and maybe classical
+    processing, and returns the results. Quantum programs, also known as
+    Qiskit Runtime programs, run in a special runtime environment that significantly
+    reduces waiting time during computational iterations.
 
     A sample workflow of using the runtime service::
 

From 0d8b9be65db1a987e11c7edbd111571c95deed2e Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Thu, 6 May 2021 21:57:50 -0400
Subject: [PATCH 57/59] minor doc update

---
 qiskit/providers/ibmq/runtime/__init__.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/qiskit/providers/ibmq/runtime/__init__.py b/qiskit/providers/ibmq/runtime/__init__.py
index ddc03ce0f..b7c9b94a5 100644
--- a/qiskit/providers/ibmq/runtime/__init__.py
+++ b/qiskit/providers/ibmq/runtime/__init__.py
@@ -169,7 +169,7 @@ def interim_result_callback(job_id, interim_result):
 can be used to submit circuits. The ``user_messenger`` is a :class:`UserMessenger`
 instance whose :meth:`UserMessenger.publish` method can be used to publish interim and
 final results.
-See :file:`qiskit.providers.ibmq.runtime.program.program_template.py` for a program data
+See `qiskit/providers/ibmq/runtime/program/program_template.py` for a program data
 template file.
 
 Each program metadata must include at least the program name, description, and
@@ -177,7 +177,8 @@ def interim_result_callback(job_id, interim_result):
 the :meth:`IBMRuntimeService.upload_program` method. Instead of passing in
 the metadata fields individually, you can pass in a JSON file or a dictionary
 to :meth:`IBMRuntimeService.upload_program` via the ``metadata`` parameter.
-:file:`program/program_metadata_sample.json` is a sample file of program metadata.
+`qiskit/providers/ibmq/runtime/program/program_metadata_sample.json`
+is a sample file of program metadata.
 
 You can use the :meth:`IBMRuntimeService.upload_program` to upload a program.
 For example::

From 227fdc6e2b023fca130f25cc4d1889d71925ea06 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Fri, 7 May 2021 08:25:02 -0400
Subject: [PATCH 58/59] more minor doc update

---
 qiskit/providers/ibmq/accountprovider.py     | 11 +++++++++++
 qiskit/providers/ibmq/api/rest/runtime.py    |  2 +-
 qiskit/providers/ibmq/runtime/runtime_job.py |  2 +-
 3 files changed, 13 insertions(+), 2 deletions(-)

diff --git a/qiskit/providers/ibmq/accountprovider.py b/qiskit/providers/ibmq/accountprovider.py
index e57626b6d..77d8a4c03 100644
--- a/qiskit/providers/ibmq/accountprovider.py
+++ b/qiskit/providers/ibmq/accountprovider.py
@@ -216,6 +216,17 @@ def run_circuits(
             This method uses the IBM Quantum runtime service which is not
             available to all accounts.
 
+        Note:
+            This method returns a :class:``~qiskit.provider.ibmq.runtime.RuntimeJob``.
+            To get the job result, you'll need to use the
+            ``qiskit_runtime.circuit_runner.RunnerResult`` class
+            as the ``decoder``, e.g.::
+
+                result = provider.run_circuits(...).result(decoder=RunnerResult)
+
+            You can find more about the ``RunnerResult`` class in the
+            `qiskit-runtime repository<https://github.com/Qiskit-Partners/qiskit-runtime>`_.
+
         Args:
             circuits: Circuit(s) to execute.
 
diff --git a/qiskit/providers/ibmq/api/rest/runtime.py b/qiskit/providers/ibmq/api/rest/runtime.py
index f46c93635..60a2e9b2c 100644
--- a/qiskit/providers/ibmq/api/rest/runtime.py
+++ b/qiskit/providers/ibmq/api/rest/runtime.py
@@ -10,7 +10,7 @@
 # copyright notice, and modified files need to carry a notice indicating
 # that they have been altered from the originals.
 
-"""Random REST adapter."""
+"""Runtime REST adapter."""
 
 import logging
 from typing import Dict, List, Any, Union, Optional
diff --git a/qiskit/providers/ibmq/runtime/runtime_job.py b/qiskit/providers/ibmq/runtime/runtime_job.py
index 220668817..c717dc480 100644
--- a/qiskit/providers/ibmq/runtime/runtime_job.py
+++ b/qiskit/providers/ibmq/runtime/runtime_job.py
@@ -40,7 +40,7 @@ class RuntimeJob:
 
     A new ``RuntimeJob`` instance is returned when you call
     :meth:`IBMRuntimeService.run<qiskit.providers.ibmq.runtime.IBMRuntimeService.run>`
-    to execute a runtime program, and when you call
+    to execute a runtime program, or
     :meth:`IBMRuntimeService.job<qiskit.providers.ibmq.runtime.IBMRuntimeService.job>`
     to retrieve a previously executed job.
 

From b500e98777a1cf9450275626c7aa673cc1617171 Mon Sep 17 00:00:00 2001
From: jessieyu <jessieyu@us.ibm.com>
Date: Fri, 7 May 2021 08:55:59 -0400
Subject: [PATCH 59/59] fix doc link

---
 qiskit/providers/ibmq/accountprovider.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/qiskit/providers/ibmq/accountprovider.py b/qiskit/providers/ibmq/accountprovider.py
index 77d8a4c03..f55ef8c9d 100644
--- a/qiskit/providers/ibmq/accountprovider.py
+++ b/qiskit/providers/ibmq/accountprovider.py
@@ -225,7 +225,7 @@ def run_circuits(
                 result = provider.run_circuits(...).result(decoder=RunnerResult)
 
             You can find more about the ``RunnerResult`` class in the
-            `qiskit-runtime repository<https://github.com/Qiskit-Partners/qiskit-runtime>`_.
+            `qiskit-runtime repository <https://github.com/Qiskit-Partners/qiskit-runtime>`_.
 
         Args:
             circuits: Circuit(s) to execute.