feat: Read and append table data (#36)

Author: mure

nisystemlink/clients/dataframe/_data_frame_client.py

@@ -31,14 +31,14 @@ def api_info(self) -> models.ApiInfo:
 
     @get(
         "tables",
-        args=(
+        args=[
             Query("take"),
             Query("id"),
             Query("orderBy"),
             Query("orderByDescending"),
             Query("continuationToken"),
             Query("workspace"),
-        ),
+        ],
     )
     def list_tables(
         self,
@@ -148,3 +148,52 @@ def modify_tables(
             tables were modified successfully.
         """
         ...
+
+    @get(
+        "tables/{id}/data",
+        args=[
+            Path("id"),
+            Query("columns"),
+            Query("orderBy"),
+            Query("orderByDescending"),
+            Query("take"),
+            Query("continuationToken"),
+        ],
+    )
+    def get_table_data(
+        self,
+        id: str,
+        columns: Optional[List[str]] = None,
+        order_by: Optional[List[str]] = None,
+        order_by_descending: Optional[bool] = None,
+        take: Optional[int] = None,
+        continuation_token: Optional[str] = None,
+    ) -> models.PagedTableRows:
+        """Reads raw data from the table identified by its ID.
+
+        Args:
+            id: Unique ID of a DataFrame table.
+            columns: Columns to include in the response. Data will be returned in the same order as
+                the columns. If not specified, all columns are returned.
+            order_by: List of columns to sort by. Multiple columns may be specified to order rows
+                that have the same value for prior columns. The columns used for ordering do not
+                need to be included in the columns list, in which case they are not returned. If
+                not specified, then the order in which results are returned is undefined.
+            order_by_descending: Whether to sort descending instead of ascending. Defaults to false.
+            take: Limits the returned list to the specified number of results. Defaults to 500.
+            continuation_token: The token used to paginate results.
+
+        Returns:
+            models.PagedTableRows: The table data and total number of rows with a continuation token.
+        """
+        ...
+
+    @post("tables/{id}/data", args=[Path, Body])
+    def append_table_data(self, id: str, data: models.AppendTableDataRequest) -> None:
+        """Appends one or more rows of data to the table identified by its ID.
+
+        Args:
+            id: Unique ID of a DataFrame table.
+            data: The rows of data to append and any additional options.
+        """
+        ...

nisystemlink/clients/dataframe/models/__init__.py

@@ -1,14 +1,17 @@
+from ._append_table_data_request import AppendTableDataRequest
 from ._api_info import ApiInfo, Operation, OperationsV1
 from ._create_table_request import CreateTableRequest
 from ._column import Column
 from ._column_type import ColumnType
+from ._data_frame import DataFrame
 from ._data_type import DataType
 from ._delete_tables_partial_success import DeleteTablesPartialSuccess
 from ._modify_tables_partial_success import ModifyTablesPartialSuccess
 from ._modify_table_request import ColumnMetadataPatch, ModifyTableRequest
 from ._modify_tables_request import ModifyTablesRequest, TableMetdataModification
 from ._order_by import OrderBy
 from ._paged_tables import PagedTables
+from ._paged_table_rows import PagedTableRows
 from ._query_tables_request import QueryTablesRequest
 from ._table_metadata import TableMetadata
 

nisystemlink/clients/dataframe/models/_append_table_data_request.py

@@ -0,0 +1,17 @@
+from typing import Optional
+
+from nisystemlink.clients.core._uplink._json_model import JsonModel
+
+from ._data_frame import DataFrame
+
+
+class AppendTableDataRequest(JsonModel):
+    """Contains the rows to append and optional flags. The ``frame`` field is
+    required unless ``endOfData`` is true.
+    """
+
+    frame: Optional[DataFrame] = None
+    """The data frame containing the rows to append."""
+
+    end_of_data: Optional[bool] = None
+    """Whether the table should expect any additional rows to be appended in future requests."""

nisystemlink/clients/dataframe/models/_data_frame.py

@@ -0,0 +1,57 @@
+from typing import List, Optional
+
+from nisystemlink.clients.core._uplink._json_model import JsonModel
+
+
+class DataFrame(JsonModel):
+    """Data read from or to be written to a table.
+
+    Values may be ``None`` (if the column is of type ``NULLABLE``) or encoded as
+    a string in a format according to each column's datatype:
+
+    * BOOL: One of ``"true"`` or ``"false"``, case-insensitive.
+    * INT32: Any integer number in the range [-2147483648, 2147483647],
+      surrounded by quotes.
+    * INT64: Any integer number in the range [-9223372036854775808,
+      9223372036854775807], surrounded by quotes.
+    * FLOAT32: A decimal number using a period for the decimal point, optionally
+      in scientific notation, in the range [-3.40282347E+38, 3.40282347E+38],
+      surrounded by quotes. Not all values within the range can be represented
+      with 32 bits. To preserve the exact binary encoding of the value when
+      converting to a string, clients should serialize 9 digits after the
+      decimal. Instead of a number, the value may be ``"NaN"`` (not a number),
+      ``"Infinity"`` (positive infinity), or ``"-Infinity"`` (negative
+      infinity), case-sensitive.
+    * FLOAT64: A decimal number using a period for the decimal point, optionally
+      in scientific notation, in the range [-1.7976931348623157E+308,
+      1.7976931348623157E+308], surrounded by quotes. Not all values within the
+      range can be represented with 64 bits. To preserve the exact binary
+      encoding of the value when converting to a string, clients should
+      serialize 17 digits after the decimal. Instead of a number, the value may
+      be ``"NaN"`` (not a number), ``"Infinity"`` (positive infinity), or
+      ``"-Infinity"`` (negative infinity), case-sensitive.
+    * STRING: Any quoted string.
+    * TIMESTAMP: A date and time with millisecond precision in ISO-8601 format
+      and time zone. For example: ``"2022-08-19T16:17:30.123Z"``. If a time zone
+      is not provided, UTC is assumed. If a time zone other than UTC is
+      provided, the value will be converted to UTC. If more than three digits of
+      fractional seconds are provided, the time will be truncated to three
+      digits (i.e. milliseconds).
+
+    The format is the same as a serialized Pandas DataFrame with orient="split"
+    and index=False. See
+    https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.to_json.html.
+
+    When providing a DataFrame for appending rows, any of the table's columns
+    not specified will receive a value of ``None``. If any such columns aren't
+    nullable, an error will be returned. If the entire columns property is left
+    out, each row is assumed to contain all columns in the order specified when
+    the table was created.
+    """
+
+    columns: Optional[List[str]] = None
+    """The names and order of the columns included in the data frame."""
+
+    data: List[List[Optional[str]]]
+    """The data for each row with the order specified in the columns property.
+    Must contain a value for each column in the columns property."""

nisystemlink/clients/dataframe/models/_paged_table_rows.py

@@ -0,0 +1,13 @@
+from nisystemlink.clients.core._uplink._with_paging import WithPaging
+
+from ._data_frame import DataFrame
+
+
+class PagedTableRows(WithPaging):
+    """Contains the result of a query for rows of data."""
+
+    frame: DataFrame
+    """The data frame containing the rows of data."""
+
+    total_row_count: int
+    """The total number of rows matched by the query across all pages of results."""

poetry.lock

@@ -3,7 +3,11 @@ name = "nisystemlink-clients"
 version = "0.7.0"
 description = "NI-SystemLink Python API"
 authors = ["National Instruments"]
-maintainers = ["Carson Moore <carson.moore@ni.com>", "Paul Spangler <paul.spangler@ni.com>", "Cameron Waterman <cameron.waterman@ni.com>"]
+maintainers = [
+    "Carson Moore <carson.moore@ni.com>",
+    "Paul Spangler <paul.spangler@ni.com>",
+    "Cameron Waterman <cameron.waterman@ni.com>",
+]
 keywords = ["nisystemlink", "systemlink"]
 license = "MIT"
 readme = "README.rst"
@@ -42,6 +46,7 @@ mypy                = "^0.982"
 flake8-docstrings   = "^1.6.0"
 poethepoet          = "^0.16.4"
 types-requests      = "^2.28.11.4"
+responses           = "^0.22.0"
 
 [tool.poe.tasks]
 test    = "pytest tests -m \"(not slow) and (not cloud) and (not enterprise)\""
@@ -67,10 +72,10 @@ markers = [
 exclude = ".*\\.pyi"
 
 [tool.semantic_release]
-branch               = "master"
-version_toml         = ["pyproject.toml:tool.poetry.version"]
-build_command        = "poetry build"
-major_on_zero        = false
+branch        = "master"
+version_toml  = ["pyproject.toml:tool.poetry.version"]
+build_command = "poetry build"
+major_on_zero = false
 
 [build-system]
 requires      = ["poetry-core"]

pyproject.toml

@@ -3,6 +3,7 @@
 from typing import List, Optional
 
 import pytest  # type: ignore
+import responses
 from nisystemlink.clients.core import ApiException
 from nisystemlink.clients.dataframe import DataFrameClient
 from nisystemlink.clients.dataframe import models
@@ -276,3 +277,73 @@ def test__modify_tables__returns_partial_success(self, client: DataFrameClient):
         assert response.modified_table_ids == [id]
         assert response.failed_modifications == [updates[1]]
         assert len(response.error.inner_errors) == 1
+
+    def test__read_and_write_data__works(self, client: DataFrameClient, create_table):
+        id = create_table(
+            models.CreateTableRequest(
+                columns=[
+                    models.Column(
+                        name="index",
+                        data_type=models.DataType.Int32,
+                        column_type=models.ColumnType.Index,
+                    ),
+                    models.Column(
+                        name="value",
+                        data_type=models.DataType.Float64,
+                        column_type=models.ColumnType.Nullable,
+                    ),
+                    models.Column(
+                        name="ignore_me",
+                        data_type=models.DataType.Bool,
+                        column_type=models.ColumnType.Nullable,
+                    ),
+                ]
+            )
+        )
+
+        frame = models.DataFrame(
+            columns=["index", "value", "ignore_me"],
+            data=[["1", "3.3", "True"], ["2", None, "False"], ["3", "1.1", "True"]],
+        )
+
+        client.append_table_data(
+            id, models.AppendTableDataRequest(frame=frame, end_of_data=True)
+        )
+
+        # TODO: Remove mock when service supports flushing
+        with responses.RequestsMock() as rsps:
+            rsps.add(
+                responses.GET,
+                f"{client.session.base_url}tables/{id}/data",
+                json={
+                    "frame": {
+                        "columns": ["index", "value"],
+                        "data": [["3", "1.1"], ["1", "3.3"], ["2", None]],
+                    },
+                    "totalRowCount": 3,
+                    "continuationToken": None,
+                },
+            )
+
+            response = client.get_table_data(
+                id, columns=["index", "value"], order_by=["value"]
+            )
+
+            assert response.total_row_count == 3
+            assert response.frame == models.DataFrame(
+                columns=["index", "value"],
+                data=[["3", "1.1"], ["1", "3.3"], ["2", None]],
+            )
+
+    def test__write_invalid_data__raises(
+        self, client: DataFrameClient, test_tables: List[str]
+    ):
+        id = test_tables[0]
+
+        frame = models.DataFrame(
+            columns=["index", "non_existent_column"],
+            data=[["1", "2"], ["2", "2"], ["3", "3"]],
+        )
+
+        with pytest.raises(ApiException, match="400 Bad Request"):
+            client.append_table_data(id, models.AppendTableDataRequest(frame=frame))