ragulate: ci and release process

.github/changes-filter.yaml

@@ -38,3 +38,5 @@ libs_knowledge_graph:
   - "libs/knowledge-graph/**"
 libs_knowledge_store:
   - "libs/knowledge-store/**"
+libs_ragulate:
+  - "libs/ragulate/**"

.github/workflows/ci-unit-tests.yml

@@ -19,6 +19,7 @@ jobs:
       libs_llamaindex: ${{ steps.filter.outputs.libs_llamaindex }}
       libs_knowledge_store: ${{ steps.filter.outputs.libs_knowledge_store }}
       libs_knowledge_graph: ${{ steps.filter.outputs.libs_knowledge_graph }}
+      libs_ragulate: ${{ steps.filter.outputs.libs_ragulate }}
       integration_tests: ${{ steps.filter.outputs.integration_tests }}
     steps:
       - uses: actions/checkout@v4
@@ -76,8 +77,13 @@ jobs:
         uses: ./.github/actions/lint
 
       - name: "Unit tests (root)"
+        if: ${{ needs.preconditions.outputs.libs_langchain == 'true' || needs.preconditions.outputs.libs_colbert == 'true' || needs.preconditions.outputs.libs_llamaindex == 'true' }}
         run: tox -e unit-tests && rm -rf .tox
 
+      - name: "Unit tests (ragulate)"
+        if: ${{ needs.preconditions.outputs.libs_ragulate == 'true' && matrix.python-version != '3.9' }}
+        run: tox -e unit-tests -c libs/ragulate && rm -rf libs/ragulate/.tox
+
       - name: "Unit tests (colbert)"
         if: ${{ needs.preconditions.outputs.libs_colbert == 'true' }}
         run: tox -e unit-tests -c libs/colbert && rm -rf libs/colbert/.tox

.github/workflows/release-ragstack-ragulate.yml

@@ -0,0 +1,25 @@
+name: Release RAGStack Ragulate package
+
+on:
+  push:
+    tags:
+      - "ragstack-ai-ragulate-*"
+
+jobs:
+  release:
+    name: Release package
+    runs-on: ubuntu-latest
+    permissions: write-all
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v4
+
+      - name: "Setup: Python 3.11"
+        uses: ./.github/actions/setup-python
+
+      - name: Release
+        uses: ./.github/actions/release-package
+        with:
+          package-directory: "libs/ragulate"
+          pypi-token: ${{ secrets.PYPI_API_TOKEN }}
+          pypi-test-token: ${{ secrets.PYPI_TEST_API_TOKEN }}

.gitignore

@@ -100,6 +100,7 @@ ipython_config.py
 #   commonly ignored for libraries.
 #   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
 poetry.lock
+!libs/ragulate/python.lock
 
 # pdm
 #   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.

libs/ragulate/.gitignore

@@ -0,0 +1,14 @@
+# default dataset download location
+datasets/
+!ragulate/datasets/
+
+# database files
+*.sqlite
+
+# generated images
+*.png
+!images/
+!images/*.png
+
+# streamlit cache
+.streamlit/

libs/ragulate/pyproject.toml

@@ -1,12 +1,13 @@
 [tool.poetry]
-name = "ragstack_ragulate"
-version = "0.0.1"
+name = "ragstack-ai-ragulate"
+version = "0.0.0"
 description = "A tool for evaluating RAG pipelines"
-authors = ["Eric Pinzur <epinzur@gmail.com>"]
 license = "Apache 2.0"
+authors = ["DataStax"]
 readme = "README.md"
-packages = [{include = "ragstack_ragulate"}]
-maintainers = ["Eric Pinzur <epinzur@gmail.com>", "Sai Kolasani"]
+repository = "https://github.com/datastax/ragstack-ai"
+documentation = "https://docs.datastax.com/en/ragstack"
+packages = [{ include = "ragstack_ragulate" }]
 
 [tool.poetry.dependencies]
 python = ">=3.10,<3.13"
@@ -21,7 +22,7 @@ python-dotenv = ">=1.0.0"
 tqdm = ">=4.66.1"
 pyyaml = "^6.0.1"
 cerberus = "^1.3.5"
-pydantic = "^2.7.3"
+pydantic = "^2.7.4"
 setuptools = "^70.0.0"
 aiofiles = "^24.1.0"
 seaborn = "^0.13.2"

libs/ragulate/ragstack_ragulate/datasets/utils.py

@@ -9,6 +9,9 @@
 
 def find_dataset(name: str) -> BaseDataset:
     root_path = "datasets"
+    if not os.path.exists(root_path):
+        raise ValueError("please download a dataset before using ingest or query")
+
     name = inflection.underscore(name)
     for kind in os.listdir(root_path):
         kind_path = os.path.join(root_path, kind)

libs/ragulate/tests/unit_tests/test_config_validation.py

@@ -1,4 +1,5 @@
 import unittest
+import os
 
 from ragstack_ragulate.config.config_parser import ConfigParser
 from ragstack_ragulate.config.config_schema_0_1 import ConfigSchema_0_1
@@ -66,6 +67,8 @@ def test_full_config(self):
                 "answer_correctness": {"enabled": True, "eval_llm": "llama3"},
             },
         }
+        os.makedirs(os.path.join("datasets", "llama", "blockchain_solana"), exist_ok=True)
+        os.makedirs(os.path.join("datasets", "llama", "braintrust_coda_help_desk"), exist_ok=True)
         parser = ConfigParser(config_schema=ConfigSchema_0_1(), config=config)
 
         for field, errors in parser.errors.items():
@@ -110,6 +113,8 @@ def test_minimal_config(self):
                 "answer_correctness",
             ],
         }
+        os.makedirs(os.path.join("datasets", "llama", "blockchain_solana"), exist_ok=True)
+        os.makedirs(os.path.join("datasets", "llama", "other_dataset"), exist_ok=True)
         parser = ConfigParser(config_schema=ConfigSchema_0_1(), config=config)
 
         for field, errors in parser.errors.items():

libs/ragulate/tox.ini

@@ -0,0 +1,12 @@
+[tox]
+min_version = 4.0
+envlist = py311
+
+[testenv:unit-tests]
+description = run unit tests
+deps =
+    poetry
+commands =
+    poetry install
+    poetry build
+    poetry run pytest --disable-warnings {toxinidir}/tests/unit_tests

scripts/release.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 set -e
-ALL_PACKAGES="ragstack-ai, ragstack-ai-langchain, ragstack-ai-llamaindex, ragstack-ai-colbert, ragstack-ai-knowledge-graph, ragstack-ai-knowledge-store"
+ALL_PACKAGES="ragstack-ai, ragstack-ai-langchain, ragstack-ai-llamaindex, ragstack-ai-colbert, ragstack-ai-knowledge-graph, ragstack-ai-knowledge-store, ragstack-ai-ragulate"
 version=$1
 package=$2
 if [[ -z "$version" || -z "$package" ]]; then
@@ -28,6 +28,8 @@ elif [ "$package" == "ragstack-ai-knowledge-graph" ]; then
     directory="libs/knowledge-graph"
 elif [ "$package" == "ragstack-ai-knowledge-store" ]; then
     directory="libs/knowledge-store"
+elif [ "$package" == "ragstack-ai-ragulate" ]; then
+    directory="libs/ragulate"
 else
     echo "Invalid package. Please choose from: $ALL_PACKAGES."
     exit 1