Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

ragulate: ci and release process #531

Merged
merged 11 commits into from
Jun 28, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/changes-filter.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -38,3 +38,5 @@ libs_knowledge_graph:
- "libs/knowledge-graph/**"
libs_knowledge_store:
- "libs/knowledge-store/**"
libs_ragulate:
- "libs/ragulate/**"
6 changes: 6 additions & 0 deletions .github/workflows/ci-unit-tests.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ jobs:
libs_llamaindex: ${{ steps.filter.outputs.libs_llamaindex }}
libs_knowledge_store: ${{ steps.filter.outputs.libs_knowledge_store }}
libs_knowledge_graph: ${{ steps.filter.outputs.libs_knowledge_graph }}
libs_ragulate: ${{ steps.filter.outputs.libs_ragulate }}
integration_tests: ${{ steps.filter.outputs.integration_tests }}
steps:
- uses: actions/checkout@v4
Expand Down Expand Up @@ -76,8 +77,13 @@ jobs:
uses: ./.github/actions/lint

- name: "Unit tests (root)"
if: ${{ needs.preconditions.outputs.libs_langchain == 'true' || needs.preconditions.outputs.libs_colbert == 'true' || needs.preconditions.outputs.libs_llamaindex == 'true' }}
run: tox -e unit-tests && rm -rf .tox

- name: "Unit tests (ragulate)"
if: ${{ needs.preconditions.outputs.libs_ragulate == 'true' && matrix.python-version != '3.9' }}
run: tox -e unit-tests -c libs/ragulate && rm -rf libs/ragulate/.tox

- name: "Unit tests (colbert)"
if: ${{ needs.preconditions.outputs.libs_colbert == 'true' }}
run: tox -e unit-tests -c libs/colbert && rm -rf libs/colbert/.tox
Expand Down
25 changes: 25 additions & 0 deletions .github/workflows/release-ragstack-ragulate.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
name: Release RAGStack Ragulate package

on:
push:
tags:
- "ragstack-ai-ragulate-*"

jobs:
release:
name: Release package
runs-on: ubuntu-latest
permissions: write-all
steps:
- name: Check out the repo
uses: actions/checkout@v4

- name: "Setup: Python 3.11"
uses: ./.github/actions/setup-python

- name: Release
uses: ./.github/actions/release-package
with:
package-directory: "libs/ragulate"
pypi-token: ${{ secrets.PYPI_API_TOKEN }}
pypi-test-token: ${{ secrets.PYPI_TEST_API_TOKEN }}
1 change: 1 addition & 0 deletions .gitignore
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,7 @@ ipython_config.py
# commonly ignored for libraries.
# https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
poetry.lock
!libs/ragulate/python.lock

# pdm
# Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
Expand Down
14 changes: 14 additions & 0 deletions libs/ragulate/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
# default dataset download location
datasets/
!ragulate/datasets/

# database files
*.sqlite

# generated images
*.png
!images/
!images/*.png

# streamlit cache
.streamlit/
4,229 changes: 4,229 additions & 0 deletions libs/ragulate/poetry.lock

Large diffs are not rendered by default.

13 changes: 7 additions & 6 deletions libs/ragulate/pyproject.toml
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
[tool.poetry]
name = "ragstack_ragulate"
version = "0.0.1"
name = "ragstack-ai-ragulate"
version = "0.0.0"
description = "A tool for evaluating RAG pipelines"
authors = ["Eric Pinzur <epinzur@gmail.com>"]
license = "Apache 2.0"
authors = ["DataStax"]
readme = "README.md"
packages = [{include = "ragstack_ragulate"}]
maintainers = ["Eric Pinzur <epinzur@gmail.com>", "Sai Kolasani"]
repository = "https://github.com/datastax/ragstack-ai"
documentation = "https://docs.datastax.com/en/ragstack"
packages = [{ include = "ragstack_ragulate" }]

[tool.poetry.dependencies]
python = ">=3.10,<3.13"
Expand All @@ -21,7 +22,7 @@ python-dotenv = ">=1.0.0"
tqdm = ">=4.66.1"
pyyaml = "^6.0.1"
cerberus = "^1.3.5"
pydantic = "^2.7.3"
pydantic = "^2.7.4"
setuptools = "^70.0.0"
aiofiles = "^24.1.0"
seaborn = "^0.13.2"
Expand Down
3 changes: 3 additions & 0 deletions libs/ragulate/ragstack_ragulate/datasets/utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,9 @@

def find_dataset(name: str) -> BaseDataset:
root_path = "datasets"
if not os.path.exists(root_path):
raise ValueError("please download a dataset before using ingest or query")

name = inflection.underscore(name)
for kind in os.listdir(root_path):
kind_path = os.path.join(root_path, kind)
Expand Down
5 changes: 5 additions & 0 deletions libs/ragulate/tests/unit_tests/test_config_validation.py
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
import unittest
import os

from ragstack_ragulate.config.config_parser import ConfigParser
from ragstack_ragulate.config.config_schema_0_1 import ConfigSchema_0_1
Expand Down Expand Up @@ -66,6 +67,8 @@ def test_full_config(self):
"answer_correctness": {"enabled": True, "eval_llm": "llama3"},
},
}
os.makedirs(os.path.join("datasets", "llama", "blockchain_solana"), exist_ok=True)
os.makedirs(os.path.join("datasets", "llama", "braintrust_coda_help_desk"), exist_ok=True)
parser = ConfigParser(config_schema=ConfigSchema_0_1(), config=config)

for field, errors in parser.errors.items():
Expand Down Expand Up @@ -110,6 +113,8 @@ def test_minimal_config(self):
"answer_correctness",
],
}
os.makedirs(os.path.join("datasets", "llama", "blockchain_solana"), exist_ok=True)
os.makedirs(os.path.join("datasets", "llama", "other_dataset"), exist_ok=True)
parser = ConfigParser(config_schema=ConfigSchema_0_1(), config=config)

for field, errors in parser.errors.items():
Expand Down
12 changes: 12 additions & 0 deletions libs/ragulate/tox.ini
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
[tox]
min_version = 4.0
envlist = py311

[testenv:unit-tests]
description = run unit tests
deps =
poetry
commands =
poetry install
poetry build
poetry run pytest --disable-warnings {toxinidir}/tests/unit_tests
4 changes: 3 additions & 1 deletion scripts/release.sh
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
#!/bin/bash
set -e
ALL_PACKAGES="ragstack-ai, ragstack-ai-langchain, ragstack-ai-llamaindex, ragstack-ai-colbert, ragstack-ai-knowledge-graph, ragstack-ai-knowledge-store"
ALL_PACKAGES="ragstack-ai, ragstack-ai-langchain, ragstack-ai-llamaindex, ragstack-ai-colbert, ragstack-ai-knowledge-graph, ragstack-ai-knowledge-store, ragstack-ai-ragulate"
version=$1
package=$2
if [[ -z "$version" || -z "$package" ]]; then
Expand Down Expand Up @@ -28,6 +28,8 @@ elif [ "$package" == "ragstack-ai-knowledge-graph" ]; then
directory="libs/knowledge-graph"
elif [ "$package" == "ragstack-ai-knowledge-store" ]; then
directory="libs/knowledge-store"
elif [ "$package" == "ragstack-ai-ragulate" ]; then
directory="libs/ragulate"
else
echo "Invalid package. Please choose from: $ALL_PACKAGES."
exit 1
Expand Down