Merge branch '829-no-more-torchtext-dependency' into 756-add-notebook-for-eulaw

# Conflicts:
#	dianna/utils/downloader.py

Author: cwmeijer

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.5.0
+current_version = 1.6.0
 
 [comment]
 comment = The contents of this file cannot be merged with that of setup.cfg until https://github.com/c4urself/bump2version/issues/185 is resolved

.github/actions/install-python-and-package/action.yml

@@ -36,11 +36,12 @@ runs:
         python -m pip install --upgrade pip setuptools wheel
 
     # only necessary on linux to avoid bloated installs
+    # pining the version of torch is temporary, see see #829
     - name: Install tensorflow/pytorch cpu version
       if: runner.os == 'Linux' && steps.cache-python-env.outputs.cache-hit != 'true'
       shell: bash {0}
       run: |
-        python -m pip install torch --index-url https://download.pytorch.org/whl/cpu
+        python -m pip install torch==2.3.1 --index-url https://download.pytorch.org/whl/cpu
         python -m pip install tensorflow-cpu
 
     - name: Install DIANNA

.github/workflows/build.yml

@@ -26,8 +26,8 @@ jobs:
         with:
           python-version: '3.11'
 
-      - name: Run unit tests including downloader
-        run: pytest -v --downloader
+      - name: Run unit tests
+        run: pytest -v
 
       - name: Verify that we can build the package
         run: python setup.py sdist bdist_wheel
@@ -53,11 +53,24 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Run unit tests
-        run: python -m pytest -v --downloader
+        run: python -m pytest -v
 
       - name: Verify that we can build the package
         run: python setup.py sdist bdist_wheel
 
+  test_downloader:
+    name: Test file downloader
+    if: github.event.pull_request.draft == false
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - uses: ./.github/actions/install-python-and-package
+        with:
+          python-version: '3.11'
+          extras-require: dev
+      - name: Run downloader test
+        run: python -m pytest -v --downloader -k downloader
+
   test_dashboard:
     name: Test dashboard
     if: github.event.pull_request.draft == false

.github/workflows/linting.yml

@@ -33,4 +33,4 @@ jobs:
 
       - name: Check code style
         run: |
-          ruff dianna tests
+          ruff check dianna tests

.github/workflows/notebooks.yml

@@ -17,17 +17,19 @@ on:
 jobs:
 
   notebooks:
-    name: Run notebooks on (3.10, ${{ matrix.os }})
+    name: Run notebooks on (${{ matrix.python-version }}, ${{ matrix.os }})
     if: github.event.pull_request.draft == false
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: false
       matrix:
         os: ['ubuntu-latest', 'macos-latest', 'windows-latest']
+        python-version: ['3.10']
     steps:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/install-python-and-package
         with:
+          python-version: ${{ matrix.python-version }}
           extras-require: dev,notebooks
       - name: Run tutorial notebooks
         run: pytest --nbmake tutorials

CITATION.cff

@@ -53,7 +53,7 @@ authors:
     name-particle: "van der"
 
 doi: 10.5281/zenodo.5801485
-version: "1.5.0"
+version: "1.6.0"
 repository-code: "https://github.com/dianna-ai/dianna"
 keywords:
   - XAI

README.md

@@ -32,7 +32,7 @@
 
 __author__ = 'DIANNA Team'
 __email__ = 'dianna-ai@esciencecenter.nl'
-__version__ = '1.5.0'
+__version__ = '1.6.0'
 
 
 def explain_timeseries(model_or_function: Union[Callable, str],

dianna/__init__.py

@@ -8,3 +8,6 @@ secondaryBackgroundColor="#e4f3f9"
 
 [browser]
 gatherUsageStats = false
+
+[client]
+showSidebarNavigation = false

dianna/dashboard/.streamlit/config.toml

@@ -1,10 +1,11 @@
+import importlib
 import streamlit as st
-from _shared import add_sidebar_logo
 from _shared import data_directory
+from streamlit_option_menu import option_menu
 
 st.set_page_config(page_title="Dianna's dashboard",
                    page_icon='📊',
-                   layout='centered',
+                   layout='wide',
                    initial_sidebar_state='auto',
                    menu_items={
                        'Get help':
@@ -16,26 +17,68 @@
                         'https://github.com/dianna-ai/dianna')
                    })
 
-add_sidebar_logo()
+# Define dictionary of dashboard pages
+pages = {
+    "Home": "home",
+    "Images": "pages.Images",
+    "Tabular": "pages.Tabular",
+    "Text": "pages.Text",
+    "Time series": "pages.Time_series"
+}
 
-st.image(str(data_directory / 'logo.png'))
+# Set up the top menu
+selected = option_menu(
+    menu_title=None,
+    options=list(pages.keys()),
+    icons=["house", "camera", "table", "alphabet", "clock"],
+    menu_icon="cast",
+    default_index=0,
+    orientation="horizontal"
+)
 
-st.markdown("""
-DIANNA is a Python package that brings explainable AI (XAI) to your research project.
-It wraps carefully selected XAI methods in a simple, uniform interface. It's built by,
-with and for (academic) researchers and research software engineers working on machine
-learning projects.
+# Display the content of the selected page
+if selected == "Home":
+    st.image(str(data_directory / 'logo.png'))
 
-### Pages
+    st.markdown("""
+    DIANNA is a Python package that brings explainable AI (XAI) to your research project.
+    It wraps carefully selected XAI methods in a simple, uniform interface. It's built by,
+    with and for (academic) researchers and research software engineers working on machine
+    learning projects.
 
-- <a href="/Images" target="_parent">Images</a>
-- <a href="/Text" target="_parent">Text</a>
-- <a href="/Time_series" target="_parent">Time series</a>
+    ### Pages
 
+    - <a href="/Images" target="_parent">Image data</a>
+    - <a href="/Tabular" target="_parent">Tabular data</a>
+    - <a href="/Text" target="_parent">Text data</a>
+    - <a href="/Time_series" target="_parent">Time series data</a>
 
-### More information
 
-- [Source code](https://github.com/dianna-ai/dianna)
-- [Documentation](https://dianna.readthedocs.io/)
-""",
-            unsafe_allow_html=True)
+    ### More information
+
+    - [Source code](https://github.com/dianna-ai/dianna)
+    - [Documentation](https://dianna.readthedocs.io/)
+    """,
+                unsafe_allow_html=True)
+
+else:
+    # Dynamically import and execute the page
+    page_module = pages[selected]
+    # Make sure that all variables are reset when switching page
+    if selected != 'Images':
+        for k in st.session_state.keys():
+            if 'Image' in k:
+                st.session_state.pop(k, None)
+    if selected != 'Tabular':
+        for k in st.session_state.keys():
+            if 'Tabular' in k:
+                st.session_state.pop(k, None)
+    if selected != 'Text':
+        for k in st.session_state.keys():
+            if 'Text' in k:
+                st.session_state.pop(k, None)
+    if selected != 'Time series':
+        for k in st.session_state.keys():
+            if 'TS' in k:
+                st.session_state.pop(k, None)
+    page = importlib.import_module(page_module)

dianna/dashboard/Home.py

@@ -1,6 +1,15 @@
 from pathlib import Path
 import numpy as np
 import onnx
+import pandas as pd
+
+
+def load_data(file):
+    """Open data from a file and returns it as pandas DataFrame."""
+    df = pd.read_csv(file, parse_dates=True)
+    # Add index column
+    df.insert(0, 'Index', df.index)
+    return df
 
 
 def preprocess_function(image):
@@ -29,3 +38,7 @@ def load_labels(file):
     if labels is None or labels == ['']:
         raise ValueError(labels)
     return labels
+
+
+def load_training_data(file):
+    return np.float32(np.load(file, allow_pickle=False))

dianna/dashboard/_model_utils.py

@@ -0,0 +1,57 @@
+import tempfile
+import numpy as np
+import streamlit as st
+from dianna import explain_tabular
+from dianna.utils.onnx_runner import SimpleModelRunner
+
+
+@st.cache_data
+def predict(*, model, tabular_input):
+    model_runner = SimpleModelRunner(model)
+    predictions = model_runner(tabular_input.reshape(1,-1).astype(np.float32))
+    return predictions
+
+
+@st.cache_data
+def _run_rise_tabular(_model, table, training_data, **kwargs):
+    relevances = explain_tabular(
+        _model,
+        table,
+        method='RISE',
+        training_data=training_data,
+        **kwargs,
+    )
+    return relevances
+
+
+@st.cache_data
+def _run_lime_tabular(_model, table, training_data, _feature_names, **kwargs):
+    relevances = explain_tabular(
+        _model,
+        table,
+        method='LIME',
+        training_data=training_data,
+        feature_names=_feature_names,
+        **kwargs,
+    )
+    return relevances
+
+@st.cache_data
+def _run_kernelshap_tabular(model, table, training_data, **kwargs):
+    # Kernelshap interface is different. Write model to temporary file.
+    with tempfile.NamedTemporaryFile() as f:
+        f.write(model)
+        f.flush()
+        relevances = explain_tabular(f.name,
+                table,
+                method='KernelSHAP',
+                training_data=training_data,
+                **kwargs)
+    return relevances[0]
+
+
+explain_tabular_dispatcher = {
+    'RISE': _run_rise_tabular,
+    'LIME': _run_lime_tabular,
+    'KernelSHAP': _run_kernelshap_tabular
+}

dianna/dashboard/_models_tabular.py

@@ -21,6 +21,10 @@ def predict(*, model, ts_data):
 
 @st.cache_data
 def _run_rise_timeseries(_model, ts_data, **kwargs):
+    # convert streamlit kwarg requirement back to dianna kwarg requirement
+    if "_preprocess_function" in kwargs:
+        kwargs["preprocess_function"] = kwargs["_preprocess_function"]
+        del kwargs["_preprocess_function"]
 
     def run_model(ts_data):
         return predict(model=_model, ts_data=ts_data)
@@ -37,6 +41,10 @@ def run_model(ts_data):
 
 @st.cache_data
 def _run_lime_timeseries(_model, ts_data, **kwargs):
+    # convert streamlit kwarg requirement back to dianna kwarg requirement
+    if "_preprocess_function" in kwargs:
+        kwargs["preprocess_function"] = kwargs["_preprocess_function"]
+        del kwargs["_preprocess_function"]
 
     def run_model(ts_data):
         return predict(model=_model, ts_data=ts_data)