diff --git a/dianna/__init__.py b/dianna/__init__.py index 8070944d..5e9d0c42 100644 --- a/dianna/__init__.py +++ b/dianna/__init__.py @@ -131,7 +131,7 @@ def explain_text(model_or_function: Union[Callable, def explain_tabular(model_or_function: Union[Callable, str], input_tabular: np.ndarray, method: str, - labels=(1, ), + labels=None, **kwargs) -> np.ndarray: """Explain tabular (input_text) given a model and a chosen method. diff --git a/dianna/methods/kernelshap_tabular.py b/dianna/methods/kernelshap_tabular.py index 336fadd8..19b8125c 100644 --- a/dianna/methods/kernelshap_tabular.py +++ b/dianna/methods/kernelshap_tabular.py @@ -34,7 +34,8 @@ def __init__( weighted kmeans """ if training_data_kmeans: - self.training_data = shap.kmeans(training_data, training_data_kmeans) + self.training_data = shap.kmeans(training_data, + training_data_kmeans) else: self.training_data = training_data self.feature_names = feature_names @@ -65,17 +66,15 @@ def explain( An array (np.ndarray) containing the KernelExplainer explanations for each class. """ init_instance_kwargs = utils.get_kwargs_applicable_to_function( - KernelExplainer, kwargs - ) - self.explainer = KernelExplainer( - model_or_function, self.training_data, link, **init_instance_kwargs - ) + KernelExplainer, kwargs) + self.explainer = KernelExplainer(model_or_function, self.training_data, + link, **init_instance_kwargs) explain_instance_kwargs = utils.get_kwargs_applicable_to_function( - self.explainer.shap_values, kwargs - ) + self.explainer.shap_values, kwargs) - saliency = self.explainer.shap_values(input_tabular, **explain_instance_kwargs) + saliency = self.explainer.shap_values(input_tabular, + **explain_instance_kwargs) if self.mode == 'regression': saliency = saliency[0] diff --git a/dianna/methods/lime_tabular.py b/dianna/methods/lime_tabular.py index 23db971a..f6b4b1fd 100644 --- a/dianna/methods/lime_tabular.py +++ b/dianna/methods/lime_tabular.py @@ -1,6 +1,8 @@ """LIME tabular explainer.""" +import sys from typing import Iterable from typing import List +from typing import Optional from typing import Union import numpy as np from lime.lime_tabular import LimeTabularExplainer @@ -58,12 +60,10 @@ def __init__( """ self.mode = mode init_instance_kwargs = utils.get_kwargs_applicable_to_function( - LimeTabularExplainer, kwargs - ) + LimeTabularExplainer, kwargs) # temporary solution for setting num_features and top_labels self.num_features = len(feature_names) - self.top_labels = len(class_names) self.explainer = LimeTabularExplainer( training_data, @@ -83,7 +83,7 @@ def explain( self, model_or_function: Union[str, callable], input_tabular: np.array, - labels: Iterable[int] = (1,), + labels: Optional[Iterable[int]] = None, num_samples: int = 5000, **kwargs, ) -> np.array: @@ -93,7 +93,7 @@ def explain( model_or_function (callable or str): The function that runs the model to be explained or the path to a ONNX model on disk. input_tabular (np.ndarray): Data to be explained. - labels (Iterable(int), optional): Indices of classes to be explained. + labels (Iterable(int)): Indices of classes to be explained. num_samples (int, optional): Number of samples kwargs: These parameters are passed on @@ -105,15 +105,14 @@ def explain( """ # run the explanation. explain_instance_kwargs = utils.get_kwargs_applicable_to_function( - self.explainer.explain_instance, kwargs - ) + self.explainer.explain_instance, kwargs) runner = utils.get_function(model_or_function) explanation = self.explainer.explain_instance( input_tabular, runner, labels=labels, - top_labels=self.top_labels, + top_labels=sys.maxsize, num_features=self.num_features, num_samples=num_samples, **explain_instance_kwargs, @@ -126,10 +125,13 @@ def explain( elif self.mode == 'classification': # extract scores from lime explainer saliency = [] - for i in range(self.top_labels): + for i in range(len(explanation.local_exp.items())): local_exp = sorted(explanation.local_exp[i]) # shape of local_exp [(index, saliency)] selected_saliency = [x[1] for x in local_exp] saliency.append(selected_saliency[:]) + else: + raise ValueError(f'Unsupported mode "{self.mode}"') + return np.array(saliency) diff --git a/dianna/methods/lime_timeseries.py b/dianna/methods/lime_timeseries.py index d1851e2c..d6b590d7 100644 --- a/dianna/methods/lime_timeseries.py +++ b/dianna/methods/lime_timeseries.py @@ -81,7 +81,7 @@ def explain( # wrap up the input model or function using the runner runner = utils.get_function( model_or_function, preprocess_function=self.preprocess_function) - masks = generate_time_series_masks(input_timeseries, + masks = generate_time_series_masks(input_timeseries.shape, num_samples, p_keep=0.1) # NOTE: Required by `lime_base` explainer since the first instance must be the original data diff --git a/dianna/methods/rise_tabular.py b/dianna/methods/rise_tabular.py new file mode 100644 index 00000000..37ddac15 --- /dev/null +++ b/dianna/methods/rise_tabular.py @@ -0,0 +1,111 @@ +"""RISE tabular explainer.""" +from typing import Iterable +from typing import List +from typing import Optional +from typing import Union +import numpy as np +from dianna import utils +from dianna.utils.maskers import generate_tabular_masks +from dianna.utils.maskers import mask_data_tabular +from dianna.utils.predict import make_predictions +from dianna.utils.rise_utils import normalize + + +class RISETabular: + """RISE explainer for tabular data.""" + + def __init__( + self, + training_data: np.array, + mode: str = "classification", + feature_names: List[str] = None, + categorical_features: List[int] = None, + n_masks: int = 1000, + feature_res: int = 8, + p_keep: float = 0.5, + preprocess_function: Optional[callable] = None, + class_names=None, + keep_masks: bool = False, + keep_masked: bool = False, + keep_predictions: bool = False, + ) -> np.ndarray: + """RISE initializer. + + Args: + n_masks: Number of masks to generate. + feature_res: Resolution of features in masks. + p_keep: Fraction of input data to keep in each mask (Default: auto-tune this value). + preprocess_function: Function to preprocess input data with + categorical_features: list of categorical features + class_names: Names of the classes + feature_names: Names of the features + mode: Either classification of regression + training_data: Training data used for imputation of masked features + keep_masks: keep masks in memory for the user to inspect + keep_masked: keep masked data in memory for the user to inspect + keep_predictions: keep model predictions in memory for the user to inspect + """ + self.training_data = training_data + self.n_masks = n_masks + self.feature_res = feature_res + self.p_keep = p_keep + self.preprocess_function = preprocess_function + self.masks = None + self.masked = None + self.predictions = None + self.keep_masks = keep_masks + self.keep_masked = keep_masked + self.keep_predictions = keep_predictions + self.mode = mode + + def explain( + self, + model_or_function: Union[str, callable], + input_tabular: np.array, + labels: Optional[Iterable[int]] = None, + mask_type: Optional[Union[str, callable]] = 'most_frequent', + batch_size: Optional[int] = 100, + ) -> np.array: + """Run the RISE explainer. + + Args: + model_or_function: The function that runs the model to be explained + or the path to a ONNX model on disk. + input_tabular: Data to be explained. + labels: Indices of classes to be explained. + num_samples: Number of samples + mask_type: Imputation strategy for masked features + batch_size: Number of samples to process by the model per batch + + Returns: + explanation: An Explanation object containing the LIME explanations for each class. + """ + # run the explanation. + runner = utils.get_function(model_or_function) + + masks = np.stack( + list( + generate_tabular_masks(input_tabular.shape, + number_of_masks=self.n_masks, + p_keep=self.p_keep))) + self.masks = masks if self.keep_masks else None + + masked = mask_data_tabular(input_tabular, + masks, + self.training_data, + mask_type=mask_type) + self.masked = masked if self.keep_masked else None + predictions = make_predictions(masked, runner, batch_size) + self.predictions = predictions if self.keep_predictions else None + n_labels = predictions.shape[1] + + masks_reshaped = masks.reshape(self.n_masks, -1) + + saliency = predictions.T.dot(masks_reshaped).reshape( + n_labels, *input_tabular.shape) + + if self.mode == 'regression': + return saliency[0] + + selected_saliency = saliency if labels is None else saliency[labels] + return normalize(selected_saliency, self.n_masks, self.p_keep) diff --git a/dianna/methods/rise_timeseries.py b/dianna/methods/rise_timeseries.py index 059d54d9..7479a8f0 100644 --- a/dianna/methods/rise_timeseries.py +++ b/dianna/methods/rise_timeseries.py @@ -68,7 +68,7 @@ def explain(self, runner = utils.get_function( model_or_function, preprocess_function=self.preprocess_function) - masks = generate_time_series_masks(input_timeseries, + masks = generate_time_series_masks(input_timeseries.shape, number_of_masks=self.n_masks, feature_res=self.feature_res, p_keep=self.p_keep) diff --git a/dianna/utils/maskers.py b/dianna/utils/maskers.py index 4c1bf916..5cb59a1a 100644 --- a/dianna/utils/maskers.py +++ b/dianna/utils/maskers.py @@ -5,10 +5,40 @@ import numpy as np from numpy import ndarray from skimage.transform import resize +from sklearn.impute import SimpleImputer + + +def generate_tabular_masks( + input_data_shape: tuple[int], + number_of_masks: int, + p_keep: float = 0.5, +): + """Generator function to create masks for tabular data. + + Args: + input_data_shape: Shape of the tabular data to be masked. + number_of_masks: Number of masks to generate. + p_keep: probability that any value should remain unmasked. + + Returns: + Single array containing all masks where the first dimension represents the batch. + """ + instance_length = np.product(input_data_shape) + + for i in range(number_of_masks): + n_masked = _determine_number_masked(p_keep, instance_length) + trues = n_masked * [False] + falses = (instance_length - n_masked) * [True] + options = trues + falses + yield np.random.choice( + a=options, + size=input_data_shape, + replace=False, + ) def generate_time_series_masks( - input_data: np.ndarray, + input_data_shape: tuple[int], number_of_masks: int, feature_res: int = 8, p_keep: float = 0.5, @@ -26,7 +56,7 @@ def generate_time_series_masks( For univariate data, only time step masks are returned. Args: - input_data: Timeseries data to be masked. + input_data_shape: Shape of the time series data to be masked. number_of_masks: Number of masks to generate. p_keep: the probability that any value remains unmasked. feature_res: Resolution of features in masks. @@ -34,8 +64,8 @@ def generate_time_series_masks( Returns: Single array containing all masks where the first dimension represents the batch. """ - if input_data.shape[-1] == 1: # univariate data - return generate_time_step_masks(input_data, + if input_data_shape[-1] == 1: # univariate data + return generate_time_step_masks(input_data_shape, number_of_masks, p_keep, number_of_features=feature_res) @@ -45,30 +75,30 @@ def generate_time_series_masks( number_of_time_step_masks = number_of_channel_masks number_of_combined_masks = number_of_masks - number_of_time_step_masks - number_of_channel_masks - time_step_masks = generate_time_step_masks(input_data, + time_step_masks = generate_time_step_masks(input_data_shape, number_of_time_step_masks, p_keep, feature_res) - channel_masks = generate_channel_masks(input_data, number_of_channel_masks, - p_keep) + channel_masks = generate_channel_masks(input_data_shape, + number_of_channel_masks, p_keep) # Product of two masks: we need sqrt p_keep to ensure correct resulting p_keep sqrt_p_keep = np.sqrt(p_keep) combined_masks = generate_time_step_masks( - input_data, number_of_combined_masks, + input_data_shape, number_of_combined_masks, sqrt_p_keep, feature_res) * generate_channel_masks( - input_data, number_of_combined_masks, sqrt_p_keep) + input_data_shape, number_of_combined_masks, sqrt_p_keep) return np.concatenate([time_step_masks, channel_masks, combined_masks], axis=0) -def generate_channel_masks(input_data: np.ndarray, number_of_masks: int, +def generate_channel_masks(input_data_shape: tuple[int], number_of_masks: int, p_keep: float): """Generate masks that mask one or multiple channels independently at a time.""" - number_of_channels = input_data.shape[1] + number_of_channels = input_data_shape[1] number_of_channels_masked = _determine_number_masked( p_keep, number_of_channels) - masked_data_shape = [number_of_masks] + list(input_data.shape) + masked_data_shape = [number_of_masks] + list(input_data_shape) masks = np.ones(masked_data_shape, dtype=bool) for i in range(number_of_masks): channels_to_mask = np.random.choice(number_of_channels, @@ -77,6 +107,43 @@ def generate_channel_masks(input_data: np.ndarray, number_of_masks: int, return masks +def mask_data_tabular(data: np.array, masks: np.array, training_data: np.array, + mask_type: Union[object, str]) -> np.array: + """Mask tabular data given using a set of masks. + + Args: + data: Input data. + masks: an array with shape [number_of_masks] + data.shape + mask_type: Masking strategy. Can be 'most_frequent', 'mean' or a function f(data, masks, training_data). + training_data: Data used to sample from for imputation of masked values. + + Returns: + Single array containing all masked input where the first dimension represents the batch. + """ + if isinstance(mask_type, str): + + def strategy(data, masks, training_data): + imputer = SimpleImputer(missing_values=np.nan, strategy=mask_type) + imputer.fit(training_data) + masked_data_list = [] + for mask in masks: + current_data = np.array(data) + current_data[~mask] = np.nan + current_data_masked = imputer.transform(current_data[None, + ...])[0] + masked_data_list.append(current_data_masked) + masked_data = np.stack(masked_data_list) + return masked_data + elif callable(mask_type): + strategy = mask_type + else: + raise ValueError( + f'Mask type must be callable or type str but got type `{type(mask_type)}` instead.' + ) + + return strategy(data, masks, training_data) + + def mask_data(data: np.array, masks: np.array, mask_type: Union[object, str]): """Mask data given using a set of masks. @@ -107,7 +174,9 @@ def _get_mask_value(data: np.array, mask_type: object) -> int: raise ValueError(f'Unknown mask_type selected: {mask_type}') -def _determine_number_masked(p_keep: float, series_length: int) -> int: +def _determine_number_masked(p_keep: float, + series_length: int, + element_name='feature') -> int: """Determine the number of time steps that need to be masked.""" mean = series_length * (1 - p_keep) floor = np.floor(mean) @@ -121,26 +190,27 @@ def _determine_number_masked(p_keep: float, series_length: int) -> int: if user_requested_steps >= series_length: warnings.warn( - 'Warning: p_keep chosen too low. Continuing with leaving 1 time step unmasked per mask.' + f'Warning: p_keep chosen too low. Continuing with leaving 1 {element_name} unmasked per mask.' ) return series_length - 1 if user_requested_steps <= 0: warnings.warn( - 'Warning: p_keep chosen too high. Continuing with masking 1 time step per mask.' + f'Warning: p_keep chosen too high. Continuing with masking 1 {element_name} per mask.' ) return 1 return user_requested_steps -def generate_time_step_masks(input_data: np.ndarray, number_of_masks: int, - p_keep: float, number_of_features: int): - """Generate masks that masks complete time steps at a time while masking time steps in a segmented fashion. +def generate_time_step_masks(input_data_shape: tuple[int], + number_of_masks: int, p_keep: float, + number_of_features: int): + """Generate masks that mask all channels simultaneously for clusters of time steps. For a conceptual description see: https://medium.com/escience-center/masking-time-series-for-explainable-ai-90247ac252b4. """ - time_series_length = input_data.shape[0] - number_of_channels = input_data.shape[1] + time_series_length = input_data_shape[0] + number_of_channels = input_data_shape[1] float_masks = generate_interpolated_float_masks_for_timeseries( [time_series_length, 1], number_of_masks, number_of_features)[:, :, 0] @@ -169,7 +239,7 @@ def _mask_bottom_ratio(float_mask: np.ndarray, p_keep: float) -> np.ndarray: flat = float_mask.flatten() time_indices = list(range(len(flat))) number_of_unmasked_cells = _determine_number_masked( - p_keep, len(time_indices)) + p_keep, len(time_indices), element_name='time step') top_indices = heapq.nsmallest(number_of_unmasked_cells, time_indices, key=lambda time_step: flat[time_step]) diff --git a/docs/tutorials/rise_tabular_penguin.nblink b/docs/tutorials/rise_tabular_penguin.nblink new file mode 100644 index 00000000..5aa4971d --- /dev/null +++ b/docs/tutorials/rise_tabular_penguin.nblink @@ -0,0 +1,3 @@ +{ + "path": "../../tutorials/explainers/RISE/rise_tabular_penguin.ipynb" +} diff --git a/tests/methods/test_lime_image.py b/tests/methods/test_lime_image.py index ae63a294..16797e3f 100644 --- a/tests/methods/test_lime_image.py +++ b/tests/methods/test_lime_image.py @@ -4,7 +4,7 @@ import dianna from dianna.methods.lime_image import LIMEImage from tests.methods.test_onnx_runner import generate_data -from tests.utils import run_model +from tests.utils import get_dummy_model_function class LimeOnImages(TestCase): @@ -18,7 +18,7 @@ def test_lime_function(): labels = [1] explainer = LIMEImage(random_state=42) - heatmap = explainer.explain(run_model, + heatmap = explainer.explain(get_dummy_model_function(n_outputs=2), input_data, labels, num_samples=100) @@ -53,7 +53,7 @@ def test_lime_values(): labels = [1] explainer = LIMEImage(random_state=42) - heatmap = explainer.explain(run_model, + heatmap = explainer.explain(get_dummy_model_function(n_outputs=2), input_data, labels, return_masks=False, diff --git a/tests/methods/test_lime_tabular.py b/tests/methods/test_lime_tabular.py deleted file mode 100644 index 3627712e..00000000 --- a/tests/methods/test_lime_tabular.py +++ /dev/null @@ -1,36 +0,0 @@ -"""Test LIME tabular method.""" -from unittest import TestCase -import numpy as np -import dianna -from dianna.methods.lime_tabular import LIMETabular -from tests.utils import run_model - - -class LIMEOnTabular(TestCase): - """Suite of LIME tests for the tabular case.""" - - def test_lime_tabular_classification_correct_output_shape(self): - """Test the output of explainer.""" - training_data = np.random.random((10, 2)) - input_data = np.random.random(2) - feature_names = ["feature_1", "feature_2"] - explainer = LIMETabular(training_data, - mode ='classification', - feature_names=feature_names, - class_names = ["class_1", "class_2"]) - exp = explainer.explain( - run_model, - input_data, - ) - assert len(exp[0]) == len(feature_names) - - def test_lime_tabular_regression_correct_output_shape(self): - """Test the output of explainer.""" - training_data = np.random.random((10, 2)) - input_data = np.random.random(2) - feature_names = ["feature_1", "feature_2"] - exp = dianna.explain_tabular(run_model, input_tabular=input_data, method='lime', - mode ='regression', training_data = training_data, - feature_names=feature_names, class_names=['class_1']) - - assert len(exp) == len(feature_names) diff --git a/tests/methods/test_lime_timeseries.py b/tests/methods/test_lime_timeseries.py index 77ef85b2..1ca65d3b 100644 --- a/tests/methods/test_lime_timeseries.py +++ b/tests/methods/test_lime_timeseries.py @@ -3,7 +3,7 @@ from dianna.methods.lime_timeseries import LIMETimeseries from dianna.utils.maskers import generate_time_series_masks from dianna.utils.maskers import mask_data -from tests.utils import run_model +from tests.utils import get_dummy_model_function class LIMEOnTimeseries(TestCase): @@ -15,7 +15,7 @@ def test_lime_timeseries_correct_output_shape(self): num_features = 10 explainer = LIMETimeseries() exp = explainer.explain( - run_model, + get_dummy_model_function(n_outputs=2), input_data, labels=(0, ), class_names=("test", ), @@ -30,7 +30,7 @@ def test_distance_shape(self): """Test the shape of returned distance array.""" dummy_timeseries = np.random.random((50, 1)) number_of_masks = 50 - masks = generate_time_series_masks(dummy_timeseries, + masks = generate_time_series_masks(dummy_timeseries.shape, number_of_masks, p_keep=0.9) masked = mask_data(dummy_timeseries, masks, mask_type="mean") diff --git a/tests/methods/test_maskers.py b/tests/methods/test_maskers.py index b790f554..ad829cfe 100644 --- a/tests/methods/test_maskers.py +++ b/tests/methods/test_maskers.py @@ -4,27 +4,28 @@ from dianna.utils.maskers import generate_channel_masks from dianna.utils.maskers import generate_interpolated_float_masks_for_image from dianna.utils.maskers import generate_interpolated_float_masks_for_timeseries +from dianna.utils.maskers import generate_tabular_masks from dianna.utils.maskers import generate_time_series_masks from dianna.utils.maskers import generate_time_step_masks from dianna.utils.maskers import mask_data -def test_mask_has_correct_shape_univariate(): +def test_timeseries_mask_has_correct_shape_univariate(): """Test masked data has the correct shape for a univariate input.""" input_data = _get_univariate_time_series() number_of_masks = 5 - result = generate_time_series_masks(input_data, number_of_masks) + result = generate_time_series_masks(input_data.shape, number_of_masks) assert result.shape == tuple([number_of_masks] + list(input_data.shape)) -def test_mask_has_correct_type_univariate(): +def test_timeseries_mask_has_correct_type_univariate(): """Test masked data has the correct dtype for a univariate input.""" input_data = _get_univariate_time_series() number_of_masks = 5 - result = generate_time_series_masks(input_data, + result = generate_time_series_masks(input_data.shape, number_of_masks=number_of_masks) assert result.dtype == bool @@ -35,7 +36,7 @@ def test_generate_time_step_masks_dtype_multivariate(): input_data = _get_multivariate_time_series() number_of_masks = 5 - result = generate_time_step_masks(input_data, + result = generate_time_step_masks(input_data.shape, number_of_masks=number_of_masks, number_of_features=8, p_keep=0.5) @@ -48,7 +49,7 @@ def test_generate_segmented_time_step_masks_dtype_multivariate(): input_data = _get_multivariate_time_series() number_of_masks = 5 - result = generate_time_step_masks(input_data, + result = generate_time_step_masks(input_data.shape, number_of_masks=number_of_masks, number_of_features=8, p_keep=0.5) @@ -56,13 +57,13 @@ def test_generate_segmented_time_step_masks_dtype_multivariate(): assert result.dtype == bool -def test_mask_has_correct_shape_multivariate(): +def test_timeseries_mask_has_correct_shape_multivariate(): """Test masked data has the correct shape for a multivariate input.""" input_data = _get_multivariate_time_series() number_of_masks = 5 - result = _call_masking_function(input_data, - number_of_masks=number_of_masks) + result = _call_timeseries_masking_function(input_data, + number_of_masks=number_of_masks) assert result.shape == tuple([number_of_masks] + list(input_data.shape)) @@ -76,24 +77,24 @@ def test_mask_has_correct_shape_multivariate(): (0.5, 0.5), (0.99, 0.9), # Mask only 1 ]) -def test_mask_contains_correct_number_of_unmasked_parts( +def test_timeseries_mask_contains_correct_number_of_unmasked_parts( p_keep_and_expected_rate): """Number of unmasked parts should be conforming the given p_keep.""" p_keep, expected_rate = p_keep_and_expected_rate input_data = _get_univariate_time_series() - result = _call_masking_function(input_data, p_keep=p_keep) + result = _call_timeseries_masking_function(input_data, p_keep=p_keep) assert np.sum(result == input_data) / np.product( result.shape) == expected_rate -def test_mask_contains_correct_parts_are_mean_masked(): +def test_timeseries_mask_contains_correct_parts_are_mean_masked(): """All parts that are masked should now contain the mean of the input.""" input_data = _get_univariate_time_series() mean = np.mean(input_data) - result = _call_masking_function(input_data, mask_type='mean') + result = _call_timeseries_masking_function(input_data, mask_type='mean') masked_parts = result[(result != input_data)] assert np.alltrue( @@ -115,7 +116,7 @@ def _get_multivariate_time_series(number_of_channels: int = 6) -> np.array: ]) -def _call_masking_function( +def _call_timeseries_masking_function( input_data, number_of_masks=5, p_keep=.3, @@ -123,7 +124,7 @@ def _call_masking_function( feature_res=5, ): """Helper function with some defaults to call the code under test.""" - masks = generate_time_series_masks(input_data, + masks = generate_time_series_masks(input_data.shape, number_of_masks, feature_res, p_keep=p_keep) @@ -135,7 +136,7 @@ def test_channel_mask_has_correct_shape_multivariate(): number_of_masks = 15 input_data = _get_multivariate_time_series() - result = generate_channel_masks(input_data, number_of_masks, 0.5) + result = generate_channel_masks(input_data.shape, number_of_masks, 0.5) assert result.shape == tuple([number_of_masks] + list(input_data.shape)) @@ -145,7 +146,7 @@ def test_channel_mask_has_does_not_contain_conflicting_values(): number_of_masks = 15 input_data = _get_multivariate_time_series() - result = generate_channel_masks(input_data, number_of_masks, 0.5) + result = generate_channel_masks(input_data.shape, number_of_masks, 0.5) unexpected_results = [] for mask_i, mask in enumerate(result): @@ -165,33 +166,33 @@ def test_channel_mask_masks_correct_number_of_cells(): input_data = _get_multivariate_time_series(number_of_channels=10) p_keep = 0.3 - result = generate_channel_masks(input_data, number_of_masks, p_keep) + result = generate_channel_masks(input_data.shape, number_of_masks, p_keep) assert result.sum() / np.product(result.shape) == p_keep -def test_masking_has_correct_shape_multivariate(): +def test_timeseries_masking_has_correct_shape_multivariate(): """Test for the correct output shape for the general masking function.""" number_of_masks = 15 input_data = _get_multivariate_time_series() - result = generate_time_series_masks(input_data, number_of_masks) + result = generate_time_series_masks(input_data.shape, number_of_masks) assert result.shape == tuple([number_of_masks] + list(input_data.shape)) -def test_masking_univariate_leaves_anything_unmasked(): +def test_timeseries_masking_univariate_leaves_anything_unmasked(): """Tests that something remains unmasked and some parts are masked for the univariate case.""" number_of_masks = 1 input_data = _get_univariate_time_series() - result = generate_time_series_masks(input_data, number_of_masks) + result = generate_time_series_masks(input_data.shape, number_of_masks) assert np.any(result) assert np.any(~result) -def test_masking_keep_first_instance(): +def test_timeseries_masking_keep_first_instance(): """First instance must be the original data for Lime timeseries. Required by `lime_base` explainer, the first instance of masked (or perturbed) @@ -202,7 +203,9 @@ def test_masking_keep_first_instance(): """ input_data = _get_multivariate_time_series() number_of_masks = 5 - masks = generate_time_series_masks(input_data, number_of_masks, p_keep=0.9) + masks = generate_time_series_masks(input_data.shape, + number_of_masks, + p_keep=0.9) masks[0, :, :] = 1.0 masked = mask_data(input_data, masks, mask_type="mean") assert np.array_equal(masked[0, :, :], input_data) @@ -216,7 +219,7 @@ def test_masks_approximately_correct_number_of_masked_parts_per_time_step( number_of_masks = 500 input_data = _get_univariate_time_series(num_steps=num_steps) - masks = generate_time_series_masks(input_data, + masks = generate_time_series_masks(input_data.shape, number_of_masks=number_of_masks, feature_res=num_steps, p_keep=p_keep)[:, :, 0] @@ -235,7 +238,7 @@ def test_masks_approximately_correct_number_of_masked_parts_per_time_step_projec number_of_masks = 500 input_data = _get_univariate_time_series(num_steps=num_steps) - masks = generate_time_series_masks(input_data, + masks = generate_time_series_masks(input_data.shape, number_of_masks=number_of_masks, feature_res=6, p_keep=p_keep)[:, :, 0] @@ -310,3 +313,66 @@ def test_generate_interpolated_mean_float_masks_for_image( print('\n') print(masks_mean) assert np.allclose(masks_mean, p_keep, atol=0.1) + + +def test_tabular_mask_has_correct_shape(): + """Test whether tabular masks has the correct shape.""" + input_data_shape = (10, ) + number_of_masks = 30 + + masks = np.stack( + list( + generate_tabular_masks( + input_data_shape, + number_of_masks, + p_keep=0.2, + ))) + + assert masks.shape == (number_of_masks, *input_data_shape) + + +@pytest.mark.parametrize('p_keep_and_n_unmasked', [ + (0.2, 0.2), + (0.7, 0.7), + (0.75, 0.75), + (0.999, 0.9), + (0.01, 0.1), +]) +def test_tabular_mask_has_correct_number_masked(p_keep_and_n_unmasked): + """Test whether the expected number of features was masked. + + Also taking into account min=1 and max=n-1 of features per instance and edge cases where the exact p_keep can't + be met. + """ + input_data_shape = (10, ) + number_of_masks = 50 + p_keep, n_unmasked = p_keep_and_n_unmasked + + masks = np.stack( + list( + generate_tabular_masks( + input_data_shape, + number_of_masks, + p_keep=p_keep, + ))) + + mean_element = masks.sum() / (np.product(input_data_shape) * + number_of_masks) + assert np.isclose(mean_element, n_unmasked, atol=0.03) + + +def test_tabular_mask_prob_masked_per_feature_correct(): + """Test whether every feature has the same probability of being masked.""" + input_data_shape = (10, ) + number_of_masks = 1000 + p_keep = 0.2 + + masks = np.stack( + list( + generate_tabular_masks( + input_data_shape, + number_of_masks, + p_keep=p_keep, + ))) + + assert np.allclose(masks.mean(axis=0), p_keep, atol=0.05) diff --git a/tests/methods/test_rise_image.py b/tests/methods/test_rise_image.py index 51ae742b..5a517f7c 100644 --- a/tests/methods/test_rise_image.py +++ b/tests/methods/test_rise_image.py @@ -5,8 +5,8 @@ from dianna.methods.rise_image import RISEImage from dianna.utils import get_function from tests.methods.test_onnx_runner import generate_data +from tests.utils import get_dummy_model_function from tests.utils import get_mnist_1_data -from tests.utils import run_model class RiseOnImages(TestCase): @@ -22,7 +22,7 @@ def test_rise_function(): "tests/test_data/heatmap_rise_function.npy") heatmaps = dianna.explain_image( - run_model, + get_dummy_model_function(n_outputs=2), input_data, "RISE", labels, diff --git a/tests/methods/test_rise_timeseries.py b/tests/methods/test_rise_timeseries.py index f2457951..3479ff0b 100644 --- a/tests/methods/test_rise_timeseries.py +++ b/tests/methods/test_rise_timeseries.py @@ -4,7 +4,7 @@ from dianna.methods.rise_timeseries import RISETimeseries from tests.methods.time_series_test_case import average_temperature_timeseries_with_1_cold_and_1_hot_day from tests.methods.time_series_test_case import run_expert_model_3_step -from tests.utils import run_model +from tests.utils import get_dummy_model_function def test_rise_timeseries_correct_output_shape(): @@ -12,7 +12,7 @@ def test_rise_timeseries_correct_output_shape(): input_data = np.random.random((10, 1)) labels = [1] - heatmaps = dianna.explain_timeseries(run_model, + heatmaps = dianna.explain_timeseries(get_dummy_model_function(n_outputs=2), input_data, "RISE", labels, diff --git a/tests/methods/test_shap_tabular.py b/tests/methods/test_shap_tabular.py deleted file mode 100644 index f2ecc7fe..00000000 --- a/tests/methods/test_shap_tabular.py +++ /dev/null @@ -1,35 +0,0 @@ -"""Test LIME tabular method.""" -from unittest import TestCase -import numpy as np -import dianna -from dianna.methods.kernelshap_tabular import KERNELSHAPTabular -from tests.utils import run_model - - -class LIMEOnTabular(TestCase): - """Suite of LIME tests for the tabular case.""" - - def test_shap_tabular_classification_correct_output_shape(self): - """Test whether the output of explainer has the correct shape.""" - training_data = np.random.random((10, 2)) - input_data = np.random.random(2) - feature_names = ["feature_1", "feature_2"] - explainer = KERNELSHAPTabular(training_data, - mode ='classification', - feature_names=feature_names,) - exp = explainer.explain( - run_model, - input_data, - ) - assert len(exp[0]) == len(feature_names) - - def test_shap_tabular_regression_correct_output_shape(self): - """Test whether the output of explainer has the correct length.""" - training_data = np.random.random((10, 2)) - input_data = np.random.random(2) - feature_names = ["feature_1", "feature_2"] - exp = dianna.explain_tabular(run_model, input_tabular=input_data, method='kernelshap', - mode ='regression', training_data = training_data, - training_data_kmeans = 2, feature_names=feature_names) - - assert len(exp) == len(feature_names) diff --git a/tests/methods/test_tabular.py b/tests/methods/test_tabular.py new file mode 100644 index 00000000..08288adc --- /dev/null +++ b/tests/methods/test_tabular.py @@ -0,0 +1,103 @@ +import numpy as np +import pytest +import dianna +from dianna.methods.kernelshap_tabular import KERNELSHAPTabular +from dianna.methods.lime_tabular import LIMETabular +from dianna.methods.rise_tabular import RISETabular +from tests.utils import get_dummy_model_function + +explainer_names = [ + 'rise', + 'lime', + 'kernelshap', +] + +explainer_classes = [ + RISETabular, + LIMETabular, + KERNELSHAPTabular, +] + + +@pytest.mark.parametrize('method', explainer_names) +def test_tabular_regression_correct_output_shape(method): + """Runs the explainer class with random data and asserts the output shape.""" + number_of_features = 2 + number_of_outputs = 3 # Only the first is used for regression explanation + training_data = np.random.random((10, number_of_features)) + input_data = np.random.random(number_of_features) + feature_names = ["feature_1", "feature_2"] + exp = dianna.explain_tabular( + get_dummy_model_function(n_outputs=number_of_outputs), + input_tabular=input_data, + method=method, + mode='regression', + training_data=training_data, + feature_names=feature_names, + ) + assert exp.shape == (number_of_features, ) + + +@pytest.mark.parametrize('explainer_class', explainer_classes) +def test_tabular_classification_correct_output_shape(explainer_class): + """Runs the explainer class with random data and asserts the output shape.""" + number_of_features = 3 + number_of_classes = 2 + training_data = np.random.random((10, number_of_features)) + input_data = np.random.random(number_of_features) + feature_names = ["feature_1", "feature_2", "feature_3"] + explainer = explainer_class( + training_data, + mode='classification', + feature_names=feature_names, + ) + exp = explainer.explain( + get_dummy_model_function(n_outputs=number_of_classes), + input_data, + ) + assert exp.shape == (number_of_classes, number_of_features) + + +def _pprint(explanations): + """Pretty prints the explanation for each class while classifying tabular data.""" + print() + rows = [' '.join([f'{v:>4d}' for v in range(25)])] + rows += [ + ' '.join([f'{v:.2f}' for v in explanation]) + for explanation in explanations + ] + print('\n'.join(rows)) + + +@pytest.mark.parametrize('explainer_class', explainer_classes) +def test_tabular_simple_dummy_model(explainer_class): + """Tests if the explainer can find the single important feature in otherwise random data.""" + np.random.seed(0) + num_features = 25 + input_data = np.array(num_features // 2 * [1.0] + + (num_features - num_features // 2) * [0.0]) + training_data = np.stack([input_data for _ in range(len(input_data))]).T + + feature_names = [f"feature_{i}" for i in range(num_features)] + important_feature_i = 2 + + def dummy_model(tabular_data): + """Model with output dependent on a single feature of the first instance.""" + prediction = tabular_data[:, important_feature_i] + return np.vstack([prediction, -prediction + 1]).T + + explainer = explainer_class( + training_data, + mode='classification', + feature_names=feature_names, + ) + explanations = explainer.explain( + dummy_model, + input_data, + labels=[0, 1], + ) + + _pprint(explanations) + + assert np.argmax(explanations[0]) == important_feature_i + assert np.argmin(explanations[1]) == important_feature_i diff --git a/tests/test_common_usage.py b/tests/test_common_usage.py index 6db271bd..27e4df90 100644 --- a/tests/test_common_usage.py +++ b/tests/test_common_usage.py @@ -1,7 +1,7 @@ import numpy as np import dianna import dianna.visualization -from tests.utils import run_model +from tests.utils import get_dummy_model_function def test_common_RISE_image_pipeline(): # noqa: N802 ignore case @@ -10,7 +10,7 @@ def test_common_RISE_image_pipeline(): # noqa: N802 ignore case axis_labels = {-1: 'channels'} labels = [0, 1] - heatmap = dianna.explain_image(run_model, + heatmap = dianna.explain_image(get_dummy_model_function(n_outputs=2), input_image, 'RISE', labels, @@ -26,8 +26,8 @@ def test_common_RISE_timeseries_pipeline(): # noqa: N802 ignore case input_timeseries = np.random.random((31, 1)) labels = [0] - heatmap = dianna.explain_timeseries(run_model, input_timeseries, 'RISE', - labels)[0] + heatmap = dianna.explain_timeseries(get_dummy_model_function(n_outputs=2), + input_timeseries, 'RISE', labels)[0] segments = [] for channel_number in range(heatmap.shape[1]): heatmap_channel = heatmap[:, channel_number] diff --git a/tests/test_kwargs.py b/tests/test_kwargs.py index 028be46e..166da1c9 100644 --- a/tests/test_kwargs.py +++ b/tests/test_kwargs.py @@ -3,8 +3,8 @@ import pytest import dianna from tests.methods.test_onnx_runner import generate_data +from tests.utils import get_dummy_model_function from tests.utils import load_movie_review_model -from tests.utils import run_model class ImageKwargs(TestCase): @@ -17,24 +17,25 @@ def test_lime_image_correct_kwargs(self): axis_labels = ('channels', 'y', 'x') labels = [1] - dianna.explain_image(model_filename, - input_data, - method='LIME', - labels=labels, - kernel=None, - kernel_width=25, - verbose=False, - feature_selection='auto', - random_state=None, - axis_labels=axis_labels, - preprocess_function=None, - top_labels=None, - num_features=10, - num_samples=10, - return_masks=True, - positive_only=False, - hide_rest=True, - ) + dianna.explain_image( + model_filename, + input_data, + method='LIME', + labels=labels, + kernel=None, + kernel_width=25, + verbose=False, + feature_selection='auto', + random_state=None, + axis_labels=axis_labels, + preprocess_function=None, + top_labels=None, + num_features=10, + num_samples=10, + return_masks=True, + positive_only=False, + hide_rest=True, + ) def test_lime_image_extra_kwarg(self): """Test to ensure extra kwargs to lime raise warnings.""" @@ -45,24 +46,24 @@ def test_lime_image_extra_kwarg(self): error_message = "Error due to following unused kwargs: {'extra_kwarg': None}" with pytest.raises(TypeError, match=error_message): dianna.explain_image(model_filename, - input_data, - method='LIME', - labels=labels, - kernel=None, - kernel_width=25, - verbose=False, - feature_selection='auto', - random_state=None, - axis_labels=axis_labels, - preprocess_function=None, - top_labels=None, - num_features=10, - num_samples=10, - return_masks=True, - positive_only=False, - hide_rest=True, - extra_kwarg=None - ) + input_data, + method='LIME', + labels=labels, + kernel=None, + kernel_width=25, + verbose=False, + feature_selection='auto', + random_state=None, + axis_labels=axis_labels, + preprocess_function=None, + top_labels=None, + num_features=10, + num_samples=10, + return_masks=True, + positive_only=False, + hide_rest=True, + extra_kwarg=None) + class TextKwargs(TestCase): """Suite of tests for kwargs to explainers for Images.""" @@ -71,8 +72,7 @@ def test_rise_text_correct_kwargs(self): """Test to ensure correct kwargs to lime run without issues.""" review = "such a bad movie" - dianna.explain_text( - self.runner, + dianna.explain_text(self.runner, review, tokenizer=self.runner.tokenizer, method='RISE', @@ -81,8 +81,7 @@ def test_rise_text_correct_kwargs(self): feature_res=8, p_keep=0.5, preprocess_function=None, - batch_size=100 - ) + batch_size=100) def test_rise_text_extra_kwarg(self): """Test to ensure extra kwargs to lime raise warnings.""" @@ -90,8 +89,7 @@ def test_rise_text_extra_kwarg(self): error_message = "Error due to following unused kwargs: {'extra_kwarg': None}" with pytest.raises(TypeError, match=error_message): - dianna.explain_text( - self.runner, + dianna.explain_text(self.runner, review, tokenizer=self.runner.tokenizer, method='RISE', @@ -101,14 +99,14 @@ def test_rise_text_extra_kwarg(self): p_keep=0.5, preprocess_function=None, batch_size=100, - extra_kwarg=None - ) + extra_kwarg=None) def setUp(self) -> None: """Set seed and load runner.""" np.random.seed(0) self.runner = load_movie_review_model() + class TimeseriesKwargs(TestCase): """Suite of tests for kwargs to explainers for Images.""" @@ -117,22 +115,22 @@ def test_lime_timeseries_correct_kwargs(self): input_data = np.random.random((10, 1)) dianna.explain_timeseries( - run_model, - input_timeseries=input_data, - method='LIME', - labels=[0,1], - class_names=["summer", "winter"], - kernel_width=25, - verbose=False, - preprocess_function=None, - feature_selection='auto', - num_features=10, - num_samples=10, - num_slices=10, - batch_size=10, - mask_type='mean', - distance_method='cosine', - ) + get_dummy_model_function(n_outputs=2), + input_timeseries=input_data, + method='LIME', + labels=[0, 1], + class_names=["summer", "winter"], + kernel_width=25, + verbose=False, + preprocess_function=None, + feature_selection='auto', + num_features=10, + num_samples=10, + num_slices=10, + batch_size=10, + mask_type='mean', + distance_method='cosine', + ) def test_lime_timeseries_extra_kwargs(self): """Test to ensure extra kwargs to lime raise warnings.""" @@ -140,24 +138,23 @@ def test_lime_timeseries_extra_kwargs(self): error_message = "Error due to following unused kwargs: {'extra_kwarg': None}" with pytest.raises(TypeError, match=error_message): - dianna.explain_timeseries( - run_model, - input_timeseries=input_data, - method='LIME', - labels=[0,1], - class_names=["summer", "winter"], - kernel_width=25, - verbose=False, - preprocess_function=None, - feature_selection='auto', - num_features=10, - num_samples=10, - num_slices=10, - batch_size=10, - mask_type='mean', - distance_method='cosine', - extra_kwarg=None - ) + dianna.explain_timeseries(get_dummy_model_function(n_outputs=2), + input_timeseries=input_data, + method='LIME', + labels=[0, 1], + class_names=["summer", "winter"], + kernel_width=25, + verbose=False, + preprocess_function=None, + feature_selection='auto', + num_features=10, + num_samples=10, + num_slices=10, + batch_size=10, + mask_type='mean', + distance_method='cosine', + extra_kwarg=None) + class TabularKwargs(TestCase): """Suite of tests for kwargs to explainers for Images.""" diff --git a/tests/utils.py b/tests/utils.py index c5ec7bb0..0bfd1313 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -42,20 +42,31 @@ def get_mnist_1_data(): return np.loadtxt(_mnist_1_data.split()).reshape((1, 1, 28, 28)) -def run_model(input_data): - """Simulate a model that outputs 2-classes. +def get_dummy_model_function(n_outputs): + """Create a function that simulates a model that outputs n_outputs. Args: - input_data: input data for the dummy model + n_outputs: number of outputs Returns: - semi random output + dummy model as a function """ - n_class = 2 - batch_size = input_data.shape[0] - np.random.seed(42) - return np.random.random((batch_size, n_class)) + def run_model(input_data): + """Simulate a model that outputs n_outputs. + + Args: + input_data: input data for the dummy model + + Returns: + semi random output + """ + batch_size = input_data.shape[0] + + np.random.seed(42) + return np.random.random((batch_size, n_outputs)) + + return run_model class ModelRunner: diff --git a/tutorials/README.md b/tutorials/README.md index 21020a66..5c0d7a38 100644 --- a/tutorials/README.md +++ b/tutorials/README.md @@ -52,7 +52,7 @@ The ONNX models used in the tutorials are available at [dianna/models](https://g |*Text* |[nlp-logo_half_size](./explainers/RISE/rise_text.ipynb) or [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/dianna-ai/dianna/blob/main/tutorials/explainers/RISE/rise_text.ipynb) |[nlp-logo_half_size](./explainers/LIME/lime_text.ipynb) or [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/dianna-ai/dianna/blob/main/tutorials/explainers/LIME/lime_text.ipynb) |[]()| | *Time series*| [Weather Logo](./explainers/RISE/rise_timeseries_weather.ipynb) or [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/dianna-ai/dianna/blob/main/tutorials/explainers/RISE/rise_timeseries_weather.ipynb)| [Weather Logo](./explainers/LIME/lime_timeseries_weather.ipynb) or [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/dianna-ai/dianna/blob/main/tutorials/explainers/LIME/lime_timeseries_weather.ipynb)| | | | | [Coffee Logo](./explainers/LIME/lime_timeseries_coffee.ipynb) or [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/dianna-ai/dianna/blob/main/tutorials/explainers/LIME/lime_timeseries_coffee.ipynb) | | -| *Tabular* | | [Penguin Logo](./explainers/LIME/lime_tabular_penguin.ipynb) or [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/dianna-ai/dianna/blob/main/tutorials/explainers/LIME/lime_tabular_penguin.ipynb) |[Penguin Logo](./explainers/KernelSHAP/kernelshap_tabular_penguin.ipynb) or [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/dianna-ai/dianna/blob/main/tutorials/explainers/KernelSHAP/kernelshap_tabular_penguin.ipynb) | +| *Tabular* | Penguin Logo](./explainers/RISE/rise_tabular_penguin.ipynb) or [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/dianna-ai/dianna/blob/main/tutorials/explainers/RISE/rise_tabular_penguin.ipynb) | [Penguin Logo](./explainers/LIME/lime_tabular_penguin.ipynb) or [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/dianna-ai/dianna/blob/main/tutorials/explainers/LIME/lime_tabular_penguin.ipynb) |[Penguin Logo](./explainers/KernelSHAP/kernelshap_tabular_penguin.ipynb) or [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/dianna-ai/dianna/blob/main/tutorials/explainers/KernelSHAP/kernelshap_tabular_penguin.ipynb) | | | | [Weather Logo](./explainers/LIME/lime_tabular_weather.ipynb) or [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/dianna-ai/dianna/blob/main/tutorials/explainers/KernelSHAP/kernelshap_tabular_weather.ipynb)|[Weather Logo](./explainers/KernelSHAP/kernelshap_tabular_weather.ipynb) or [![Open In Colab](https://colab.research.google.com/assets/colab-badge.svg)](https://colab.research.google.com/github/dianna-ai/dianna/blob/main/tutorials/explainers/KernelSHAP/kernelshap_tabular_weather.ipynb) | To learn more about how we aproach the masking for time-series data, please read our [Masking time-series for XAI](https://blog.esciencecenter.nl/masking-time-series-for-explainable-ai-90247ac252b4) blog-post. @@ -68,7 +68,7 @@ To learn more about how we aproach the masking for time-series data, please read ### IMPORTANT: Hyperparameters The XAI methods (explainers) are sensitive to the choice of their hyperparameters! In this [master Thesis](https://staff.fnwi.uva.nl/a.s.z.belloum/MSctheses/MScthesis_Willem_van_der_Spec.pdf), this sensitivity is researched and useful conclusions are drawn. -The default hyperparameters used in DIANNA for each explainer as well as the choices for some tutorials and their data modality (*i* - images, *txt* - text, *ts* - time series and *tab* - tabular) are given in the tables below. +The default hyperparameters used in DIANNA for each explainer as well as the choices for some tutorials and their data modality (*i* - images, *txt* - text, *ts* - time series and *tab* - tabular) are given in the tables below. Also the main conclusions (🠊) from the thesis (on images and text) about the hyperparameters effect are listed. #### RISE @@ -102,4 +102,4 @@ Also the main conclusions (🠊) from the thesis (on images and text) about the 🠊 The most crucial parameter is the nubmer of super-pixels $n_{segments}. Higher values led to higher sensitivity, however that observaiton was dependant on the evaluaiton metric. -🠊 Regularization had only a marginal detrimental effect, the best results were obtained using no regularization (no smoothing, $sigma = 0$) or least squares regression. +🠊 Regularization had only a marginal detrimental effect, the best results were obtained using no regularization (no smoothing, $sigma = 0$) or least squares regression. diff --git a/tutorials/explainers/RISE/rise_tabular_penguin.ipynb b/tutorials/explainers/RISE/rise_tabular_penguin.ipynb new file mode 100644 index 00000000..53974931 --- /dev/null +++ b/tutorials/explainers/RISE/rise_tabular_penguin.ipynb @@ -0,0 +1,403 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "\"Logo_ER10\"\n", + "\n", + "### Model Interpretation using LIME for penguin dataset classifier\n", + "This notebook demonstrates the use of DIANNA with the RISE tabular method on the penguins dataset.\n", + "\n", + "RISE is short for Randomized Input Sampling for Explanation of Black-box Models. It estimates each feature's relevance to the model's decision empirically by probing the model with randomly masked versions of the input text and obtaining the corresponding outputs. More details about this method can be found in the [paper introducing RISE](https://arxiv.org/abs/1806.07421)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Colab setup" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "ExecuteTime": { + "start_time": "2024-05-30T11:11:52.728074Z", + "end_time": "2024-05-30T11:11:52.815285Z" + } + }, + "outputs": [], + "source": [ + "running_in_colab = 'google.colab' in str(get_ipython())\n", + "if running_in_colab:\n", + " # install dianna\n", + " !python3 -m pip install dianna[notebooks]\n", + " \n", + " # download data used in this demo\n", + " import os \n", + " base_url = 'https://raw.githubusercontent.com/dianna-ai/dianna/main/tutorials/'\n", + " paths_to_download = ['models/penguin_model.onnx']\n", + " for path in paths_to_download:\n", + " !wget {base_url + path} -P {os.path.dirname(path)}" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 0 - Import libraries" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": { + "ExecuteTime": { + "start_time": "2024-05-30T11:11:52.734426Z", + "end_time": "2024-05-30T11:11:52.815285Z" + } + }, + "outputs": [], + "source": [ + "import dianna\n", + "import numpy as np\n", + "import pandas as pd\n", + "import seaborn as sns\n", + "from sklearn.model_selection import train_test_split\n", + "from dianna.utils.onnx_runner import SimpleModelRunner\n", + "from pathlib import Path\n", + "\n", + "root_dir = Path(dianna.__file__).parent" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 1 - Loading the data\n", + "Load penguins dataset." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": { + "ExecuteTime": { + "start_time": "2024-05-30T11:11:52.748720Z", + "end_time": "2024-05-30T11:11:52.831285Z" + } + }, + "outputs": [], + "source": [ + "penguins = sns.load_dataset('penguins')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Prepare the data" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": { + "ExecuteTime": { + "start_time": "2024-05-30T11:11:52.756731Z", + "end_time": "2024-05-30T11:11:52.891178Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": " bill_length_mm bill_depth_mm flipper_length_mm body_mass_g\n0 39.1 18.7 181.0 3750.0\n1 39.5 17.4 186.0 3800.0\n2 40.3 18.0 195.0 3250.0\n4 36.7 19.3 193.0 3450.0\n5 39.3 20.6 190.0 3650.0\n.. ... ... ... ...\n338 47.2 13.7 214.0 4925.0\n340 46.8 14.3 215.0 4850.0\n341 50.4 15.7 222.0 5750.0\n342 45.2 14.8 212.0 5200.0\n343 49.9 16.1 213.0 5400.0\n\n[342 rows x 4 columns]", + "text/html": "
\n\n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n
bill_length_mmbill_depth_mmflipper_length_mmbody_mass_g
039.118.7181.03750.0
139.517.4186.03800.0
240.318.0195.03250.0
436.719.3193.03450.0
539.320.6190.03650.0
...............
33847.213.7214.04925.0
34046.814.3215.04850.0
34150.415.7222.05750.0
34245.214.8212.05200.0
34349.916.1213.05400.0
\n

342 rows × 4 columns

\n
" + }, + "execution_count": 4, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Remove categorial columns and NaN values\n", + "penguins_filtered = penguins.drop(columns=['island', 'sex']).dropna()\n", + "\n", + "# Get the species\n", + "species = penguins['species'].unique()\n", + "\n", + "# Extract inputs and target\n", + "input_features = penguins_filtered.drop(columns=['species'])\n", + "target = pd.get_dummies(penguins_filtered['species'])\n", + "\n", + "# Let's explore the features of the dataset\n", + "input_features" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The data-set currently has four features that were used to train the model: bill length, bill depth, flipper length, and body mass. These features were used to classify the different species." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Training, validation, and test data split." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": { + "ExecuteTime": { + "start_time": "2024-05-30T11:11:52.775984Z", + "end_time": "2024-05-30T11:11:52.893178Z" + } + }, + "outputs": [], + "source": [ + "X_train, X_test, y_train, y_test = train_test_split(input_features, target, test_size=0.2,\n", + " random_state=0, shuffle=True, stratify=target)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Get an instance to explain." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": { + "ExecuteTime": { + "start_time": "2024-05-30T11:11:52.781926Z", + "end_time": "2024-05-30T11:11:52.893178Z" + } + }, + "outputs": [], + "source": [ + "# get an instance from test data\n", + "data_instance = X_test.iloc[10].to_numpy()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 2. Loading ONNX model\n", + "DIANNA supports ONNX models. Here we demonstrate the use of LIME explainer for tabular data with a pre-trained ONNX model, which is a MLP classifier for the penguins dataset.
\n", + "\n", + "The model is trained following this notebook:
\n", + "https://github.com/dianna-ai/dianna-exploration/blob/main/example_data/model_generation/penguin_species/generate_model.ipynb" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "ExecuteTime": { + "start_time": "2024-05-30T11:11:52.786936Z", + "end_time": "2024-05-30T11:11:52.921188Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": "'Gentoo'" + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# load onnx model and check the prediction with it\n", + "model_path = Path(root_dir, 'models', 'penguin_model.onnx')\n", + "loaded_model = SimpleModelRunner(model_path)\n", + "predictions = loaded_model(data_instance.reshape(1,-1).astype(np.float32))\n", + "species[np.argmax(predictions)]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "A runner function is created to prepare data for the ONNX inference session." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": { + "ExecuteTime": { + "start_time": "2024-05-30T11:11:52.881179Z", + "end_time": "2024-05-30T11:11:52.922190Z" + } + }, + "outputs": [], + "source": [ + "import onnxruntime as ort\n", + "\n", + "def run_model(data):\n", + " # get ONNX predictions\n", + " sess = ort.InferenceSession(model_path)\n", + " input_name = sess.get_inputs()[0].name\n", + " output_name = sess.get_outputs()[0].name\n", + "\n", + " onnx_input = {input_name: data.astype(np.float32)}\n", + " pred_onnx = sess.run([output_name], onnx_input)[0]\n", + " \n", + " return pred_onnx" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 3. Applying RISE with DIANNA\n", + "The simplest way to run DIANNA on image data is with `dianna.explain_tabular`.\n", + "\n", + "DIANNA requires input in numpy format, so the input data is converted into a numpy array.\n", + "\n", + "Note that the training data is also required for proper perturbation of the input data." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": { + "ExecuteTime": { + "start_time": "2024-05-30T11:11:52.885179Z", + "end_time": "2024-05-30T11:11:53.364609Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Explaining: 100%|██████████| 10/10 [00:00<00:00, 322.57it/s]\n" + ] + } + ], + "source": [ + "explanation = dianna.explain_tabular(run_model, input_tabular=data_instance, method='rise',\n", + " mode ='classification', training_data = X_train.to_numpy(),\n", + " feature_names=input_features.columns, class_names=species)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "outputs": [ + { + "data": { + "text/plain": "array([[1.3488360e-05, 3.1694275e-05, 9.9995482e-01]], dtype=float32)" + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "run_model(data_instance[None,...])" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "start_time": "2024-05-30T11:11:53.332880Z", + "end_time": "2024-05-30T11:11:53.369138Z" + } + } + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### 4. Visualization\n", + "The output can be visualized with the DIANNA built-in visualization function. It shows the importance of each feature contributing to the prediction.\n", + "\n", + "The prediction is \"Gentoo\", so let's visualize the feature importance scores for \"Gentoo\".\n", + "\n", + "It can be noticed that the body mass feature has the biggest weight in the prediction." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "ExecuteTime": { + "start_time": "2024-05-30T11:11:53.370141Z", + "end_time": "2024-05-30T11:11:53.560552Z" + } + }, + "outputs": [ + { + "data": { + "text/plain": "
", + "image/png": "iVBORw0KGgoAAAANSUhEUgAAApMAAAGwCAYAAAD1xu74AAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjguMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/H5lhTAAAACXBIWXMAAA9hAAAPYQGoP6dpAAA8D0lEQVR4nO3dd3QU9f7/8dcmgEESWrBcAkIUCTUkIKJUCVhAWmg/AkSqICDgRaVXKUq5cL8BKRK8EC69Rb1SpIgFpUkEUUJJ6HiVhAAhlJDs/P5A9hppybDsJJvn45w97M5ndua9b0L2xXxmdm2GYRgCAAAATPCwugAAAADkXIRJAAAAmEaYBAAAgGmESQAAAJhGmAQAAIBphEkAAACYRpgEAACAaYRJAAAAmEaYBAAAgGl5rC4Auce5c8my262uInex2SRfXx8lJiaL77pyPfpvHXpvHXpvLWf2/+a27oUwCZcxDPGLxSL03lr03zr03jr03lqu7D/T3AAAADCNMAkAAADTCJMAAAAwjTAJAAAA0wiTAAAAMI0wCQAAANMIkwAAADCNMAkAAADTCJMAAAAwjTAJAAAA0wiTAAAAMI0wCQAAANMIkwAAADAtj9UFIPfw8PCQB/99sYSnp/s03m43ZLcbVpcBAPgDYRIuU6RIAatLyLXcqfdGWprOXbhKoASAbIIwCdfp3l3as8fqKpCTlS8v26JF8vCwESYBIJsgTMJ1Dh6UYmKsrgIAADiR+5xIBQAAAJcjTAIAAMA0wiQAAABMI0wCAADANMIkAAAATCNMAgAAwDTCJAAAAEwjTAIAAMA0wiQAAABMI0wCAADANMIkAAAATCNMAgAAwDTCJAAAAEwjTAIAAMA0wiQAAABMI0wCAADANMIkAAAATCNMAgAAwDTCJAAAAEwjTAIAAMA0wiQAAABMI0wCAADANMIkAAAATCNMAgAAwDTCJAAAAEwjTAIAAMA0wiQAAABMI0wCAADANLcKk6dOnVJAQIBOnTp1y9jq1asVEhIiSdqxY4cCAgLu+Zy/+vPzHrTExEStW7fO8TggIEA7duxwyb4BAAAyK4/VBbhK48aN9cILL1hdRqZNmTJFhmGoUaNGVpcCAABwR7kmTHp5ecnLy8vqMjLNMAyrSwAAALgntwyT69evV1RUlC5duqQmTZpo+PDh+s9//qMZM2Zoy5YtTtvPr7/+qjFjxuj777+Xr6+vWrZsqV69esnT01OrV6/WmjVrVL16dS1atEjp6elq1aqVBg8eLJvNJkmaP3++5s2bp5SUFLVs2VIHDx5UaGioTp8+rTVr1kiSdu7c6ah59+7dGjdunI4dO6YqVapo4sSJ8vPzu2ed06dP18mTJ+Xj46PVq1erSJEieu+993Ts2DHNnDlTdrtdvXv31muvvSbpxpT6P//5T0VEROjMmTNq0KCB/v73v2vYsGHau3evKlasqGnTpumxxx5zWi+BrPrjn1G2drPGnFCru6H31qH31nJm/zO7DbcMk8uXL9e0adOUnp6ugQMHas6cOZkKXVlhGIbefPNNlStXTmvWrNHZs2c1cuRI2Ww29enTR5IUExOjYsWKacmSJfrpp580ePBg1a1bV7Vq1dKnn36qiIgIjR8/XmXKlNE//vEP7dq1S6Ghoeratavi4uIkSSNHjnTsc8WKFZo4caIKFy6sd955R1OmTNG0adMyVe/atWvVvXt3ffLJJ5o6dareeustPfPMM1q4cKHWr1+viRMnqkmTJipatKgkKSIiQh988IGuXLmi7t27a+fOnRo2bJiGDBmifv36ae7cuRo+fLhTewpkVpEiBawuIUt8fX2sLiHXovfWoffWcmX/3TJMDh06VNWqVZMk9e/fX1OmTNHbb7/t1H1s375dZ86c0YoVK+Th4aEnn3xSgwYN0pAhQxxhMj09XWPHjpW3t7eefPJJzZ8/Xz/99JNq1aqlxYsXq1OnTo5zIidOnKh69epJkgoUKOCYkr8Z7iSpV69eqlGjhiSpdevWWrp0aabrLVKkiPr37y+bzabQ0FCtW7dOw4YNU8mSJdWtWzdFRETo+PHjjv117txZVapUkSSVL19e/v7+jlpfeuklxcbG3k/7gPuSlJSi9HS71WXck8124xd6YmKyOHPFtei9dei9tZzZ/5vbuhe3DJOBgYGO+xUqVFBCQoIuXrzo1H3ExcXp/PnzjtAqSXa7XVevXlVSUpIkydfXV97e3o5xb29vpaWlSZIOHjyoHj16OMYKFSokf3//u+7ziSeecNz38fHRtWvXMl1viRIlHNPrN4PqzaO1Nx+npqY61i9ZsqTjvpeXV4Yju15eXhnWBayQk96kDCNn1etO6L116L21XNl/twyTHh7/+8Sjmxey5M2b16n7SEtL05NPPqmZM2feMubjcyPF58uX75axm/V4enrecpHNvS66+fPryqo8eW79q77b9jw9PZ22bwAA4L7cMiEcOnTIcX/fvn16/PHHlT9/fqfuw9/fX2fOnFHRokVVqlQplSpVSqdOnVJERITjCODdlClTRj///LPj8aVLl3T8+HHH48xsAwAAwGpuGSbHjh2rvXv3atu2bYqIiFDnzp2dvo/atWvLz89P7777rg4ePKjdu3drxIgRyp8//y1H9W4nPDxcUVFR+uKLLxQXF6ehQ4fq8uXLjhCZP39+nT59Wr/99pvTawcAAHAWtwyTYWFh6tWrl9566y01b95cnTp1cvo+PD09NWvWLNntdrVt21Z9+/ZVvXr1Mn2F86uvvqquXbtq1KhRatOmjfz8/OTn5+eYjm/evLmOHj2qZs2a8ZmTAAAg27IZJBVL7Ny5UyVLltTf/vY3STfOwXzuuef04YcfOq7Ydjt16kjffmt1FcjJgoOlPXuUlJSitLSccTV3sWI+SkjgqlZXo/fWoffWcmb/b27rXtzyApycYNOmTYqJidGYMWNUoEABRUVFydvbW0FBQVaXBgAAkGmEyT8kJiaqYcOGd10nJibGafvr16+f3nvvPXXp0kXXrl1TcHCwIiMj9dBDD2VpOxs2bNDgwYPvOF6tWjVFRkbeb7kAAAC3RZj8Q+HChRUdHe2y/Xl7e2vSpEn3vZ3atWvfte6c9H3kAAAg5yFM/sHT01OlSpWyuowsK1CggAoUyFlfLQcAANyHW17NDQAAANcgTAIAAMA0wiQAAABMI0wCAADANMIkAAAATCNMAgAAwDTCJAAAAEwjTAIAAMA0wiQAAABMI0wCAADANMIkAAAATCNMAgAAwDTCJAAAAEwjTAIAAMA0wiQAAABMI0wCAADANMIkAAAATCNMAgAAwDTCJAAAAEwjTAIAAMA0wiQAAABMI0wCAADANMIkAAAATMtjdQHIRQICpJQUq6tATla+vNUVAAD+gjAJ14mMtLoCuAEjLU12u2F1GQCAPxAm4TJJSRyVtEKRIgXcqvd2u0GYBIBshDAJl7Hb7bLbra4id7HZbvyZnm6XQf4CADwAXIADAAAA0wiTAAAAMI0wCQAAANMIkwAAADCNMAkAAADTCJMAAAAwjTAJAAAA0wiTAAAAMI0wCQAAANMIkwAAADCNMAkAAADTCJMAAAAwjTAJAAAA0/JYXQByDw8PD3nw3xdLeHrm7sbb7YbsdsPqMgDALREm4TJFihSwuoRcK7f33khL07kLVwmUAPAAECbhOt27S3v2WF0Fcpvy5WVbtEgeHjbCJAA8AIRJuM7Bg1JMjNVVAAAAJ8rdJ1IBAADgvhAmAQAAYBphEgAAAKYRJgEAAGAaYRIAAACmESYBAABgGmESAAAAphEmAQAAYBphEgAAAKYRJgEAAGAaYRIAAACmESYBAABgGmESAAAAphEmAQAAYBphEgAAAKYRJgEAAGAaYRIAAACmESYBAABgGmESAAAAphEmAQAAYBphEgAAAKYRJgEAAGAaYRIAAACmESYBAABgGmESAAAAphEmAQAAYBphEgAAAKYRJgEAAGCa24bJU6dOKSAgQKdOnbplbPXq1QoJCZEk7dixQwEBAfd8TmYMHjxYgwcPNl/0nxw4cEB79uy5pUYAAIDsxG3D5N00btxYK1eutLqMu+rTp4+OHTtmdRkAAAB3lcfqAqzg5eUlLy8vq8sAAADI8dz+yOT69etVt25dVa1aVSNHjlRqamqGae77sXv3brVo0UKBgYHq37+/rly5kmF848aNaty4sapUqaLWrVtr586djrHw8HDNmDFDYWFhqlKlitq3b6+4uDjH2OnTpzVkyJAM0+ZLlixRnTp1FBwcrCFDhig1NTVTdYaHh2vevHnq0qWLAgMD1bp1ax0/flwjRoxQcHCwXnrpJUdtO3bsUEhIiFauXKlatWqpevXqmjt3rnbt2qVXXnlFwcHBGjhwoOx2+/22DwAAuAG3D5PLly/XtGnTNHv2bH399deaM2eOU7Z77tw59ezZUzVr1lR0dLTKlCmj9evXO8ZjY2M1aNAg9erVS59++qmaNWum119/XcePH3esM2fOHL388stavXq1HnvsMfXo0UOpqamaPn26Hn/8cQ0dOlTDhg1zrL9hwwbNmzdPM2bM0Pr167Vq1apM1/vhhx+qbdu2Wr16tZKTk9W6dWsVK1ZMK1eu1NNPP61x48Y51v3999+1adMmLVy4UG+88YamTp2qCRMm6IMPPtDUqVO1du1abd68+T47CLiezeb6m1X75Ubv6X3uvTmz/5nh9tPcQ4cOVbVq1SRJ/fv315QpU/T222/f93bXrVunokWL6t1335XNZlPfvn311VdfOcbnzZuntm3bqmnTppKk1157Tbt27dKSJUscRxvr1q2rzp07S5LGjh2rOnXqaNu2bapfv748PT3l4+MjHx8fxzZHjRolf39/lS1bVjVr1lRsbGym661fv74aNWokSWrYsKHWrl2rfv36yWazqW3bturTp49j3evXr2vQoEHy9/dX8eLFNWnSJHXo0EFBQUGSpPLlyys+Pt5U3wCrFClSwLJ9+/r63HslPBD03jr03lqu7L/bh8nAwEDH/QoVKighIUEXL1687+0eOXJE5cqVk+1Psb1y5cqOqe64uDitW7dOy5Ytc4xfv35dtWvXdjyuWrWq4763t7f8/f0VFxen+vXr33afTzzxhOO+j49Ppqe5JalEiRKO+15eXipevLijdi8vL12/fj3D+iVLlnSMSZKfn1+G52dl30B2kJSUovR0156eYbPd+IWemJgsw3DprnM9em8dem8tZ/b/5rbuxe3DpIfH/2byjT+6mjdvXqds2/jL31LevHkdYTI9PV2vv/66WrRokWGdP1/4kydPxvanp6dnqPevPD0977r/u/nrvu62HzPrAzmBVW9shmHdvnM7em8dem8tV/bf7RPCoUOHHPf37dunxx9/XPnz57/v7T799NP65ZdflJ6e7lh24MABx31/f3+dOnVKpUqVctyWLVumr7/+2rHOn6epk5OTdeLECT5PEgAA5ChuHybHjh2rvXv3atu2bYqIiHCco3i/Xn31VV25ckXjx49XfHy8IiMj9cMPPzjGO3furLVr1yoqKkonTpzQ/PnzNX/+fJUuXdqxzmeffabo6GjFxcVp2LBhKl68uGrUqCFJevjhhxUfH6/z5887pV4AAIAHwe3DZFhYmHr16qW33npLzZs3V6dOnZyy3UKFCikyMlI//fSTmjdvru+++07Nmzd3jAcFBWnSpElavHixGjdurOXLl+sf//iHqlev7linadOmWrp0qVq2bKmUlBTNnTvXMb0cFhamRYsWafjw4U6pFwAA4EGwGVk58Q5OEx4ermeffVZ9+/a1uhTXqVNH+vZbq6tAbhMcLO3Zo6SkFKWluf4CnGLFfJSQwIUIrkbvrUPvreXM/t/c1r24/ZFJAAAAPDhufzW3GYmJiWrYsOFd14mJiXFRNXc3fvz4u37PeM+ePfXGG2+4sCIAAJCbECZvo3DhwoqOjn6g+1i4cKFTttOrVy917NjxjuOFChVyyn4AAABuhzB5G56enipVqpTVZWRK0aJFVbRoUavLAAAAuRTnTAIAAMA0wiQAAABMI0wCAADANMIkAAAATCNMAgAAwDTCJAAAAEwjTAIAAMA0wiQAAABMI0wCAADANKeEyWvXrmnfvn1KTk52xuYAAACQQ5gKk0eOHFHbtm21Z88eXbx4US1atFDbtm1Vt25dbd++3dk1AgAAIJsyFSbHjBmjkiVLyt/fXytXrlRycrK+/fZbvfHGG5o4caKzawQAAEA2ZSpM7tu3T2+99ZaKFCmiTZs26cUXX1SxYsXUpEkTxcfHO7tGAAAAZFOmwqSPj48SEhL066+/6scff9QLL7wgSTpw4IB8fX2dWR8AAACysTxmntSyZUv16tVL+fLlU4kSJVS7dm0tWbJEkyZNUv/+/Z1dIwAAALIpU2FywIABqly5sk6fPq0mTZrI09NTxYsX19SpU1W/fn1n1wgAAIBsylSYlKQXX3xRly5d0okTJ1SwYEFVq1ZN3t7ezqwNAAAA2ZypcyavXbum4cOH69lnn1Xr1q3122+/afDgwerWrZsuXLjg7BoBAACQTZkKk5MnT9aRI0e0Zs0aPfTQQ5Kkvn37KikpSePGjXNqgQAAAMi+TIXJL774QsOGDVNAQIBjWUBAgMaOHauvv/7aacUBAAAgezMVJlNSUpQ/f/5bltvtdqWnp993UQAAAMgZTIXJkJAQTZs2TZcuXXIsO3nypMaNG6d69eo5rTgAAABkb6bC5MiRI+Xh4aFnn31WV65cUatWrfTSSy+pYMGCGjFihLNrBAAAQDZl6qOBkpKSNH36dJ08eVJxcXFKS0uTv7+/nnrqKWfXB3cSECClpFhdBXKb8uWtrgAA3JqpMBkWFqY5c+aoUqVKKlmypLNrgruKjLS6AuRSRlqa7HbD6jIAwC2ZCpPFihVTYmKis2uBm0tK4qikFYoUKZDre2+3G4RJAHhATIXJChUqqHfv3qpcubL8/PyUL1++DOPvv/++U4qDe7Hb7bLbra4id7HZbvyZnm6XQZYCADwApr9OsVmzZs6sAwAAADmQqTDJkUcAAABIJsPkjBkz7jr+5ptvmioGAAAAOYupMLljx44Mj9PT03Xq1CldvHhRL7/8slMKAwAAQPZnKkwuXLjwtssnTJgg280z/gEAAOD2TH0Dzp2Eh4dr9erVztwkAAAAsjGnhsmvvvpKDz30kDM3CQAAgGzM1DR3SEjILdPZKSkpunDhggYNGuSUwgAAAJD9mQqTffv2zfDYZrMpb968qlSpkkqVKuWUwgAAAJD9mQqTp0+fVrdu3ZQ/f/4Myy9duqQPPvhAgwcPdkpxAAAAyN4yHSbj4+Md38f94Ycfqly5cipUqFCGdQ4dOqSlS5cSJgEAAHKJTIfJ33//XZ07d3Y8vt0Hk+fPn1+dOnVySmEAAADI/jIdJp977jnFxsZKunEBzsqVK1W0aNEHVhgAAACyP1MfDbRly5Y7Bsnff//9vgoCAABAzmHqApz4+HhNmTJFR44cUXp6uiTJMAylpqbq3Llz+uWXX5xaJNyDh4eHPJz6yabILE9PGv9Xdrshu92wugwAyPFMhckRI0YoPT1d3bp104QJEzRw4ECdPn1aixcv1vjx451dI9xEkSIFrC4h16L3tzLS0nTuwlUCJQDcJ1Nh8qefftKyZctUvnx5RUdH68knn1SHDh3k7++vlStXKjQ01Nl1wh107y7t2WN1FYBUvrxsixbJw8NGmASA+2QqTObJk0c+Pj6SpCeffFIHDhzQ888/r5o1a2rixIlOLRBu5OBBKSbG6ioAAIATmTqRKjg4WPPmzdPVq1dVqVIlbdmyRYZhaP/+/Xw3NwAAQC5i6sjkkCFD1KtXL5UsWVLt2rVTVFSUnn32WV2+fFm9e/d2do0AAADIpkyFyTJlyuiLL77Q1atXlT9/fq1atUo7d+5U4cKFFRQU5OQSAQAAkF2Z/rwQu92uHTt2aP78+UpLS1ORIkX01FNPObM2AAAAZHOmjkz++uuv6tq1qy5cuKALFy6oQYMGioyMVExMjObNm6eAgABn1wkAAIBsyNSRyffee0/PPPOMvvnmG+XLl0+SNHXqVNWsWVPjxo1zaoEAAADIvkyFyd27d6tr167y9PR0LMubN6969+6t/fv3O604AAAAZG+mwqSXl5cSExNvWX706FF5e3vfd1EAAADIGUyFyXbt2mnkyJHaunWrpBshctWqVRoxYoRat27tzPoAAACQjZm6AKdPnz4qWLCgRo8erStXrqhHjx7y9fVV586d1a1bN2fXCAAAgGwq02GyQ4cOmjVrlgoWLChJCg8PV+vWrWW322W32x1frwgAAIDcI9PT3D/88IOuX7+eYVmtWrV07tw5giQAAEAuZfpDyyXJMAxn1QEAAIAc6L7CJAAAAHI3wiQAAABMy9LV3OvWrcvwOZJ2u10bN25U0aJFM6zXokULpxQHAACA7C3TYbJ48eL6+OOPMyzz9fXVv//97wzLbDYbYRIAACCXyHSY3LJly4OsAwAAADkQ50wCAADANMIkAAAATCNMAgAAwDTCJAAAAEwjTAIAAMA0wiQAAABMI0wCAADANMIkAAAATCNMAgAAwDTCJAAAAEyzNExu3rxZdevWVZUqVRQQEKBTp05JkgICArRjxw4rS7ut6dOnKzw83CX7OnDggPbs2SNJ2rFjhwICAlyyXwAAgKywNExGRESodu3amjt3bobl3377rYKDgy2qKnvo06ePjh07ZnUZAAAAd5XHyp0nJyerWrVqKl68eIbljzzyiEUVAQAAICssOzIZEhKi06dPa+jQoXrttdcyjP15mjskJETz589X06ZNFRQUpB49eujs2bOSbkz/1q1bV1FRUapRo4Zq1qypWbNmZdjW0qVLFRISouDgYIWHh+vgwYMZapg8ebJq166tFi1ayDCMLL2G3bt3q2XLlgoMDFTTpk21YcMGx9jgwYP1/vvv66233lKVKlVUr149RUdHO8avXr2qYcOGqVq1aqpTp45WrFihChUq6NSpUwoPD9fp06c1ZMgQDR482PGcJUuWqE6dOgoODtaQIUOUmpqaqTrDw8M1b948denSRYGBgWrdurWOHz+uESNGKDg4WC+99JJ27tzp6GlISIhWrlypWrVqqXr16po7d6527dqlV155RcHBwRo4cKDsdnuWegVkVzbbg725Yh/c6H12u9F79+l/phgWSUxMNOrWrWvMnz/f2Lt3r1G2bFnj5MmThmEYRtmyZY3t27cbhmEY9evXN6pWrWpER0cbsbGxRseOHY3/9//+n2EYhrF9+3ajQoUKRmhoqLF//35j48aNRtWqVY1ly5YZhmEYmzdvNmrVqmVs2bLFOHr0qDFt2jTj2WefNc6fP+/Ydp06dYzY2FjjwIED96w5IiLC6Nixo2EYhvH7778bVatWNRYuXGgcO3bMiI6ONoKCgoxdu3YZhmEYgwYNMipWrGh89NFHxokTJ4xx48YZgYGBxsWLFw3DMIxhw4YZjRo1MmJiYoxdu3YZL730kqMHSUlJjt5cvHjR2L59u1G2bFmjU6dOxsGDB41vv/3WCAoKMhYvXpypXnfs2NEIDg421q5daxw+fNh46aWXjGeeecb45z//aRw5csTo3bu30bRpU0dPK1asaPTs2dOIi4szIiMjjXLlyhktWrQwYmJijC1bthgVK1Y0vvjii0ztO4PatQ1D4sbN+ltwcNZ/fgEAt2XZNHfRokXl6ekpHx8fFS1a9K7rtmrVSs2bN5ckTZgwQQ0bNtShQ4ckSWlpaZowYYLKlSunihUrqlOnTlq6dKnatm2ryMhI9ezZU/Xr15ckvfXWW/r666/16aefOi6kadasmamLWxYtWqSaNWuqY8eOkqRSpUrpwIEDWrBggZ555hlJN46wvv7665Kk/v37KyoqSocPH1ZAQICio6M1d+5cBQUFSZKGDx+u7t27S5IKFy7s6I2Pj49jn6NGjZK/v7/Kli2rmjVrKjY2NtP11q9fX40aNZIkNWzYUGvXrlW/fv1ks9nUtm1b9enTx7Hu9evXNWjQIPn7+6t48eKaNGmSOnTo4Ki1fPnyio+Pz3LPgOwmKSlF6ekP7ii7zSb5+vooMTFZhvHAdoPboPfWoffWcmb/b27rXiw9ZzKzqlat6rhfsmRJFS5cWHFxcSpatKgefvhhlStXzjFeqVIlffzxx5KkuLg4TZ48WVOnTnWMX7t2LcOFLX5+fqZqio+P15dffpnhQqHr16/L39/f8bh06dKO+97e3pJuhN/4+Hhdv35dlStXdoxn5oKjJ554wnHfx8cn09PcklSiRAnHfS8vLxUvXly2P45fe3l56fr16xnWL1mypGNMytgnLy+vLO0byM5c8WZ385AoXI/eW4feW8uV/c8RYTJPnoxlpqeny8PD47ZjdrvdEZLS09M1dOhQPf/88xnWuRnsJOmhhx4yVVNaWpqaNm2qN95444615s2b95bnGYZxS803l9+Lp6dnlp9zu7okOfrnrPUBAEDulCMSwp+nc48fP67k5GTH1PTFixcdn08pST/99JNjzN/fX//9739VqlQpx2327Nn68ccf77smf39/HT9+PMO2N2/erM8+++yez33iiSeUN29e7d+/37Hsz/cBAAByihwRJqOiorR582bFxsZq6NChqlWrVoYp5BEjRujQoUPasGGDFi5cqA4dOkiSunTpogULFig6OlonTpzQ5MmTtW7dOj311FP3XVP79u21f/9+TZs2TceOHdNnn32mqVOn3vIxR7dToEABtWzZUuPHj9fevXv1448/avz48ZLkOKr68MMPKz4+XufPn7/vWgEAAB6UHDHNHRoaqqlTp+rMmTOqV6+exowZk2G8bt26at++vR5++GENGDBATZs2lSQ1btxYCQkJioiIUEJCgsqUKaNZs2ZlCKJm+fn5afbs2ZoyZYrmzZunxx57TIMHD1azZs0y9fxBgwZp1KhR6ty5s7y9vdWhQwdNmzbNMTUeFhamKVOm6NixYy771h0AAICsshlZOfHOAiEhIXrzzTfVsmXLW8Z27Nih1157LcNnR+YUmzZt0vPPP68CBQpIkvbt26f27dsrJibmtudauoU6daRvv7W6CkAKDpb27FFSUorS0h7s1dzFivkoIYGrWl2N3luH3lvLmf2/ua17yRFHJt3RjBkz9OWXX6pHjx5KSUnR5MmTFRIS4r5BEgAAuCXC5B82bNiQ4dtm/qpatWqKjIx02v6mTJmisWPHqkWLFsqXL59CQkI0dOjQLG9n/PjxWrly5R3He/bsecsV5wAAAM6S7ae5XSUlJUUJCQl3HPfy8tJjjz3mwooy59y5c0pOTr7jeKFChVS4cGHXFXQ3THMju2Ca2+3Re+vQe2sxzW2hAgUKOM5fzEmKFi16z28QAgAAeFByxEcDAQAAIHsiTAIAAMA0wiQAAABMI0wCAADANMIkAAAATCNMAgAAwDTCJAAAAEwjTAIAAMA0wiQAAABMI0wCAADANMIkAAAATCNMAgAAwDTCJAAAAEwjTAIAAMA0wiQAAABMI0wCAADANMIkAAAATCNMAgAAwDTCJAAAAEwjTAIAAMA0wiQAAABMI0wCAADANMIkAAAATMtjdQHIRQICpJQUq6sApPLlra4AANwGYRKuExlpdQWAg5GWJrvdsLoMAMjxCJNwmaQkjkpaoUiRAvT+Nux2gzAJAE5AmITL2O122e1WV5G72Gw3/kxPt8sgNwEAHgAuwAEAAIBphEkAAACYRpgEAACAaYRJAAAAmEaYBAAAgGmESQAAAJhGmAQAAIBphEkAAACYRpgEAACAaYRJAAAAmEaYBAAAgGmESQAAAJhGmAQAAIBpeawuALmHh4eHPPjviyU8PWm8lei/dei9dei9a9jthux2w9IabIZhWFsBAAAATDHS0nTuwlVHoLTZpGLFfJSQkKz7TXg3t3UvHJmE63TvLu3ZY3UVAAC4h/LlZVu0SB4eNkuPThIm4ToHD0oxMVZXAQAAnIgTGgAAAGAaYRIAAACmESYBAABgGmESAAAAphEmAQAAYBphEgAAAKYRJgEAAGAaYRIAAACmESYBAABgGmESAAAAphEmAQAAYBphEgAAAKYRJgEAAGAaYRIAAACmESYBAABgGmESAAAAphEmAQAAYBphEgAAAKYRJgEAAGAaYRIAAACmESYBAABgGmESAAAAphEmAQAAYBphEgAAAKYRJgEAAGAaYRIAAACmESYBAABgGmESAAAApuWaMHnq1CkFBATo1KlTTtleQECAduzY4ZRtAQAA5FS5JkwCAADA+QiTAAAAMC3Xhcn169erbt26qlq1qkaOHKnU1FRJUkxMjMLCwhQUFKSQkBAtWbIkw/NmzJih559/XjVq1NCKFSscyz/99FPVqFFDaWlpjmUbNmzQCy+8IMMw7lrLzan3rVu3KiQkRMHBwRo3bpwOHTqkli1bKigoSD179tSlS5ckSampqXr//fdVp04dVaxYUSEhIVq2bJlje99//72aN2+uypUrq0GDBlq6dKljbO3atXr55ZdVuXJlNW7cWJs2bcp0z7799ls1bdpUgYGB6t69u8aOHavBgwdn+vkAAMB95bowuXz5ck2bNk2zZ8/W119/rTlz5iguLk6dOnVS9erVtXr1avXt21cTJ07Uxo0bJUnLli1TVFSUJkyYoPnz52vVqlWO7TVo0EBXr17V9u3bHcvWrVunRo0ayWazZaqmjz76SDNnztTYsWO1cOFCvfnmm3r77bc1b948/fjjj1q5cqVjva1bt2r69Olav369WrRoobFjxyohIUHp6el666239Morr2jdunXq37+/xowZoyNHjigxMVEDBw5Uz549tX79erVq1UoDBgzQ+fPn71nbyZMn1atXLzVq1EjR0dGqXLmyFi1alIWOAwCAB81m+9/tr4/v55YZeR7cy8qehg4dqmrVqkmS+vfvrylTpujSpUuqUKGCBgwYIEl68sknFRcXp8jISL344otavny5OnXqpPr160uSxo0bp1dffVWSVKBAAdWvX1/r169X7dq1deXKFX311VdauHBhpmvq3bu3ypUrp3LlymnChAl69dVXVatWLUnS888/r/j4eElSuXLl9NxzzykoKEiS9MYbb+jDDz/UsWPHlCdPHp0/f17FihVTiRIlVKJECT366KN65JFHdPr0aV2/fl2PP/64/Pz81LVrVwUEBOihhx66Z20rVqxQYGCgevfu7ejZd999l+nXBgAAHqwiRQrcsszX18dl+891YTIwMNBxv0KFCkpISFBcXFyG5ZIUHBzsmCaOi4tTnz59HGNlypTRww8/7HjcpEkTDR8+XKNHj9bWrVv16KOPqlKlSpmuqWTJko77Xl5e8vPzy/D45lR8w4YNtW3bNn3wwQeKj4/XL7/8IklKT09X4cKFFRYWpuHDh2vmzJmqX7++WrVqpUKFCqlgwYJ64YUX1KVLF/n7+6tBgwZq06aN8ufPf8/aDh48qMqVK2dYFhQUpAsXLmT69QEAgAcnKSlF6el2STeOJvr6+igxMVn3ONvunm5u615y3TS3h8f/XvLNcxpvd4TObrcrPT39lnVvypPnfzm8bt26Sk9P165du7RhwwY1atQoSzV5enrescY/mzZtmt59913lyZNHLVq0yHC+pCSNHj1a//nPf9S2bVvt3btXbdu21VdffSWbzaY5c+ZoxYoVevnll/Xll18qNDRUBw4cyFRtf33t9zoXFAAAuJZh/O/218f3c8uMXBcmDx065Li/b98+Pf744/L399fevXszrBcTEyN/f39J0tNPP62ffvrJMXbq1CldvHjR8Thfvnx68cUXtXHjRm3bts0xBe5sS5cu1YgRI/TOO++ocePGunLliqQb4e7s2bMaM2aMSpUqpV69emnVqlV67rnntGXLFsXFxWnixIkKDAzU3//+d33++ef629/+pm+++eae+3z66af1888/Z1j218cAACD3ynVhcuzYsdq7d6+2bdumiIgIde7cWe3bt9eBAwc0depUHT16VGvWrNHixYvVoUMHSVLHjh0VFRWlDRs26NChQxo2bNgtRw+bNGmilStX6vHHH9fTTz/9QGovXLiwvvzyS508eVK7d+/WwIEDJd24yrtQoULauHGjJkyYoBMnTmjXrl2KjY1VhQoVVLBgQS1ZskQzZ87UyZMntXXrVp0+fVoVKlS45z7btm2rH3/8UR999JGOHj2q2bNna/fu3Zm+uAgAALi3XHfOZFhYmHr16qXr16+rbdu26tSpkzw8PDRnzhxNmjRJH3/8sYoXL67BgwerVatWkqTmzZsrKSlJY8eO1dWrV9WjRw/FxsZm2G6NGjVUoEABNW7c+IHVPmHCBI0ePVqvvvqqHnvsMbVp00aenp46cOCA6tatq5kzZ2rChAlq1qyZChQooNatW6tNmzby8PDQ9OnTNWXKFM2ePVu+vr4aMGCAateufc99+vn5KSIiQhMnTlRERIRq1aqlBg0aKG/evA/sdQIAgJzDZnACnFNcunRJtWrV0n/+858MF9TkdIcOHVJaWlqGo5g9evRQ5cqV1bdv36xtrE4d6dtvnVwhAAC5VHCwtGePkpJSlJb2vwtwihXzUUKCcy7AKVaMC3AeOMMwtH79eo0cOVLBwcFuFSQl6cSJE+rSpYu2bdum06dPa8WKFfr+++/14osvWl0aAADIBnLdNLez2Ww2TZ48WZ6enpo1a1aGsRo1ajg+1ud2Pv/8cxUvXvxBl3hH+/btU6dOne44Xrx4cX3++ec6fPiwhg0bpsTERPn7+2vatGkqV66cCysFAADZFdPcD9DJkydlt9vvOO7n55fhI4ZcLTU1Vb/++usdx/PkyZPhMy/vG9PcAAA4TzaZ5ubI5AOU3ae88+XLp1KlSlldBgAAyME4ZxIAAACmESYBAABgGmESAAAAphEmAQAAYBphEgAAAKYRJgEAAGAaYRIAAACmESYBAABgGmESAAAAphEmAQAAYBphEgAAAKYRJgEAAGAaYRIAAACmESYBAABgGmESAAAAphEmAQAAYBphEgAAAKYRJgEAAGAaYRIAAACmESYBAABgGmESAAAAphEmAQAAYBphEgAAAKblsboA5CIBAVJKitVVAADgHsqXt7oCSYRJuFJkpNUVAADgVoy0NNnthqU1ECbhMklJHJW0QpEiBei9hei/dei9dei969jtBmESuYfdbpfdbnUVuYvNduPP9HS7DGt/1+RK9N869N469D734QIcAAAAmEaYBAAAgGmESQAAAJhGmAQAAIBphEkAAACYRpgEAACAaYRJAAAAmEaYBAAAgGmESQAAAJhGmAQAAIBphEkAAACYRpgEAACAaYRJAAAAmJbH6gKQe9hsN25wnZv9pu/WoP/WoffWoffWcmb/M7sNm2EYxv3vDgAAALkR09wAAAAwjTAJAAAA0wiTAAAAMI0wCQAAANMIkwAAADCNMAkAAADTCJMAAAAwjTAJAAAA0wiTAAAAMI0wCae4du2ahg4dqmeeeUa1a9fWxx9/fMd1f/nlF7Vp00ZVqlRRq1attH//fhdW6n6y0vutW7eqefPmCg4OVtOmTbV582YXVuqestL/m06dOqXg4GDt2LHDBRW6r6z0/uDBgwoLC1NgYKCaNm2q7du3u7BS95OV3m/cuFGNGjVScHCwwsLC9PPPP7uwUveWmpqqJk2a3PV3iSvecwmTcIpJkyZp//79WrBggUaNGqUZM2Zo/fr1t6x3+fJl9ejRQ88884xWr16t4OBg9ezZU5cvX7agaveQ2d7HxsbqzTffVKtWrRQdHa127dqpf//+io2NtaBq95HZ/v/Z6NGj+Zl3gsz2Pjk5WV27dlWZMmX02Wef6cUXX9Sbb76pxMREC6p2D5nt/eHDh/X222+rZ8+e+uSTT1S+fHn17NlTV65csaBq93Lt2jUNGDBAhw8fvuM6LnvPNYD7lJKSYlSuXNnYvn27Y9mHH35odOzY8ZZ1V6xYYYSEhBh2u90wDMOw2+3Giy++aKxatcpl9bqTrPR+8uTJRrdu3TIs69q1qzF16tQHXqe7ykr/b/rkk0+Mdu3aGWXLls3wPGRNVnq/YMECo2HDhkZaWppjWcuWLY2tW7e6pFZ3k5Xe/+tf/zJCQ0Mdj5OTk42yZcsa+/btc0mt7urw4cNGs2bNjKZNm971d4mr3nM5Mon7Fhsbq7S0NAUHBzuWVatWTXv37pXdbs+w7t69e1WtWjXZbDZJks1mU9WqVfXjjz+6smS3kZXeh4aG6p133rllG8nJyQ+8TneVlf5LUlJSkiZPnqz33nvPlWW6paz0fufOnWrQoIE8PT0dy1atWqV69eq5rF53kpXeFy5cWEeOHNEPP/wgu92u1atXy9vbW0888YSry3YrO3fuVI0aNbRs2bK7rueq99w8Tt0acqWzZ8+qSJEiypcvn2NZsWLFdO3aNZ0/f15FixbNsG6ZMmUyPN/X1/euh+lxZ1np/VNPPZXhuYcPH9b333+vdu3auaxed5OV/kvSBx98oNDQUD399NOuLtXtZKX3J0+eVGBgoEaMGKEtW7bIz89PgwYNUrVq1awoPcfLSu8bN26sLVu2qH379vL09JSHh4fmzJmjQoUKWVG622jfvn2m1nPVey5HJnHfrly5kuGXiiTH49TU1Eyt+9f1kDlZ6f2fnTt3Tn379lXVqlXVoEGDB1qjO8tK/7/77jv98MMP6t27t8vqc2dZ6f3ly5f10Ucf6ZFHHtHcuXNVvXp1devWTb/++qvL6nUnWel9UlKSzp49q5EjR2r58uVq3ry5hgwZwvmqLuKq91zCJO7bQw89dMsP5s3HXl5emVr3r+shc7LS+5sSEhLUqVMnGYahiIgIeXjwa8CszPb/6tWrGjlypEaNGsXPupNk5Wff09NT5cuXV79+/VShQgW9++67Kl26tD755BOX1etOstL7KVOmqGzZsurQoYMqVaqksWPHKn/+/Fq1apXL6s3NXPWey7sI7ttjjz2mpKQkpaWlOZadPXtWXl5eKliw4C3rJiQkZFiWkJCgRx991CW1upus9F6SfvvtN3Xo0EGpqamKioq6ZRoWWZPZ/u/bt08nT55Uv379FBwc7DjX7PXXX9fIkSNdXrc7yMrP/iOPPKInn3wyw7LSpUtzZNKkrPT+559/Vrly5RyPPTw8VK5cOZ05c8Zl9eZmrnrPJUzivpUvX1558uTJcELvDz/8oMqVK99y1KtKlSqKiYmRYRiSJMMwtGfPHlWpUsWVJbuNrPT+8uXL6t69uzw8PPTvf/9bjz32mIurdT+Z7X9gYKC++OILRUdHO26SNG7cOPXv39/FVbuHrPzsBwUF6eDBgxmWxcfHy8/PzxWlup2s9P7RRx9VXFxchmVHjx5ViRIlXFFqrueq91zCJO5b/vz51aJFC40ePVr79u3Tpk2b9PHHH+u1116TdON/rFevXpUkvfLKK7p48aLGjx+vI0eOaPz48bpy5YoaNWpk5UvIsbLS+zlz5ujEiROaOHGiY+zs2bNczX0fMtt/Ly8vlSpVKsNNunHUwNfX18qXkGNl5We/Xbt2OnjwoKZPn67jx4/r//7v/3Ty5Ek1b97cypeQY2Wl923bttXy5csVHR2t48ePa8qUKTpz5oxCQ0OtfAluzZL3XKd+0BByrcuXLxsDBw40goKCjNq1axv/+te/HGNly5bN8JlWe/fuNVq0aGFUrlzZaN26tfHzzz9bULH7yGzvX375ZaNs2bK33AYNGmRR5e4hKz/7f8bnTN6/rPR+9+7dRmhoqFGpUiWjefPmxs6dOy2o2H1kpffLly83XnnlFSMoKMgICwsz9u/fb0HF7uuvv0useM+1GcYfxz4BAACALGKaGwAAAKYRJgEAAGAaYRIAAACmESYBAABgGmESAAAAphEmAQAAYBphEgAAAKYRJgEAAGAaYRIAXCg8PFzTp0+3ugxJUmJiotatW2d1GQByOMIkAORSU6ZM0VdffWV1GQByOMIkAORSfJsuAGcgTAKARVavXq3w8HDNmjVL1atXV61atRQdHa3169erfv36euaZZzR58mTH+iEhIZo/f76aNm2qoKAg9ejRQ2fPnnWMx8XFqVu3bqpatarq1KmjGTNmyG63S5KmT5+u3r17q0OHDnr22WcVHh6uNWvWaM2aNQoJCZEkHTlyRN26dVNwcLAqV66s9u3bKy4uTpK0Y8cOhYSEaPHixapTp46CgoL07rvvKjU11bH/Tz75RK+88oqqVKmidu3a6ZdffnGMLV26VCEhIQoODlZ4eLgOHjx4x76sXbtWL7/8sipXrqzGjRtr06ZNjrHjx487anzhhRcUFRVl6vXv3LlTqampGjdunGrUqKEaNWronXfe0fnz5838VQK5GmESACwUExOjkydPauXKlXr11Vc1evRoRUVFadasWRo8eLAiIyMzhLLp06ere/fuWrZsma5cuaK+fftKks6dO6f27dvr0Ucf1YoVKzRq1Cj9+9//zhC2Nm/erCZNmmjBggWaNWuWGjVqpEaNGmnlypWy2+1644035Ofnp08++URLly5Venp6hjD7+++/a8OGDYqMjNT06dP1xRdfKDo6WpL0zTffaNiwYerUqZM+/fRTVapUST179lRqaqq2bNmiGTNmaMSIEVqzZo2qVaum1157TRcuXLilH4mJiRo4cKB69uyp9evXq1WrVhowYIDOnz+va9euqWvXripQoICWL1+ukSNHatq0afryyy+z/PoDAwM1depU7d+/X3PnzlVUVJQuXbqk/v37O/uvGHB/BgDAZTp27GhEREQYhmEYq1atMipUqGCkpKQYhmEYR44cMcqWLWt89913jvWff/5547PPPjMMwzDq169vjB8/3jF24sQJo2zZssbBgweNBQsWGPXq1TOuX7/uGF+8eLFRq1YtwzAMIyIiwqhZs2aGWgYNGmQMGjTIMAzDSElJMebOneuoxTAMY8mSJUaDBg0MwzCM7du3G2XLljUOHTrkGO/Tp48xfPhwx/2b2zIMw7h27ZrxwQcfGL///rsRFhZmREVFZdh3aGjoLcsMwzB+/vlno2zZssa2bdsMwzAMu91ufPPNN8bly5eNTZs2GUFBQUZycrJj/ZUrVxpbt27N8uu/fPmyUbFiRSM2Ntax7MKFC0a5cuUyLANwb3msDrMAkJv5+vrq4YcfliQ99NBDkqQSJUo4xr28vDJMJVetWtVxv2TJkipcuLDi4uIUFxenihUrKk+e//1aDw4O1tmzZ3Xx4kVJkp+f3x3rePjhhxUWFqbo6Gjt379f8fHx+uWXX1SsWLEM65UqVcpx39vbW2lpaZKko0ePql27do6xfPnyadCgQZJuTD9PnjxZU6dOdYxfu3ZNx44du6WO8uXL64UXXlCXLl3k7++vBg0aqE2bNsqfP7+OHj0qf39/eXt7O9Zv1aqVJGnUqFFZev0nT57U9evXM9QsSXa7XceOHVNAQMAdewUgI8IkAFjoz+HnJpvNlun109PT5eHh4Qiif3bzfMH09HRJuu06N6WkpKh169YqUqSIQkJC1KRJE8XHx+vjjz/OsF6+fPkyPDb+uIjndq/jzzUOHTpUzz//fIblfw6FN9lsNs2ZM0f79u3T5s2btXHjRi1evFiLFy++6z6y+vpvLlu8eLEjzN/k6+t7x/0AuBXnTAJADhIbG+u4f/z4cSUnJysgIED+/v76+eefdf36dcd4TEyMihYtqsKFC992W38OrTt37tTvv/+uqKgode/eXTVr1tSZM2cyfcV3qVKlMtSWnp6ukJAQ/fDDD/L399d///tflSpVynGbPXu2fvzxx1u2ExcXp4kTJyowMFB///vf9fnnn+tvf/ubvvnmG5UuXVrHjx/XlStXHOtPnDhR48aNy/LrL1mypDw9PXX+/HlHTd7e3nr//feVmJiYqdcM4AbCJADkIFFRUdq8ebNiY2M1dOhQ1apVS6VLl1bTpk2VmpqqkSNHKi4uTps2bdL06dMVFhZ2xyOd+fPn1+nTp/Xbb7+pcOHCunz5sjZt2qRTp05pxYoVWrRoUYYp9rsJDw/Xp59+qjVr1uj48eN6//33ZRiGKlasqC5dumjBggWKjo7WiRMnNHnyZK1bt05PPfXULdspWLCglixZopkzZ+rkyZPaunWrTp8+rQoVKqh27doqVqyY4zVu3rxZS5cuVe3atbP8+r29vdWmTRuNHj1aO3bs0JEjRzRw4EAdP348w2kGAO6NaW4AyEFCQ0M1depUnTlzRvXq1dOYMWMk3QhHkZGRGj9+vFq0aKGiRYuqU6dO6tmz5x231bx5c/Xp00fNmjXT9u3b1adPH40ZM0bXrl1TQECARo4cqWHDhum33367Z13Vq1fXqFGj9OGHH+rs2bOqVKmSZs+eLS8vLzVu3FgJCQmKiIhQQkKCypQpo1mzZql06dK3bOeRRx7R9OnTNWXKFM2ePVu+vr4aMGCAateuLUmaOXOm3nvvPYWGhqpYsWIaOHCgXnjhBUnK8usfPHiwJk6cqH79+un69euqXr26PvroI3l6et7z9QL4H5uR2TkMAIClQkJC9Oabb6ply5ZWlwIADkxzAwAAwDTCJAAAAExjmhsAAACmcWQSAAAAphEmAQAAYBphEgAAAKYRJgEAAGAaYRIAAACmESYBAABgGmESAAAAphEmAQAAYNr/B53k60wBjvrHAAAAAElFTkSuQmCC" + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "from dianna.visualization import plot_tabular\n", + "\n", + "_ = plot_tabular(explanation[np.argmax(predictions)], X_test.columns, num_features=10)" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "outputs": [ + { + "data": { + "text/plain": "array([[6.4450759e-01, 6.8236494e-01, 6.4905328e-01, 8.5704887e-06],\n [2.1436421e-02, 2.1601211e-02, 9.7484328e-04, 7.2740775e-05],\n [3.0805644e-01, 3.3603430e-01, 3.4597191e-01, 9.8991853e-01]],\n dtype=float32)" + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "explanation" + ], + "metadata": { + "collapsed": false, + "ExecuteTime": { + "start_time": "2024-05-30T11:11:53.561551Z", + "end_time": "2024-05-30T11:11:53.565301Z" + } + } + } + ], + "metadata": { + "kernelspec": { + "display_name": "dianna", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +}