Skip to content

Interpreter

Interpreter

Interpreter(
    model,
    X,
    y=None,
    *,
    layers: Sequence[str] | None = None,
    pipeline: KernelPipeline | str = "paper",
    reference_model=None,
    baselines: Sequence[KernelPipeline | str] | dict = ("activations",),
    extractor: Callable | None = None,
    feature_names: Sequence[str] | None = None,
    class_names: Sequence[str] | None = None,
    k: int = 5,
    **source_options,
)

Builds the representation space of a trained model over chosen objects, per layer.

Parameters:

Name Type Description Default
model

A trained PyTorch classifier, a fitted scikit-learn decision tree, or anything else when extractor is given.

required
X

The objects to interpret (array, tensor or, for PyTorch, a DataLoader).

required
y

True labels of X (needed for the correctness, confusion and containment readings).

None
layers Sequence[str] | None

Layers to probe (PyTorch module names); ignored for trees and custom extractors.

None
pipeline KernelPipeline | str

A KernelPipeline, or a preset name ("paper" by default). For your own Gram construction, pass a pipeline whose components use your kernels, or call space_from_grams.

'paper'
reference_model

Optional model of the same architecture used as the floor (typically the same network with its initial weights). Its geometry is measured against the trained model's behaviour.

None
baselines Sequence[KernelPipeline | str] | dict

Other pipelines or preset names evaluated alongside (default: activations alone).

('activations',)
extractor Callable | None

callable(X) -> Sources (or FunctionSources) for models that are neither PyTorch nor sklearn trees.

None
**source_options

Passed to TorchSources (target, fixed_class, grid, weight_modules, device, batch_size...).

{}
Source code in src\fannar\interpreter.py
def __init__(self, model, X, y=None, *, layers: Sequence[str] | None = None,
             pipeline: KernelPipeline | str = "paper", reference_model=None,
             baselines: Sequence[KernelPipeline | str] | dict = ("activations",),
             extractor: Callable | None = None, feature_names: Sequence[str] | None = None,
             class_names: Sequence[str] | None = None, k: int = 5, **source_options) -> None:
    self.model, self.X, self.k = model, X, k
    self.y = None if y is None else np.asarray(y)
    self.feature_names = list(feature_names) if feature_names is not None else None
    self.class_names = list(class_names) if class_names is not None else None
    self.extractor = self._make_extractor(model, extractor, layers, source_options)
    self.reference_extractor = (None if reference_model is None
                                else self._make_extractor(reference_model, None, layers, source_options))
    self.pipeline = get_pipeline(pipeline)
    if _is_sklearn_tree(model) and pipeline == "paper":
        self.pipeline = KernelPipeline(["path", "slack", "classes"])
        if tuple(baselines) == ("activations",):
            baselines = {"path": KernelPipeline(["path"])}
    if isinstance(baselines, dict):
        self.baselines = {k: get_pipeline(v) for k, v in baselines.items()}
    else:
        self.baselines = {(b if isinstance(b, str) else f"baseline{t}"): get_pipeline(b)
                          for t, b in enumerate(baselines)}
    self.sources: Sources | None = None
    self.reference_sources: Sources | None = None
    self.spaces: dict[str, GramSpace] = {}

y instance-attribute

y = None if y is None else np.asarray(y)

feature_names instance-attribute

feature_names = list(feature_names) if feature_names is not None else None

class_names instance-attribute

class_names = list(class_names) if class_names is not None else None

extractor instance-attribute

extractor = self._make_extractor(model, extractor, layers, source_options)

reference_extractor instance-attribute

reference_extractor = (
    None
    if reference_model is None
    else self._make_extractor(reference_model, None, layers, source_options)
)

pipeline instance-attribute

pipeline = get_pipeline(pipeline)

baselines instance-attribute

baselines = {k: get_pipeline(v) for k, v in baselines.items()}

sources instance-attribute

sources: Sources | None = None

reference_sources instance-attribute

reference_sources: Sources | None = None

spaces instance-attribute

spaces: dict[str, GramSpace] = {}

layers property

layers: list[str]

predictions property

predictions: ndarray

fit

fit() -> 'Interpreter'

Extracts the sources and builds the representation space of every layer.

Source code in src\fannar\interpreter.py
def fit(self) -> "Interpreter":
    """Extracts the sources and builds the representation space of every layer."""
    self.sources = self.extractor(self.X)
    if self.reference_extractor is not None:
        self.reference_sources = self.reference_extractor(self.X)
    self.spaces = {layer: self._build(self.pipeline, self.sources, layer, "joint") for layer in self.layers}
    return self

space

space(layer: str) -> GramSpace

The GramSpace of one layer.

Source code in src\fannar\interpreter.py
def space(self, layer: str) -> GramSpace:
    """The :class:`GramSpace` of one layer."""
    self._ensure()
    return self.spaces[layer]

baseline_space

baseline_space(name: str, layer: str) -> GramSpace
Source code in src\fannar\interpreter.py
def baseline_space(self, name: str, layer: str) -> GramSpace:
    self._ensure()
    return self._build(self.baselines[name], self.sources, layer, name)

floor_space

floor_space(layer: str) -> GramSpace

The same pipeline on the reference (e.g. untrained) model.

Source code in src\fannar\interpreter.py
def floor_space(self, layer: str) -> GramSpace:
    """The same pipeline on the reference (e.g. untrained) model."""
    self._ensure()
    if self.reference_sources is None:
        raise ValueError("no reference_model was given")
    return self._build(self.pipeline, self.reference_sources, layer, "floor")

space_from_grams

space_from_grams(
    grams: dict[str, ndarray],
    pipeline: KernelPipeline | None = None,
    name: str = "custom",
) -> GramSpace

A space from Gram matrices you built yourself ({component: (n, n)}), composed by pipeline (default: the interpreter's). Components not in grams must be sources.

Source code in src\fannar\interpreter.py
def space_from_grams(self, grams: dict[str, np.ndarray], pipeline: KernelPipeline | None = None,
                     name: str = "custom") -> GramSpace:
    """A space from Gram matrices you built yourself (``{component: (n, n)}``), composed by
    ``pipeline`` (default: the interpreter's). Components not in ``grams`` must be sources."""
    self._ensure()
    pipe = pipeline or KernelPipeline(list(grams))
    return pipe(grams=grams, labels=self.y, predictions=self.predictions, name=name)

evaluate

evaluate(
    layers: Sequence[str] | None = None,
    include_baselines: bool = True,
    include_floor: bool = True,
) -> Report

Behaviour readings per layer for the joint space, the baselines and the floor.

The floor uses the reference model's geometry against the trained model's behaviour (the "cross" floor), so the only difference between a reading and its floor is training.

Source code in src\fannar\interpreter.py
def evaluate(self, layers: Sequence[str] | None = None, include_baselines: bool = True,
             include_floor: bool = True) -> Report:
    """Behaviour readings per layer for the joint space, the baselines and the floor.

    The floor uses the reference model's *geometry* against the trained model's *behaviour*
    (the "cross" floor), so the only difference between a reading and its floor is training.
    """
    self._ensure()
    if self.y is None:
        raise ValueError("evaluate() needs the true labels `y`")
    layers = list(layers or self.layers)
    preds, probs = self.sources.predictions, self.sources.probabilities
    run = lambda sp: evaluate_space(sp, self.y, preds, probs, self.k)  # noqa: E731
    values = {"joint": {layer: run(self.spaces[layer]) for layer in layers}}
    if include_baselines:
        for name in self.baselines:
            ok = [layer for layer in layers if self._has(self.baselines[name], layer)]
            if ok:
                values[name] = {layer: run(self.baseline_space(name, layer)) for layer in ok}
    if include_floor and self.reference_sources is not None:
        values["floor"] = {layer: run(self.floor_space(layer)) for layer in layers}
    accuracy = float((preds == self.y).mean())
    return Report(layers, values, {"accuracy": accuracy, "n": len(self.y), "pipeline": repr(self.pipeline)})

explain_pair

explain_pair(
    i: int, j: int | None = None, layers: Sequence[str] | None = None
) -> PairExplanation

Which features distinguish object i from j for the model (j defaults to the nearest object with a different prediction). Vector inputs only.

Source code in src\fannar\interpreter.py
def explain_pair(self, i: int, j: int | None = None, layers: Sequence[str] | None = None) -> PairExplanation:
    """Which features distinguish object ``i`` from ``j`` for the model (``j`` defaults to the
    nearest object with a different prediction). Vector inputs only."""
    self._ensure()
    X = self._matrix()
    j = nearest_unlike(X, self.predictions, i) if j is None else j
    scores = pair_scores(self.extractor, self.pipeline, X[i], X[j], layers)
    return PairExplanation(i, j, X[i], X[j], scores, int(self.predictions[i]), int(self.predictions[j]),
                           self.feature_names)

explain_errors

explain_errors(
    layers: Sequence[str] | None = None,
    against: str = "correct",
    max_errors: int | None = None,
    reference: int = 100,
    seed: int = 0,
) -> ErrorExplanation

Which features drive the model's errors, aggregated over the misclassified objects.

Each error i is explained as a pair against a partner j: with against="correct" (default) the nearest object correctly predicted as the true class of i (falling back to the nearest unlike object when that class has none), so the shares say which features keep i away from its class in the representation space; with against="unlike", the nearest object with another prediction, as in explain_pair. max_errors caps the number of errors and reference the number of correctly predicted objects explained the same way as a baseline (both sampled with seed; reference=0 skips it). Vector inputs only.

Source code in src\fannar\interpreter.py
def explain_errors(self, layers: Sequence[str] | None = None, against: str = "correct",
                   max_errors: int | None = None, reference: int = 100, seed: int = 0) -> ErrorExplanation:
    """Which features drive the model's errors, aggregated over the misclassified objects.

    Each error ``i`` is explained as a pair against a partner ``j``: with ``against="correct"``
    (default) the nearest object correctly predicted as the true class of ``i`` (falling back
    to the nearest unlike object when that class has none), so the shares say which features
    keep ``i`` away from its class in the representation space; with ``against="unlike"``, the
    nearest object with another prediction, as in :meth:`explain_pair`. ``max_errors`` caps the
    number of errors and ``reference`` the number of correctly predicted objects explained the
    same way as a baseline (both sampled with ``seed``; ``reference=0`` skips it).
    Vector inputs only."""
    if self.y is None:
        raise ValueError("explaining errors needs the true labels `y`")
    if against not in ("correct", "unlike"):
        raise ValueError("against must be 'correct' or 'unlike'")
    self._ensure()
    X, pred, rng = self._matrix(), self.predictions, np.random.default_rng(seed)
    errors = np.flatnonzero(pred != self.y)
    if len(errors) == 0:
        raise ValueError("the model makes no errors on these objects")
    if max_errors is not None and len(errors) > max_errors:
        errors = np.sort(rng.choice(errors, size=max_errors, replace=False))
    partners = []
    for i in errors:
        j = nearest_correct(X, self.y, pred, int(i)) if against == "correct" else None
        partners.append(nearest_unlike(X, pred, int(i)) if j is None else j)
    partners = np.array(partners)
    correct = np.flatnonzero(pred == self.y)
    correct = np.sort(rng.choice(correct, size=min(reference, len(correct)), replace=False))
    return ErrorExplanation(errors, partners, self.y[errors], pred[errors],
                            self._pair_shares(X, errors, partners, layers), X[errors], X[partners],
                            self._pair_shares(X, correct, [nearest_unlike(X, pred, int(i)) for i in correct], layers),
                            {int(c): int((self.y == c).sum()) for c in np.unique(self.y)},
                            self.feature_names, self.class_names)

swap_test

swap_test(
    pairs: int | Sequence[tuple[int, int]] = 50,
    layer: str | None = None,
    predict: Callable | None = None,
    seed: int = 0,
) -> dict

Swap-test area per ranking (method, |Delta x|, random among differing features), averaged over pairs. pairs is a number of random correctly-predicted objects (each with its nearest unlike neighbour) or explicit (i, j) pairs.

Source code in src\fannar\interpreter.py
def swap_test(self, pairs: int | Sequence[tuple[int, int]] = 50, layer: str | None = None,
              predict: Callable | None = None, seed: int = 0) -> dict:
    """Swap-test area per ranking (method, |Delta x|, random among differing features),
    averaged over pairs. ``pairs`` is a number of random correctly-predicted objects (each with
    its nearest unlike neighbour) or explicit ``(i, j)`` pairs."""
    self._ensure()
    X = self._matrix()
    predict = predict or self._predict_fn()
    rng = np.random.default_rng(seed)
    if isinstance(pairs, int):
        pool = np.flatnonzero(self.predictions == self.y) if self.y is not None else np.arange(len(X))
        chosen = rng.choice(pool, size=min(pairs, len(pool)), replace=False)
        pairs = [(int(i), nearest_unlike(X, self.predictions, int(i))) for i in chosen]
    layer = layer or self.layers[-1]
    areas = {"method": [], "absdiff": [], "random": []}
    for i, j in pairs:
        s = pair_scores(self.extractor, self.pipeline, X[i], X[j], [layer])[layer]
        differ = np.flatnonzero(np.abs(X[j] - X[i]) > 1e-12)
        rest = np.setdiff1d(np.arange(X.shape[1]), differ)
        orders = {"method": np.argsort(-s, kind="stable"), "absdiff": np.argsort(-np.abs(X[j] - X[i]), kind="stable"),
                  "random": np.concatenate([rng.permutation(differ), rest])}
        for key, order in orders.items():
            areas[key].append(curve_area(swap_curve(predict, X[i], X[j], int(self.predictions[j]), order)))
    return {k: float(np.mean(v)) for k, v in areas.items()} | {"pairs": len(pairs), "layer": layer}

energy_map

energy_map(x, layer: str, max_side: int = 32) -> ndarray

Retained spectral energy of every position of one input (e.g. an image), as a map.

Positions are the objects; components use linear kernels without the angular transform (which would give every position the same energy).

Source code in src\fannar\interpreter.py
def energy_map(self, x, layer: str, max_side: int = 32) -> np.ndarray:
    """Retained spectral energy of every position of one input (e.g. an image), as a map.

    Positions are the objects; components use linear kernels without the angular transform
    (which would give every position the same energy)."""
    from .sources.torch import TorchSources
    if not isinstance(self.extractor, TorchSources):
        raise TypeError("energy maps need a PyTorch model")
    src = self.extractor.positions(x, layer, max_side)
    names = [s for s in ("parameters", "activations", "gradients") if s in src]
    space = positions_pipeline(names)(sources=src)
    shape = self.extractor.position_shape(x, layer, max_side)
    return space.energies()["E_par"].reshape(shape)

plot_report

plot_report(report: Report | None = None, readings=None)
Source code in src\fannar\interpreter.py
def plot_report(self, report: Report | None = None, readings=None):
    from .plots import plot_report
    return plot_report(report or self.evaluate(), readings)

plot_embedding

plot_embedding(
    layers: Sequence[str] | None = None,
    color_by: str = "prediction",
    include_floor: bool = True,
    include_baselines: bool = True,
)
Source code in src\fannar\interpreter.py
def plot_embedding(self, layers: Sequence[str] | None = None, color_by: str = "prediction",
                   include_floor: bool = True, include_baselines: bool = True):
    from .plots import plot_embedding
    self._ensure()
    layers = list(layers or self.layers)
    rows = {"joint": [self.spaces[layer] for layer in layers]}
    if include_baselines:
        for name in self.baselines:
            if all(self._has(self.baselines[name], layer) for layer in layers):
                rows[name] = [self.baseline_space(name, layer) for layer in layers]
    if include_floor and self.reference_sources is not None:
        rows["floor"] = [self.floor_space(layer) for layer in layers]
    colors = self.predictions if color_by == "prediction" else self.y
    return plot_embedding(rows, layers, colors, self.y, self.predictions, self.class_names)

plot_errors

plot_errors(k: int | None = None, layer: str | None = None, rows: str = 'class', **kw)
Source code in src\fannar\interpreter.py
def plot_errors(self, k: int | None = None, layer: str | None = None, rows: str = "class", **kw):
    return self.explain_errors(**kw).plot(k, layer, rows)

plot_gram

plot_gram(layer: str)
Source code in src\fannar\interpreter.py
def plot_gram(self, layer: str):
    from .plots import plot_gram
    return plot_gram(self.space(layer), self.y if self.y is not None else self.predictions)

plot_spectrum

plot_spectrum(layer: str)
Source code in src\fannar\interpreter.py
def plot_spectrum(self, layer: str):
    from .plots import plot_spectrum
    return plot_spectrum(self.space(layer))

plot_energy_map

plot_energy_map(x, layer: str, max_side: int = 32, image=None)
Source code in src\fannar\interpreter.py
def plot_energy_map(self, x, layer: str, max_side: int = 32, image=None):
    from .plots import plot_energy_map
    return plot_energy_map(self.energy_map(x, layer, max_side), image)