Skip to content

Sources

Sources of information indexed by object -- the instantiation of the method.

The method needs, for each object and each layer, one array per source. Any extractor that returns a Sources works with Interpreter:

  • TorchSources -- activations, contrastive gradient and activated parameters of any PyTorch module (MLP, CNN, transformer);
  • TreeSources -- path, slack and class distribution per depth of a scikit-learn tree;
  • FunctionSources -- wrap your own fn(X) -> {layer: {source: array}}.

Sources dataclass

Sources(
    layers: dict[str, dict[str, ndarray]],
    predictions: ndarray,
    probabilities: ndarray | None = None,
    logits: ndarray | None = None,
    meta: dict = dict(),
)

Per-layer sources for n objects and the model's behaviour on them.

layers[layer][source] is an array whose first axis indexes the objects.

layers instance-attribute

layers: dict[str, dict[str, ndarray]]

predictions instance-attribute

predictions: ndarray

probabilities class-attribute instance-attribute

probabilities: ndarray | None = None

logits class-attribute instance-attribute

logits: ndarray | None = None

meta class-attribute instance-attribute

meta: dict = field(default_factory=dict)

layer_names property

layer_names: list[str]

TorchSources

TorchSources(
    model: Module,
    layers: Sequence[str | Module] | None = None,
    *,
    target: str | int = "fixed",
    fixed_class: int = 0,
    weight_modules: Mapping[str, str | Module] | None = None,
    sources: Sequence[str] = SOURCE_NAMES,
    grid: int | None = 4,
    layout: str = "channels",
    margin: str | Callable = "contrastive",
    device=None,
    batch_size: int = 256,
    dtype=float32,
)

Extracts the three sources of the paper from any torch.nn.Module classifier.

Parameters:

Name Type Description Default
model Module

A trained classifier returning logits (N, K).

required
layers Sequence[str | Module] | None

Module names (as in model.named_modules()) or modules to probe. None probes every direct child that has parameters.

None
target str | int

"fixed" (class fixed_class for every object), "predicted" or an int.

'fixed'
weight_modules Mapping[str, str | Module] | None

Optional {layer: module or name} giving the layer whose weight defines the parameter source; by default it is found automatically.

None
sources Sequence[str]

Which sources to extract (subset of ("parameters", "activations", "gradients")).

SOURCE_NAMES
grid int | None

Positions kept per spatial/token axis (None keeps all).

4
margin str | Callable

"contrastive" (default) or a callable fn(logits, target) -> (N,).

'contrastive'
Source code in src\fannar\sources\torch.py
def __init__(self, model: nn.Module, layers: Sequence[str | nn.Module] | None = None, *,
             target: str | int = "fixed", fixed_class: int = 0,
             weight_modules: Mapping[str, str | nn.Module] | None = None,
             sources: Sequence[str] = SOURCE_NAMES, grid: int | None = 4, layout: str = "channels",
             margin: str | Callable = "contrastive", device=None, batch_size: int = 256,
             dtype=torch.float32) -> None:
    self.model = model
    self.device = torch.device(device) if device else next(model.parameters()).device
    named = dict(model.named_modules())
    if layers is None:
        layers = [n for n, m in model.named_children() if any(True for _ in m.parameters())]
    self.layers: dict[str, nn.Module] = {}
    for layer in layers:
        if isinstance(layer, str):
            if layer not in named:
                raise KeyError(f"module {layer!r} not found; available: {list(named)[:20]}...")
            self.layers[layer] = named[layer]
        else:
            name = next((n for n, m in named.items() if m is layer), None) or f"layer{len(self.layers)}"
            self.layers[name] = layer
    self.target, self.fixed_class = target, fixed_class
    self.weight_modules = {k: (named[v] if isinstance(v, str) else v) for k, v in (weight_modules or {}).items()}
    unknown = set(sources) - set(SOURCE_NAMES)
    if unknown:
        raise ValueError(f"unknown sources {unknown}; choose from {SOURCE_NAMES}")
    self.sources, self.grid, self.layout = tuple(sources), grid, layout
    self.margin, self.batch_size, self.dtype = margin, batch_size, dtype
    self._cap = False

model instance-attribute

model = model

device instance-attribute

device = torch.device(device) if device else next(model.parameters()).device

layers instance-attribute

layers: dict[str, Module] = {}

weight_modules instance-attribute

weight_modules = {
    k: named[v] if isinstance(v, str) else v for k, v in (weight_modules or {}).items()
}

positions

positions(x, layer: str, max_side: int | None = 32) -> dict[str, ndarray]

Sources with the positions of one input as objects: {source: (P, C or F)}.

Used for energy maps over an image; max_side caps the grid (the Gram is P x P).

Source code in src\fannar\sources\torch.py
def positions(self, x, layer: str, max_side: int | None = 32) -> dict[str, np.ndarray]:
    """Sources with the *positions* of one input as objects: ``{source: (P, C or F)}``.

    Used for energy maps over an image; ``max_side`` caps the grid (the Gram is ``P x P``).
    """
    x = torch.as_tensor(np.asarray(x) if not isinstance(x, torch.Tensor) else x)[None]
    grid, self.grid, self._cap = self.grid, max_side, True
    try:
        src = self(x)
    finally:
        self.grid, self._cap = grid, False
    return {s: v[0].T for s, v in src.layers[layer].items()}

position_shape

position_shape(x, layer: str, max_side: int | None = 32) -> tuple[int, ...]

Spatial shape of the position grid returned by positions for layer.

Source code in src\fannar\sources\torch.py
def position_shape(self, x, layer: str, max_side: int | None = 32) -> tuple[int, ...]:
    """Spatial shape of the position grid returned by :meth:`positions` for ``layer``."""
    shape = {}
    def grab(_m, _i, out):  # must return None: a returned value would replace the module output
        shape.setdefault("s", _tensor_output(out).shape)

    h = self.layers[layer].register_forward_hook(grab)
    try:
        with torch.no_grad():
            self.model.eval()
            xb = torch.as_tensor(np.asarray(x) if not isinstance(x, torch.Tensor) else x)[None]
            xb = xb.to(self.device, self.dtype) if xb.is_floating_point() else xb.to(self.device)
            self.model(xb)
    finally:
        h.remove()
    s = shape["s"]
    if len(s) == 4:
        return tuple(min(max_side, v) if max_side else v for v in s[2:])
    if len(s) == 3:
        length = s[1] if self.layout == "tokens" else s[2]
        return (min(max_side, length) if max_side else length,)
    return (1,)

TreeSources

TreeSources(tree, scale=None, depths=None)

Per-depth sources of a fitted DecisionTreeClassifier (or any sklearn tree with tree_).

Parameters:

Name Type Description Default
tree

Fitted scikit-learn decision tree classifier.

required
scale

Per-feature scale for the slack (e.g. the training standard deviation); defaults to 1.

None
depths

Depths to use as layers; defaults to 1 .. max_depth.

None
Source code in src\fannar\sources\tree.py
def __init__(self, tree, scale=None, depths=None) -> None:
    t = tree.tree_
    self.tree = tree
    self.internal = t.children_left != -1
    self.feature = np.where(self.internal, t.feature, 0)
    self.threshold = t.threshold
    value = t.value[:, 0, :]
    self.value = value / np.maximum(value.sum(1, keepdims=True), 1e-300)
    self.depth = np.zeros(t.node_count, dtype=int)
    for n in range(t.node_count):  # pre-order: parents before children
        for child in (t.children_left[n], t.children_right[n]):
            if child != -1:
                self.depth[child] = self.depth[n] + 1
    self.max_depth = int(self.depth.max())
    self.scale = None if scale is None else np.asarray(scale, float)
    self.depths = list(depths) if depths is not None else list(range(1, self.max_depth + 1))

tree instance-attribute

tree = tree

internal instance-attribute

internal = t.children_left != -1

feature instance-attribute

feature = np.where(self.internal, t.feature, 0)

threshold instance-attribute

threshold = t.threshold

value instance-attribute

value = value / np.maximum(value.sum(1, keepdims=True), 1e-300)

depth instance-attribute

depth = np.zeros(t.node_count, dtype=int)

max_depth instance-attribute

max_depth = int(self.depth.max())

scale instance-attribute

scale = None if scale is None else np.asarray(scale, float)

depths instance-attribute

depths = list(depths) if depths is not None else list(range(1, self.max_depth + 1))

at_depth

at_depth(X, depth: int) -> dict[str, ndarray]
Source code in src\fannar\sources\tree.py
def at_depth(self, X, depth: int) -> dict[str, np.ndarray]:
    X = np.asarray(X, dtype=np.float64)
    scale = np.ones(X.shape[1]) if self.scale is None else self.scale
    P = self.tree.decision_path(X.astype(np.float32)).toarray().astype(np.float64)
    path = P * (self.depth <= depth)
    crossed = P * (self.depth < depth) * self.internal
    slack = crossed * (X[:, self.feature] - self.threshold) / scale[self.feature]
    node = (path * np.arange(path.shape[1])).argmax(1)  # deepest node of the prefix
    return {"path": path, "slack": slack, "classes": self.value[node]}

FunctionSources

FunctionSources(fn: Callable)

Wraps fn(X) -> Sources or fn(X) -> (layers_dict, predictions[, probabilities]).

Source code in src\fannar\sources\__init__.py
def __init__(self, fn: Callable) -> None:
    self.fn = fn

fn instance-attribute

fn = fn