Skip to content

Stopping Criteria

Stopping criterion are used to determine when to stop your active learning experiment.

Their usage are simple, but best put in practice with ActiveExperiment.

Example 

from baal.active.stopping_criteria import LabellingBudgetStoppingCriterion
from baal.active.dataset import ActiveLearningDataset

al_dataset: ActiveLearningDataset = ... # len(al_dataset) == 10
criterion = LabellingBudgetStoppingCriterion(al_dataset, labelling_budget=100)

assert not criterion.should_stop({}, [])

# len(al_dataset) == 60
al_dataset.label_randomly(50)
assert not criterion.should_stop({}, [])

# len(al_dataset) == 110, budget exhausted! We've labelled 100 items.
al_dataset.label_randomly(50)
assert criterion.should_stop({}, [])

API

baal.active.stopping_criteria

Bases: StoppingCriterion

Stops when the labelling budget is exhausted.

Source code in baal/active/stopping_criteria.py 
16
17
18
19
20
21
22
23
24
25
class LabellingBudgetStoppingCriterion(StoppingCriterion):
    """Stops when the labelling budget is exhausted."""

    def __init__(self, active_dataset: ActiveLearningDataset, labelling_budget: int):
        super().__init__(active_dataset)
        self._start_length = len(active_dataset)
        self.labelling_budget = labelling_budget

    def should_stop(self, metrics: Dict[str, float], uncertainty: Iterable[float]) -> bool:
        return (len(self._active_ds) - self._start_length) >= self.labelling_budget

Bases: StoppingCriterion

Stops when the average uncertainty is on average below a threshold.

Source code in baal/active/stopping_criteria.py 
28
29
30
31
32
33
34
35
36
37
class LowAverageUncertaintyStoppingCriterion(StoppingCriterion):
    """Stops when the average uncertainty is on average below a threshold."""

    def __init__(self, active_dataset: ActiveLearningDataset, avg_uncertainty_thresh: float):
        super().__init__(active_dataset)
        self.avg_uncertainty_thresh = avg_uncertainty_thresh

    def should_stop(self, metrics: Dict[str, float], uncertainty: Iterable[float]) -> bool:
        arr = np.array(uncertainty)
        return bool(np.mean(arr) < self.avg_uncertainty_thresh)

Bases: StoppingCriterion

Early stopping on a particular metrics.

Notes: We don't have any mandatory dependency with an early stopping implementation. So we have our own.

Source code in baal/active/stopping_criteria.py 
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
class EarlyStoppingCriterion(StoppingCriterion):
    """Early stopping on a particular metrics.

    Notes:
    We don't have any mandatory dependency with an early stopping implementation.
    So we have our own.
    """

    def __init__(
        self,
        active_dataset: ActiveLearningDataset,
        metric_name: str,
        patience: int = 10,
        epsilon: float = 1e-4,
    ):
        super().__init__(active_dataset)
        self.metric_name = metric_name
        self.patience = patience
        self.epsilon = epsilon
        self._acc: List[float] = []

    def should_stop(self, metrics: Dict[str, float], uncertainty: Iterable[float]) -> bool:
        self._acc.append(metrics[self.metric_name])
        near_threshold = np.isclose(np.array(self._acc), self._acc[-1], atol=self.epsilon)
        return len(near_threshold) >= self.patience and bool(
            near_threshold[-(self.patience + 1) :].all()
        )