ModelWrapper

ModelWrapper is an object similar to keras.Model that trains, test and predict on datasets.

Using our wrapper makes it easier to do Monte-Carlo sampling with the iterations parameters. Another optimization that we do is that instead of using a for-loop to perform MC sampling, we stack examples.

Example

from baal.modelwrapper import ModelWrapper, TrainingArgs
from baal.active.dataset import ActiveLearningDataset
from torch.utils.data import Dataset

# You define ModelWrapper with a Pytorch model and a criterion.
wrapper = ModelWrapper(model=your_model,
                       args=TrainingArgs(criterion=your_criterion,
                                         optimizer=your_optimizer,
                                         batch_size=32,
                                         epoch=10,
                                         use_cuda=True))

# Assuming you have your ActiveLearningDataset ready,
al_dataset: ActiveLearningDataset = ...
test_dataset: Dataset = ...

train_history = wrapper.train_on_dataset(al_dataset)
# We can also use BMA during test time using `average_predictions`.
test_values = wrapper.test_on_dataset(test_dataset, average_predictions=20)

# We use Monte-Carlo sampling using the `iterations` arguments.
predictions = wrapper.predict_on_dataset(al_dataset.pool, iterations=20)
predictions.shape
# > [len(al_dataset.pool), num_outputs, 20]

API

baal.ModelWrapper

Bases: MetricMixin

Wrapper created to ease the training/testing/loading.

Parameters:

Name	Type	Description	Default
model	Module	The model to optimize.	required
args	TrainingArgs	Model arguments for training/predicting.	required

Source code in baal/modelwrapper.py

class ModelWrapper(MetricMixin):
    """
    Wrapper created to ease the training/testing/loading.

    Args:
        model (nn.Module): The model to optimize.
        args (TrainingArgs): Model arguments for training/predicting.
    """

    def __init__(self, model, args: TrainingArgs):
        self.model = model
        self.args = args
        self.metrics = dict()
        self.active_learning_metrics = defaultdict(dict)
        self.add_metric("loss", lambda: Loss())
        self._active_dataset_size = -1

        raise_warnings_cache_replicated(
            self.model, replicate_in_memory=self.args.replicate_in_memory
        )

    def train_on_dataset(self, dataset):
        """
        Train for `epoch` epochs on a Dataset `dataset.

        Args:
            dataset (Dataset): Pytorch Dataset to be trained on.

        Returns:
            The training history.
        """
        dataset_size = len(dataset)
        self.train()
        self.set_dataset_size(dataset_size)
        history = []
        log.info("Starting training", epoch=self.args.epoch, dataset=dataset_size)
        for _ in range(self.args.epoch):
            self._reset_metrics("train")
            for data, target, *_ in DataLoader(
                dataset,
                self.args.batch_size,
                True,
                num_workers=self.args.workers,
                collate_fn=self.args.collate_fn,
            ):
                _ = self.train_on_batch(data, target)
            history.append(self.get_metrics("train")["train_loss"])

        self.args.optimizer.zero_grad()  # Assert that the gradient is flushed.
        log.info("Training complete", train_loss=self.get_metrics("train")["train_loss"])
        self.active_step(dataset_size, self.get_metrics("train"))
        return history

    def test_on_dataset(
        self,
        dataset: Dataset,
        average_predictions: int = 1,
    ):
        """
        Test the model on a Dataset `dataset`.

        Args:
            dataset (Dataset): Dataset to evaluate on.
            average_predictions (int): The number of predictions to average to
                compute the test loss.

        Returns:
            Average loss value over the dataset.
        """
        self.eval()
        log.info("Starting evaluating", dataset=len(dataset))
        self._reset_metrics("test")

        for data, target, *_ in DataLoader(
            dataset,
            self.args.batch_size,
            False,
            num_workers=self.args.workers,
            collate_fn=self.args.collate_fn,
        ):
            _ = self.test_on_batch(data, target, average_predictions=average_predictions)

        log.info("Evaluation complete", test_loss=self.get_metrics("test")["test_loss"])
        self.active_step(None, self.get_metrics("test"))
        return self.get_metrics("test")["test_loss"]

    def train_and_test_on_datasets(
        self,
        train_dataset: Dataset,
        test_dataset: Dataset,
        return_best_weights=False,
        patience=None,
        min_epoch_for_es=0,
        skip_epochs=1,
    ):
        """
        Train and test the model on both Dataset `train_dataset`, `test_dataset`.

        Args:
            train_dataset (Dataset): Dataset to train on.
            test_dataset (Dataset): Dataset to evaluate on.
            return_best_weights (bool): If True, will keep the best weights and return them.
            patience (Optional[int]): If provided, will use early stopping to stop after
                                        `patience` epoch without improvement.
            min_epoch_for_es (int): Epoch at which the early stopping starts.
            skip_epochs (int): Number of epochs to skip for test_on_dataset

        Returns:
            History and best weights if required.
        """
        best_weight = None
        best_loss = 1e10
        best_epoch = 0
        hist = []
        for e in range(self.args.epoch):
            _ = self.train_on_dataset(
                train_dataset,
            )
            if e % skip_epochs == 0:
                te_loss = self.test_on_dataset(test_dataset)
                hist.append(self.get_metrics())
                if te_loss < best_loss:
                    best_epoch = e
                    best_loss = te_loss
                    if return_best_weights:
                        best_weight = deepcopy(self.state_dict())

                if patience is not None and (e - best_epoch) > patience and (e > min_epoch_for_es):
                    # Early stopping
                    break
            else:
                hist.append(self.get_metrics("train"))

        if return_best_weights:
            return hist, best_weight
        else:
            return hist

    def predict_on_dataset_generator(
        self,
        dataset: Dataset,
        iterations: int,
        half=False,
        verbose=True,
    ):
        """
        Use the model to predict on a dataset `iterations` time.

        Args:
            dataset (Dataset): Dataset to predict on.
            iterations (int): Number of iterations per sample.
            half (bool): If True use half precision.
            verbose (bool): If True use tqdm to display progress

        Notes:
            The "batch" is made of `batch_size` * `iterations` samples.

        Returns:
            Generators [batch_size, n_classes, ..., n_iterations].
        """
        self.eval()
        if len(dataset) == 0:
            return None

        log.info("Start Predict", dataset=len(dataset))
        loader = DataLoader(
            dataset,
            self.args.batch_size,
            False,
            num_workers=self.args.workers,
            collate_fn=self.args.collate_fn,
        )
        if verbose:
            loader = tqdm(loader, total=len(loader), file=sys.stdout)
        for idx, (data, *_) in enumerate(loader):

            pred = self.predict_on_batch(data, iterations)
            pred = map_on_tensor(lambda x: x.detach(), pred)
            if half:
                pred = map_on_tensor(lambda x: x.half(), pred)
            yield map_on_tensor(lambda x: x.cpu().numpy(), pred)

    def predict_on_dataset(
        self,
        dataset: Dataset,
        iterations: int,
        half=False,
        verbose=True,
    ) -> Union[NDArray, List[NDArray]]:
        """
        Use the model to predict on a dataset `iterations` time.

        Args:
            dataset (Dataset): Dataset to predict on.
            iterations (int): Number of iterations per sample.
            half (bool): If True use half precision.
            verbose (bool): If True use tqdm to show progress.

        Notes:
            The "batch" is made of `batch_size` * `iterations` samples.

        Returns:
            Array [n_samples, n_outputs, ..., n_iterations].
        """
        preds = list(
            self.predict_on_dataset_generator(
                dataset=dataset,
                iterations=iterations,
                half=half,
                verbose=verbose,
            )
        )

        if len(preds) > 0 and not isinstance(preds[0], Sequence):
            # Is an Array or a Tensor
            return np.vstack(preds)
        return [np.vstack(pr) for pr in zip(*preds)]

    def train_on_batch(self, data, target):
        """
        Train the current model on a batch using `optimizer`.

        Args:
            data (Tensor): The model input.
            target (Tensor): The ground truth.

        Returns:
            Tensor, the loss computed from the criterion.
        """

        if self.args.use_cuda:
            data, target = to_cuda(data), to_cuda(target)
        self.args.optimizer.zero_grad()
        output = self.model(data)
        loss = self.args.criterion(output, target)

        if self.args.regularizer:
            regularized_loss = loss + self.args.regularizer()
            regularized_loss.backward()
        else:
            loss.backward()

        self.args.optimizer.step()
        self._update_metrics(output, target, loss, filter="train")
        return loss

    def test_on_batch(
        self,
        data: torch.Tensor,
        target: torch.Tensor,
        average_predictions: int = 1,
    ):
        """
        Test the current model on a batch.

        Args:
            data (Tensor): The model input.
            target (Tensor): The ground truth.
            average_predictions (int): The number of predictions to average to
                compute the test loss.

        Returns:
            Tensor, the loss computed from the criterion.
        """
        with torch.no_grad():
            if self.args.use_cuda:
                data, target = to_cuda(data), to_cuda(target)

            preds = map_on_tensor(
                lambda p: p.mean(-1),
                self.predict_on_batch(data, iterations=average_predictions),
            )
            loss = assert_not_none(self.args.criterion)(preds, target)
            self._update_metrics(preds, target, loss, "test")
            return loss

    def predict_on_batch(self, data, iterations=1):
        """
        Get the model's prediction on a batch.

        Args:
            data (Tensor): The model input.
            iterations (int): Number of prediction to perform.

        Returns:
            Tensor, the loss computed from the criterion.
                    shape = {batch_size, nclass, n_iteration}.

        Raises:
            Raises RuntimeError if CUDA rans out of memory during data replication.
        """
        with torch.no_grad():
            if self.args.use_cuda:
                data = to_cuda(data)
            if self.args.replicate_in_memory:
                data = map_on_tensor(lambda d: stack_in_memory(d, iterations), data)
                try:
                    out = self.model(data)
                except RuntimeError as e:
                    raise RuntimeError(
                        """CUDA ran out of memory while BaaL tried to replicate data. See the exception above.
                    Use `replicate_in_memory=False` in order to reduce the memory requirements.
                    Note that there will be some speed trade-offs"""
                    ) from e
                out = map_on_tensor(lambda o: o.view([iterations, -1, *o.size()[1:]]), out)
                out = map_on_tensor(lambda o: o.permute(1, 2, *range(3, o.ndimension()), 0), out)
            else:
                out = [self.model(data) for _ in range(iterations)]
                out = _stack_preds(out)
            return out

    def get_params(self):
        """
        Return the parameters to optimize.

        Returns:
            Config for parameters.
        """
        return self.model.parameters()

    def state_dict(self):
        """Get the state dict(s)."""
        return self.model.state_dict()

    def load_state_dict(self, state_dict, strict=True):
        """Load the model with `state_dict`."""
        self.model.load_state_dict(state_dict, strict=strict)

    def train(self):
        """Set the model in `train` mode."""
        self.model.train()

    def eval(self):
        """Set the model in `eval mode`."""
        self.model.eval()

    def reset_fcs(self):
        """Reset all torch.nn.Linear layers."""

        def reset(m):
            if isinstance(m, torch.nn.Linear):
                m.reset_parameters()

        self.model.apply(reset)

    def reset_all(self):
        """Reset all *resetable* layers."""

        def reset(m):
            for m in self.model.modules():
                getattr(m, "reset_parameters", lambda: None)()

        self.model.apply(reset)

    def set_dataset_size(self, dataset_size: int):
        """
        Set state for dataset size. Useful for tracking.

        Args:
            dataset_size: Dataset state
        """
        self._active_dataset_size = dataset_size

eval()

Set the model in eval mode.

Source code in baal/modelwrapper.py

def eval(self):
    """Set the model in `eval mode`."""
    self.model.eval()

get_params()

Return the parameters to optimize.

Returns:

Type	Description
	Config for parameters.

Source code in baal/modelwrapper.py

def get_params(self):
    """
    Return the parameters to optimize.

    Returns:
        Config for parameters.
    """
    return self.model.parameters()

load_state_dict(state_dict, strict=True)

Load the model with state_dict.

Source code in baal/modelwrapper.py

def load_state_dict(self, state_dict, strict=True):
    """Load the model with `state_dict`."""
    self.model.load_state_dict(state_dict, strict=strict)

predict_on_batch(data, iterations=1)

Get the model's prediction on a batch.

Parameters:

Name	Type	Description	Default
data	Tensor	The model input.	required
iterations	int	Number of prediction to perform.	1

Returns:

Type	Description
	Tensor, the loss computed from the criterion. shape = {batch_size, nclass, n_iteration}.

Source code in baal/modelwrapper.py

def predict_on_batch(self, data, iterations=1):
    """
    Get the model's prediction on a batch.

    Args:
        data (Tensor): The model input.
        iterations (int): Number of prediction to perform.

    Returns:
        Tensor, the loss computed from the criterion.
                shape = {batch_size, nclass, n_iteration}.

    Raises:
        Raises RuntimeError if CUDA rans out of memory during data replication.
    """
    with torch.no_grad():
        if self.args.use_cuda:
            data = to_cuda(data)
        if self.args.replicate_in_memory:
            data = map_on_tensor(lambda d: stack_in_memory(d, iterations), data)
            try:
                out = self.model(data)
            except RuntimeError as e:
                raise RuntimeError(
                    """CUDA ran out of memory while BaaL tried to replicate data. See the exception above.
                Use `replicate_in_memory=False` in order to reduce the memory requirements.
                Note that there will be some speed trade-offs"""
                ) from e
            out = map_on_tensor(lambda o: o.view([iterations, -1, *o.size()[1:]]), out)
            out = map_on_tensor(lambda o: o.permute(1, 2, *range(3, o.ndimension()), 0), out)
        else:
            out = [self.model(data) for _ in range(iterations)]
            out = _stack_preds(out)
        return out

predict_on_dataset(dataset, iterations, half=False, verbose=True)

Use the model to predict on a dataset iterations time.

Parameters:

Name	Type	Description	Default
dataset	Dataset	Dataset to predict on.	required
iterations	int	Number of iterations per sample.	required
half	bool	If True use half precision.	False
verbose	bool	If True use tqdm to show progress.	True

Notes

The "batch" is made of batch_size * iterations samples.

Returns:

Type	Description
Union[NDArray, List[NDArray]]	Array [n_samples, n_outputs, ..., n_iterations].

Source code in baal/modelwrapper.py

def predict_on_dataset(
    self,
    dataset: Dataset,
    iterations: int,
    half=False,
    verbose=True,
) -> Union[NDArray, List[NDArray]]:
    """
    Use the model to predict on a dataset `iterations` time.

    Args:
        dataset (Dataset): Dataset to predict on.
        iterations (int): Number of iterations per sample.
        half (bool): If True use half precision.
        verbose (bool): If True use tqdm to show progress.

    Notes:
        The "batch" is made of `batch_size` * `iterations` samples.

    Returns:
        Array [n_samples, n_outputs, ..., n_iterations].
    """
    preds = list(
        self.predict_on_dataset_generator(
            dataset=dataset,
            iterations=iterations,
            half=half,
            verbose=verbose,
        )
    )

    if len(preds) > 0 and not isinstance(preds[0], Sequence):
        # Is an Array or a Tensor
        return np.vstack(preds)
    return [np.vstack(pr) for pr in zip(*preds)]

predict_on_dataset_generator(dataset, iterations, half=False, verbose=True)

Use the model to predict on a dataset iterations time.

Parameters:

Name	Type	Description	Default
dataset	Dataset	Dataset to predict on.	required
iterations	int	Number of iterations per sample.	required
half	bool	If True use half precision.	False
verbose	bool	If True use tqdm to display progress	True

Notes

The "batch" is made of batch_size * iterations samples.

Returns:

Type	Description
	Generators [batch_size, n_classes, ..., n_iterations].

Source code in baal/modelwrapper.py

def predict_on_dataset_generator(
    self,
    dataset: Dataset,
    iterations: int,
    half=False,
    verbose=True,
):
    """
    Use the model to predict on a dataset `iterations` time.

    Args:
        dataset (Dataset): Dataset to predict on.
        iterations (int): Number of iterations per sample.
        half (bool): If True use half precision.
        verbose (bool): If True use tqdm to display progress

    Notes:
        The "batch" is made of `batch_size` * `iterations` samples.

    Returns:
        Generators [batch_size, n_classes, ..., n_iterations].
    """
    self.eval()
    if len(dataset) == 0:
        return None

    log.info("Start Predict", dataset=len(dataset))
    loader = DataLoader(
        dataset,
        self.args.batch_size,
        False,
        num_workers=self.args.workers,
        collate_fn=self.args.collate_fn,
    )
    if verbose:
        loader = tqdm(loader, total=len(loader), file=sys.stdout)
    for idx, (data, *_) in enumerate(loader):

        pred = self.predict_on_batch(data, iterations)
        pred = map_on_tensor(lambda x: x.detach(), pred)
        if half:
            pred = map_on_tensor(lambda x: x.half(), pred)
        yield map_on_tensor(lambda x: x.cpu().numpy(), pred)

reset_all()

Reset all resetable layers.

Source code in baal/modelwrapper.py

def reset_all(self):
    """Reset all *resetable* layers."""

    def reset(m):
        for m in self.model.modules():
            getattr(m, "reset_parameters", lambda: None)()

    self.model.apply(reset)

reset_fcs()

Reset all torch.nn.Linear layers.

Source code in baal/modelwrapper.py

def reset_fcs(self):
    """Reset all torch.nn.Linear layers."""

    def reset(m):
        if isinstance(m, torch.nn.Linear):
            m.reset_parameters()

    self.model.apply(reset)

set_dataset_size(dataset_size)

Set state for dataset size. Useful for tracking.

Parameters:

Name	Type	Description	Default
dataset_size	int	Dataset state	required

Source code in baal/modelwrapper.py

def set_dataset_size(self, dataset_size: int):
    """
    Set state for dataset size. Useful for tracking.

    Args:
        dataset_size: Dataset state
    """
    self._active_dataset_size = dataset_size

state_dict()

Get the state dict(s).

Source code in baal/modelwrapper.py

def state_dict(self):
    """Get the state dict(s)."""
    return self.model.state_dict()

test_on_batch(data, target, average_predictions=1)

Test the current model on a batch.

Parameters:

Name	Type	Description	Default
data	Tensor	The model input.	required
target	Tensor	The ground truth.	required
average_predictions	int	The number of predictions to average to compute the test loss.	1

Returns:

Type	Description
	Tensor, the loss computed from the criterion.

Source code in baal/modelwrapper.py

def test_on_batch(
    self,
    data: torch.Tensor,
    target: torch.Tensor,
    average_predictions: int = 1,
):
    """
    Test the current model on a batch.

    Args:
        data (Tensor): The model input.
        target (Tensor): The ground truth.
        average_predictions (int): The number of predictions to average to
            compute the test loss.

    Returns:
        Tensor, the loss computed from the criterion.
    """
    with torch.no_grad():
        if self.args.use_cuda:
            data, target = to_cuda(data), to_cuda(target)

        preds = map_on_tensor(
            lambda p: p.mean(-1),
            self.predict_on_batch(data, iterations=average_predictions),
        )
        loss = assert_not_none(self.args.criterion)(preds, target)
        self._update_metrics(preds, target, loss, "test")
        return loss

test_on_dataset(dataset, average_predictions=1)

Test the model on a Dataset dataset.

Parameters:

Name	Type	Description	Default
dataset	Dataset	Dataset to evaluate on.	required
average_predictions	int	The number of predictions to average to compute the test loss.	1

Returns:

Type	Description
	Average loss value over the dataset.

Source code in baal/modelwrapper.py

def test_on_dataset(
    self,
    dataset: Dataset,
    average_predictions: int = 1,
):
    """
    Test the model on a Dataset `dataset`.

    Args:
        dataset (Dataset): Dataset to evaluate on.
        average_predictions (int): The number of predictions to average to
            compute the test loss.

    Returns:
        Average loss value over the dataset.
    """
    self.eval()
    log.info("Starting evaluating", dataset=len(dataset))
    self._reset_metrics("test")

    for data, target, *_ in DataLoader(
        dataset,
        self.args.batch_size,
        False,
        num_workers=self.args.workers,
        collate_fn=self.args.collate_fn,
    ):
        _ = self.test_on_batch(data, target, average_predictions=average_predictions)

    log.info("Evaluation complete", test_loss=self.get_metrics("test")["test_loss"])
    self.active_step(None, self.get_metrics("test"))
    return self.get_metrics("test")["test_loss"]

train()

Set the model in train mode.

Source code in baal/modelwrapper.py

def train(self):
    """Set the model in `train` mode."""
    self.model.train()

train_and_test_on_datasets(train_dataset, test_dataset, return_best_weights=False, patience=None, min_epoch_for_es=0, skip_epochs=1)

Train and test the model on both Dataset train_dataset, test_dataset.

Parameters:

Name	Type	Description	Default
train_dataset	Dataset	Dataset to train on.	required
test_dataset	Dataset	Dataset to evaluate on.	required
return_best_weights	bool	If True, will keep the best weights and return them.	False
patience	Optional[int]	If provided, will use early stopping to stop after patience epoch without improvement.	None
min_epoch_for_es	int	Epoch at which the early stopping starts.	0
skip_epochs	int	Number of epochs to skip for test_on_dataset	1

Returns:

Type	Description
	History and best weights if required.

Source code in baal/modelwrapper.py

def train_and_test_on_datasets(
    self,
    train_dataset: Dataset,
    test_dataset: Dataset,
    return_best_weights=False,
    patience=None,
    min_epoch_for_es=0,
    skip_epochs=1,
):
    """
    Train and test the model on both Dataset `train_dataset`, `test_dataset`.

    Args:
        train_dataset (Dataset): Dataset to train on.
        test_dataset (Dataset): Dataset to evaluate on.
        return_best_weights (bool): If True, will keep the best weights and return them.
        patience (Optional[int]): If provided, will use early stopping to stop after
                                    `patience` epoch without improvement.
        min_epoch_for_es (int): Epoch at which the early stopping starts.
        skip_epochs (int): Number of epochs to skip for test_on_dataset

    Returns:
        History and best weights if required.
    """
    best_weight = None
    best_loss = 1e10
    best_epoch = 0
    hist = []
    for e in range(self.args.epoch):
        _ = self.train_on_dataset(
            train_dataset,
        )
        if e % skip_epochs == 0:
            te_loss = self.test_on_dataset(test_dataset)
            hist.append(self.get_metrics())
            if te_loss < best_loss:
                best_epoch = e
                best_loss = te_loss
                if return_best_weights:
                    best_weight = deepcopy(self.state_dict())

            if patience is not None and (e - best_epoch) > patience and (e > min_epoch_for_es):
                # Early stopping
                break
        else:
            hist.append(self.get_metrics("train"))

    if return_best_weights:
        return hist, best_weight
    else:
        return hist

train_on_batch(data, target)

Train the current model on a batch using optimizer.

Parameters:

Name	Type	Description	Default
data	Tensor	The model input.	required
target	Tensor	The ground truth.	required

Returns:

Type	Description
	Tensor, the loss computed from the criterion.

Source code in baal/modelwrapper.py

def train_on_batch(self, data, target):
    """
    Train the current model on a batch using `optimizer`.

    Args:
        data (Tensor): The model input.
        target (Tensor): The ground truth.

    Returns:
        Tensor, the loss computed from the criterion.
    """

    if self.args.use_cuda:
        data, target = to_cuda(data), to_cuda(target)
    self.args.optimizer.zero_grad()
    output = self.model(data)
    loss = self.args.criterion(output, target)

    if self.args.regularizer:
        regularized_loss = loss + self.args.regularizer()
        regularized_loss.backward()
    else:
        loss.backward()

    self.args.optimizer.step()
    self._update_metrics(output, target, loss, filter="train")
    return loss

train_on_dataset(dataset)

Train for epoch epochs on a Dataset `dataset.

Parameters:

Name	Type	Description	Default
dataset	Dataset	Pytorch Dataset to be trained on.	required

Returns:

Type	Description
	The training history.

Source code in baal/modelwrapper.py

def train_on_dataset(self, dataset):
    """
    Train for `epoch` epochs on a Dataset `dataset.

    Args:
        dataset (Dataset): Pytorch Dataset to be trained on.

    Returns:
        The training history.
    """
    dataset_size = len(dataset)
    self.train()
    self.set_dataset_size(dataset_size)
    history = []
    log.info("Starting training", epoch=self.args.epoch, dataset=dataset_size)
    for _ in range(self.args.epoch):
        self._reset_metrics("train")
        for data, target, *_ in DataLoader(
            dataset,
            self.args.batch_size,
            True,
            num_workers=self.args.workers,
            collate_fn=self.args.collate_fn,
        ):
            _ = self.train_on_batch(data, target)
        history.append(self.get_metrics("train")["train_loss"])

    self.args.optimizer.zero_grad()  # Assert that the gradient is flushed.
    log.info("Training complete", train_loss=self.get_metrics("train")["train_loss"])
    self.active_step(dataset_size, self.get_metrics("train"))
    return history