Skip to content

Inference#

For matters of inference, thunder offers Predictors - objects that can transform data before and after model's inference.

In ThunderModule Predictor is used as default inference runner.

Predictors#

thunder.predict.predict #

BasePredictor #

Bases: ABC

Base class for all predictors.

Source code in thunder/predict/predict.py 
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
class BasePredictor(ABC):
    """Base class for all predictors."""

    @abstractmethod
    def forward(self, batches: Iterable) -> Iterable:
        """Process stream of batches before model inference."""
        raise NotImplementedError("You must implement forward method")

    @abstractmethod
    def backward(self, predicts: Iterable) -> Iterable:
        """Post-process stream of predictions."""
        raise NotImplementedError("You must implement backward method")

    def __call__(self, batches: Iterable, predict_fn: Callable) -> Iterable:
        return self.run(batches, predict_fn)

    def run(self, batches: Iterable, predict_fn: Callable) -> Iterable:
        """Runs preprocessing, inference and postprocessing."""
        return self.backward(map(predict_fn, self.forward(batches)))

backward(predicts) abstractmethod #

Post-process stream of predictions.

Source code in thunder/predict/predict.py 
15
16
17
18
@abstractmethod
def backward(self, predicts: Iterable) -> Iterable:
    """Post-process stream of predictions."""
    raise NotImplementedError("You must implement backward method")

forward(batches) abstractmethod #

Process stream of batches before model inference.

Source code in thunder/predict/predict.py 
10
11
12
13
@abstractmethod
def forward(self, batches: Iterable) -> Iterable:
    """Process stream of batches before model inference."""
    raise NotImplementedError("You must implement forward method")

run(batches, predict_fn) #

Runs preprocessing, inference and postprocessing.

Source code in thunder/predict/predict.py 
23
24
25
def run(self, batches: Iterable, predict_fn: Callable) -> Iterable:
    """Runs preprocessing, inference and postprocessing."""
    return self.backward(map(predict_fn, self.forward(batches)))

Decorated #

Bases: Predictor

Decorates inference function Example

Decorated(f, g, h)

inside Decorated#

predict_fn = f(g(h(predict_fn)))

Source code in thunder/predict/predict.py 
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
class Decorated(Predictor):
    """
    Decorates inference function
    Example
    -----------
    Decorated(f, g, h)
    # inside Decorated
    predict_fn = f(g(h(predict_fn)))
    """

    def __init__(self, *decorators: Callable):
        self.decorators = compose(*decorators)

    def run(self, batches: Iterable, predict_fn: Callable) -> Iterable:
        return super().run(batches, self.decorators(predict_fn))

InfinitePredictor #

Bases: BasePredictor

Useful for running inference on infinite stream of data.

Source code in thunder/predict/predict.py 
28
29
30
31
32
33
34
35
class InfinitePredictor(BasePredictor):
    """Useful for running inference on infinite stream of data."""

    def forward(self, batches: Iterable) -> Iterable:
        yield from batches

    def backward(self, predicts: Iterable) -> Iterable:
        yield from predicts

Predictor #

Bases: InfinitePredictor

Assumes using finite amount of data for inference to be run on.

Source code in thunder/predict/predict.py 
38
39
40
41
42
class Predictor(InfinitePredictor):
    """Assumes using finite amount of data for inference to be run on."""

    def run(self, batches: Iterable, predict_fn: Callable) -> Iterable:
        return tuple(super().run(batches, predict_fn))