from __future__ import division
import warnings
import numpy as np
from menpo.feature import no_op
from menpo.visualize import print_dynamic
from menpo.model import PCAModel
from menpo.transform import Scale
from menpo.shape import mean_pointcloud
from menpo.base import name_of_callable
from menpofit import checks
from menpofit.aam.algorithm.lk import (LucasKanadeStandardInterface,
LucasKanadePatchInterface,
LucasKanadeLinearInterface)
from menpofit.modelinstance import OrthoPDM
from menpofit.transform import (DifferentiableThinPlateSplines,
DifferentiablePiecewiseAffine, OrthoMDTransform,
LinearOrthoMDTransform)
from menpofit.base import batch
from menpofit.builder import (
build_reference_frame, build_patch_reference_frame,
compute_features, scale_images, warp_images,
align_shapes, rescale_images_to_reference_shape, densify_shapes,
extract_patches, MenpoFitBuilderWarning, compute_reference_shape)
[docs]class AAM(object):
r"""
Class for training a multi-scale holistic Active Appearance Model. Please
see the references for a basic list of relevant papers.
Parameters
----------
images : `list` of `menpo.image.Image`
The `list` of training images.
group : `str` or ``None``, optional
The landmark group that will be used to train the AAM. If ``None`` and
the images only have a single landmark group, then that is the one
that will be used. Note that all the training images need to have the
specified landmark group.
holistic_features : `closure` or `list` of `closure`, optional
The features that will be extracted from the training images. Note
that the features are extracted before warping the images to the
reference shape. If `list`, then it must define a feature function per
scale. Please refer to `menpo.feature` for a list of potential features.
reference_shape : `menpo.shape.PointCloud` or ``None``, optional
The reference shape that will be used for building the AAM. The purpose
of the reference shape is to normalise the size of the training images.
The normalization is performed by rescaling all the training images
so that the scale of their ground truth shapes matches the scale of
the reference shape. Note that the reference shape is rescaled with
respect to the `diagonal` before performing the normalisation. If
``None``, then the mean shape will be used.
diagonal : `int` or ``None``, optional
This parameter is used to rescale the reference shape so that the
diagonal of its bounding box matches the provided value. In other
words, this parameter controls the size of the model at the highest
scale. If ``None``, then the reference shape does not get rescaled.
scales : `float` or `tuple` of `float`, optional
The scale value of each scale. They must provided in ascending order,
i.e. from lowest to highest scale. If `float`, then a single scale is
assumed.
transform : `subclass` of :map:`DL` and :map:`DX`, optional
A differential warp transform object, e.g.
:map:`DifferentiablePiecewiseAffine` or
:map:`DifferentiableThinPlateSplines`.
shape_model_cls : `subclass` of :map:`PDM`, optional
The class to be used for building the shape model. The most common
choice is :map:`OrthoPDM`.
max_shape_components : `int`, `float`, `list` of those or ``None``, optional
The number of shape components to keep. If `int`, then it sets the exact
number of components. If `float`, then it defines the variance
percentage that will be kept. If `list`, then it should
define a value per scale. If a single number, then this will be
applied to all scales. If ``None``, then all the components are kept.
Note that the unused components will be permanently trimmed.
max_appearance_components : `int`, `float`, `list` of those or ``None``, optional
The number of appearance components to keep. If `int`, then it sets the
exact number of components. If `float`, then it defines the variance
percentage that will be kept. If `list`, then it should define a value
per scale. If a single number, then this will be applied to all
scales. If ``None``, then all the components are kept. Note that the
unused components will be permanently trimmed.
verbose : `bool`, optional
If ``True``, then the progress of building the AAM will be printed.
batch_size : `int` or ``None``, optional
If an `int` is provided, then the training is performed in an
incremental fashion on image batches of size equal to the provided
value. If ``None``, then the training is performed directly on the
all the images.
References
----------
.. [1] J. Alabort-i-Medina, and S. Zafeiriou. "A Unified Framework for
Compositional Fitting of Active Appearance Models", arXiv:1601.00199.
.. [2] T.F. Cootes, G.J. Edwards, and C.J. Taylor. "Active Appearance
Models", IEEE Transactions on Pattern Analysis & Machine Intelligence
6 (2001): 681-685.
.. [3] I. Matthews, and S. Baker. "Active Appearance Models Revisited",
International Journal of Computer Vision, 60(2): 135-164, 2004.
.. [4] G. Papandreou, and P. Maragos. "Adaptive and constrained algorithms
for inverse compositional Active Appearance Model fitting", IEEE
Proceedings of International Conference on Computer Vision and
Pattern Recognition (CVPR), pp. 1-8, June 2008.
.. [5] E. Antonakos, J. Alabort-i-Medina, G. Tzimiropoulos, and S.
Zafeiriou. "Feature-Based Lucas-Kanade and Active Appearance Models",
IEEE Transactions on Image Processing, 24(9): 2617-2632, 2015.
"""
def __init__(self, images, group=None, holistic_features=no_op,
reference_shape=None, diagonal=None, scales=(0.5, 1.0),
transform=DifferentiablePiecewiseAffine,
shape_model_cls=OrthoPDM, max_shape_components=None,
max_appearance_components=None, verbose=False,
batch_size=None):
# Check parameters
checks.check_diagonal(diagonal)
scales = checks.check_scales(scales)
n_scales = len(scales)
holistic_features = checks.check_callable(holistic_features, n_scales)
shape_model_cls = checks.check_callable(shape_model_cls, n_scales)
max_shape_components = checks.check_max_components(
max_shape_components, n_scales, 'max_shape_components')
max_appearance_components = checks.check_max_components(
max_appearance_components, n_scales, 'max_appearance_components')
# Assign attributes
self.holistic_features = holistic_features
self.transform = transform
self.diagonal = diagonal
self.scales = scales
self.max_shape_components = max_shape_components
self.max_appearance_components = max_appearance_components
self.reference_shape = reference_shape
self._shape_model_cls = shape_model_cls
self.shape_models = []
self.appearance_models = []
# Train AAM
self._train(images, increment=False, group=group, verbose=verbose,
batch_size=batch_size)
def _train(self, images, increment=False, group=None,
shape_forgetting_factor=1.0, appearance_forgetting_factor=1.0,
verbose=False, batch_size=None):
# If batch_size is not None, then we may have a generator, else we
# assume we have a list.
if batch_size is not None:
# Create a generator of fixed sized batches. Will still work even
# on an infinite list.
image_batches = batch(images, batch_size)
else:
image_batches = [list(images)]
for k, image_batch in enumerate(image_batches):
if k == 0:
if self.reference_shape is None:
# If no reference shape was given, use the mean of the first
# batch
if batch_size is not None:
warnings.warn('No reference shape was provided. The '
'mean of the first batch will be the '
'reference shape. If the batch mean is '
'not representative of the true mean, '
'this may cause issues.',
MenpoFitBuilderWarning)
checks.check_landmark_trilist(image_batch[0],
self.transform, group=group)
self.reference_shape = compute_reference_shape(
[i.landmarks[group] for i in image_batch],
self.diagonal, verbose=verbose)
# After the first batch, we are incrementing the model
if k > 0:
increment = True
if verbose:
print('Computing batch {}'.format(k))
# Train each batch
self._train_batch(
image_batch, increment=increment, group=group,
shape_forgetting_factor=shape_forgetting_factor,
appearance_forgetting_factor=appearance_forgetting_factor,
verbose=verbose)
def _train_batch(self, image_batch, increment=False, group=None,
verbose=False, shape_forgetting_factor=1.0,
appearance_forgetting_factor=1.0):
# Rescale to existing reference shape
image_batch = rescale_images_to_reference_shape(
image_batch, group, self.reference_shape, verbose=verbose)
# Build models at each scale
if verbose:
print_dynamic('- Building models\n')
feature_images = []
# for each scale (low --> high)
for j in range(self.n_scales):
if verbose:
if len(self.scales) > 1:
scale_prefix = ' - Scale {}: '.format(j)
else:
scale_prefix = ' - '
else:
scale_prefix = None
# Handle holistic features
if j == 0 and self.holistic_features[j] == no_op:
# Saves a lot of memory
feature_images = image_batch
elif j == 0 or self.holistic_features[j] is not self.holistic_features[j - 1]:
# Compute features only if this is the first pass through
# the loop or the features at this scale are different from
# the features at the previous scale
feature_images = compute_features(image_batch,
self.holistic_features[j],
prefix=scale_prefix,
verbose=verbose)
# handle scales
if self.scales[j] != 1:
# Scale feature images only if scale is different than 1
scaled_images = scale_images(feature_images, self.scales[j],
prefix=scale_prefix,
verbose=verbose)
else:
scaled_images = feature_images
# Extract potentially rescaled shapes
scale_shapes = [i.landmarks[group] for i in scaled_images]
# Build the shape model
if verbose:
print_dynamic('{}Building shape model'.format(scale_prefix))
if not increment:
shape_model = self._build_shape_model(scale_shapes, j)
self.shape_models.append(shape_model)
else:
self._increment_shape_model(
scale_shapes, j, forgetting_factor=shape_forgetting_factor)
# Obtain warped images - we use a scaled version of the
# reference shape, computed here. This is because the mean
# moves when we are incrementing, and we need a consistent
# reference frame.
scaled_reference_shape = Scale(self.scales[j], n_dims=2).apply(
self.reference_shape)
warped_images = self._warp_images(scaled_images, scale_shapes,
scaled_reference_shape,
j, scale_prefix, verbose)
# obtain appearance model
if verbose:
print_dynamic('{}Building appearance model'.format(
scale_prefix))
if not increment:
appearance_model = PCAModel(warped_images)
# trim appearance model if required
if self.max_appearance_components[j] is not None:
appearance_model.trim_components(
self.max_appearance_components[j])
# add appearance model to the list
self.appearance_models.append(appearance_model)
else:
# increment appearance model
self.appearance_models[j].increment(
warped_images,
forgetting_factor=appearance_forgetting_factor)
# trim appearance model if required
if self.max_appearance_components[j] is not None:
self.appearance_models[j].trim_components(
self.max_appearance_components[j])
if verbose:
print_dynamic('{}Done\n'.format(scale_prefix))
[docs] def increment(self, images, group=None, shape_forgetting_factor=1.0,
appearance_forgetting_factor=1.0, verbose=False,
batch_size=None):
r"""
Method to increment the trained AAM with a new set of training images.
Parameters
----------
images : `list` of `menpo.image.Image`
The `list` of training images.
group : `str` or ``None``, optional
The landmark group that will be used to train the AAM. If ``None``
and the images only have a single landmark group, then that is the
one that will be used. Note that all the training images need to
have the specified landmark group.
shape_forgetting_factor : ``[0.0, 1.0]`` `float`, optional
Forgetting factor that weights the relative contribution of new
samples vs old samples for the shape model. If ``1.0``, all samples
are weighted equally and, hence, the result is the exact same as
performing batch PCA on the concatenated list of old and new
simples. If ``<1.0``, more emphasis is put on the new samples.
appearance_forgetting_factor : ``[0.0, 1.0]`` `float`, optional
Forgetting factor that weights the relative contribution of new
samples vs old samples for the appearance model. If ``1.0``,
all samples are weighted equally and, hence, the result is the
exact same as performing batch PCA on the concatenated list of
old and new simples. If ``<1.0``, more emphasis is put on the new
samples.
verbose : `bool`, optional
If ``True``, then the progress of building the AAM will be printed.
batch_size : `int` or ``None``, optional
If an `int` is provided, then the training is performed in an
incremental fashion on image batches of size equal to the provided
value. If ``None``, then the training is performed directly on the
all the images.
"""
return self._train(
images, increment=True, group=group, verbose=verbose,
shape_forgetting_factor=shape_forgetting_factor,
appearance_forgetting_factor=appearance_forgetting_factor,
batch_size=batch_size)
def _build_shape_model(self, shapes, scale_index):
return self._shape_model_cls[scale_index](
shapes, max_n_components=self.max_shape_components[scale_index])
def _increment_shape_model(self, shapes, scale_index,
forgetting_factor=None):
self.shape_models[scale_index].increment(
shapes, forgetting_factor=forgetting_factor,
max_n_components=self.max_shape_components[scale_index])
def _warp_images(self, images, shapes, reference_shape, scale_index,
prefix, verbose):
reference_frame = build_reference_frame(reference_shape)
return warp_images(images, shapes, reference_frame, self.transform,
prefix=prefix, verbose=verbose)
@property
def n_scales(self):
"""
Returns the number of scales.
:type: `int`
"""
return len(self.scales)
@property
def _str_title(self):
return 'Holistic Active Appearance Model'
[docs] def instance(self, shape_weights=None, appearance_weights=None,
scale_index=-1):
r"""
Generates a novel AAM instance given a set of shape and appearance
weights. If no weights are provided, then the mean AAM instance is
returned.
Parameters
----------
shape_weights : ``(n_weights,)`` `ndarray` or `list` or ``None``, optional
The weights of the shape model that will be used to create a novel
shape instance. If ``None``, the weights are assumed to be zero,
thus the mean shape is used.
appearance_weights : ``(n_weights,)`` `ndarray` or `list` or ``None``, optional
The weights of the appearance model that will be used to create a
novel appearance instance. If ``None``, the weights are assumed
to be zero, thus the mean appearance is used.
scale_index : `int`, optional
The scale to be used.
Returns
-------
image : `menpo.image.Image`
The AAM instance.
"""
if shape_weights is None:
shape_weights = [0]
if appearance_weights is None:
appearance_weights = [0]
sm = self.shape_models[scale_index].model
am = self.appearance_models[scale_index]
shape_instance = sm.instance(shape_weights, normalized_weights=True)
appearance_instance = am.instance(appearance_weights,
normalized_weights=True)
return self._instance(scale_index, shape_instance, appearance_instance)
[docs] def random_instance(self, scale_index=-1):
r"""
Generates a random instance of the AAM.
Parameters
----------
scale_index : `int`, optional
The scale to be used.
Returns
-------
image : `menpo.image.Image`
The AAM instance.
"""
sm = self.shape_models[scale_index].model
am = self.appearance_models[scale_index]
# TODO: this bit of logic should to be transferred down to PCAModel
shape_weights = np.random.randn(sm.n_active_components)
shape_instance = sm.instance(shape_weights, normalized_weights=True)
appearance_weights = np.random.randn(sm.n_active_components)
appearance_instance = am.instance(appearance_weights,
normalized_weights=True)
return self._instance(scale_index, shape_instance, appearance_instance)
def _instance(self, scale_index, shape_instance, appearance_instance):
template = self.appearance_models[scale_index].mean()
landmarks = template.landmarks['source']
reference_frame = build_reference_frame(shape_instance)
transform = self.transform(
reference_frame.landmarks['source'], landmarks)
return appearance_instance.as_unmasked(copy=False).warp_to_mask(
reference_frame.mask, transform, warp_landmarks=True)
[docs] def build_fitter_interfaces(self, sampling):
r"""
Method that builds the correct Lucas-Kanade fitting interface. It
only applies in case you wish to fit the AAM with a Lucas-Kanade
algorithm (i.e. :map:`LucasKanadeAAMFitter`).
Parameters
----------
sampling : `list` of `int` or `ndarray` or ``None``
It defines a sampling mask per scale. If `int`, then it
defines the sub-sampling step of the sampling mask. If `ndarray`,
then it explicitly defines the sampling mask. If ``None``, then no
sub-sampling is applied.
Returns
-------
fitter_interfaces : `list`
The `list` of Lucas-Kanade interface per scale.
"""
interfaces = []
for am, sm, s in zip(self.appearance_models, self.shape_models,
sampling):
template = am.mean()
md_transform = OrthoMDTransform(
sm, self.transform,
source=template.landmarks['source'])
interface = LucasKanadeStandardInterface(
am, md_transform, template, sampling=s)
interfaces.append(interface)
return interfaces
[docs] def appearance_reconstructions(self, appearance_parameters,
n_iters_per_scale):
r"""
Method that generates the appearance reconstructions given a set of
appearance parameters. This is to be combined with a :map:`AAMResult`
object, in order to generate the appearance reconstructions of a
fitting procedure.
Parameters
----------
appearance_parameters : `list` of ``(n_params,)`` `ndarray`
A set of appearance parameters per fitting iteration. It can be
retrieved as a property of an :map:`AAMResult` object.
n_iters_per_scale : `list` of `int`
The number of iterations per scale. This is necessary in order to
figure out which appearance parameters correspond to the model of
each scale. It can be retrieved as a property of a :map:`AAMResult`
object.
Returns
-------
appearance_reconstructions : `list` of `menpo.image.Image`
`List` of the appearance reconstructions that correspond to the
provided parameters.
"""
appearance_reconstructions = []
previous = 0
for scale, n_iters in enumerate(n_iters_per_scale):
for c in appearance_parameters[previous:previous+n_iters+1]:
instance = self.appearance_models[scale].instance(c)
appearance_reconstructions.append(instance)
previous = n_iters + 1
return appearance_reconstructions
def __str__(self):
return _aam_str(self)
[docs]class MaskedAAM(AAM):
r"""
Class for training a multi-scale patch-based Masked Active Appearance Model.
The appearance of this model is formulated by simply masking an image
with a patch-based mask.
Parameters
----------
images : `list` of `menpo.image.Image`
The `list` of training images.
group : `str` or ``None``, optional
The landmark group that will be used to train the AAM. If ``None`` and
the images only have a single landmark group, then that is the one
that will be used. Note that all the training images need to have the
specified landmark group.
holistic_features : `closure` or `list` of `closure`, optional
The features that will be extracted from the training images. Note
that the features are extracted before warping the images to the
reference shape. If `list`, then it must define a feature function per
scale. Please refer to `menpo.feature` for a list of potential features.
reference_shape : `menpo.shape.PointCloud` or ``None``, optional
The reference shape that will be used for building the AAM. The purpose
of the reference shape is to normalise the size of the training images.
The normalization is performed by rescaling all the training images
so that the scale of their ground truth shapes matches the scale of
the reference shape. Note that the reference shape is rescaled with
respect to the `diagonal` before performing the normalisation. If
``None``, then the mean shape will be used.
diagonal : `int` or ``None``, optional
This parameter is used to rescale the reference shape so that the
diagonal of its bounding box matches the provided value. In other
words, this parameter controls the size of the model at the highest
scale. If ``None``, then the reference shape does not get rescaled.
scales : `float` or `tuple` of `float`, optional
The scale value of each scale. They must provided in ascending order,
i.e. from lowest to highest scale. If `float`, then a single scale is
assumed.
patch_shape : (`int`, `int`), optional
The size of the patches of the mask that is used to sample the
appearance vectors.
shape_model_cls : `subclass` of :map:`PDM`, optional
The class to be used for building the shape model. The most common
choice is :map:`OrthoPDM`.
max_shape_components : `int`, `float`, `list` of those or ``None``, optional
The number of shape components to keep. If `int`, then it sets the exact
number of components. If `float`, then it defines the variance
percentage that will be kept. If `list`, then it should
define a value per scale. If a single number, then this will be
applied to all scales. If ``None``, then all the components are kept.
Note that the unused components will be permanently trimmed.
max_appearance_components : `int`, `float`, `list` of those or ``None``, optional
The number of appearance components to keep. If `int`, then it sets the
exact number of components. If `float`, then it defines the variance
percentage that will be kept. If `list`, then it should define a value
per scale. If a single number, then this will be applied to all
scales. If ``None``, then all the components are kept. Note that the
unused components will be permanently trimmed.
verbose : `bool`, optional
If ``True``, then the progress of building the AAM will be printed.
batch_size : `int` or ``None``, optional
If an `int` is provided, then the training is performed in an
incremental fashion on image batches of size equal to the provided
value. If ``None``, then the training is performed directly on the
all the images.
"""
def __init__(self, images, group=None, holistic_features=no_op,
reference_shape=None, diagonal=None, scales=(0.5, 1.0),
patch_shape=(17, 17), shape_model_cls=OrthoPDM,
max_shape_components=None, max_appearance_components=None,
verbose=False, batch_size=None):
# Check arguments
n_scales = len(checks.check_scales(scales))
self.patch_shape = checks.check_patch_shape(patch_shape, n_scales)
# Call superclass
super(MaskedAAM, self).__init__(
images, group=group, verbose=verbose,
reference_shape=reference_shape,
holistic_features=holistic_features,
transform=DifferentiableThinPlateSplines, diagonal=diagonal,
scales=scales, max_shape_components=max_shape_components,
max_appearance_components=max_appearance_components,
shape_model_cls=shape_model_cls, batch_size=batch_size)
def _warp_images(self, images, shapes, reference_shape, scale_index,
prefix, verbose):
reference_frame = build_patch_reference_frame(
reference_shape, patch_shape=self.patch_shape[scale_index])
return warp_images(images, shapes, reference_frame, self.transform,
prefix=prefix, verbose=verbose)
@property
def _str_title(self):
return 'Masked Active Appearance Model'
def _instance(self, scale_index, shape_instance, appearance_instance):
template = self.appearance_models[scale_index].mean()
landmarks = template.landmarks['source']
reference_frame = build_patch_reference_frame(
shape_instance, patch_shape=self.patch_shape[scale_index])
transform = self.transform(
reference_frame.landmarks['source'], landmarks)
return appearance_instance.as_unmasked(copy=False).warp_to_mask(
reference_frame.mask, transform, warp_landmarks=True)
def __str__(self):
return _aam_str(self)
[docs]class LinearAAM(AAM):
r"""
Class for training a multi-scale Linear Active Appearance Model.
Parameters
----------
images : `list` of `menpo.image.Image`
The `list` of training images.
group : `str` or ``None``, optional
The landmark group that will be used to train the AAM. If ``None`` and
the images only have a single landmark group, then that is the one
that will be used. Note that all the training images need to have the
specified landmark group.
holistic_features : `closure` or `list` of `closure`, optional
The features that will be extracted from the training images. Note
that the features are extracted before warping the images to the
reference shape. If `list`, then it must define a feature function per
scale. Please refer to `menpo.feature` for a list of potential features.
reference_shape : `menpo.shape.PointCloud` or ``None``, optional
The reference shape that will be used for building the AAM. The purpose
of the reference shape is to normalise the size of the training images.
The normalization is performed by rescaling all the training images
so that the scale of their ground truth shapes matches the scale of
the reference shape. Note that the reference shape is rescaled with
respect to the `diagonal` before performing the normalisation. If
``None``, then the mean shape will be used.
diagonal : `int` or ``None``, optional
This parameter is used to rescale the reference shape so that the
diagonal of its bounding box matches the provided value. In other
words, this parameter controls the size of the model at the highest
scale. If ``None``, then the reference shape does not get rescaled.
scales : `float` or `tuple` of `float`, optional
The scale value of each scale. They must provided in ascending order,
i.e. from lowest to highest scale. If `float`, then a single scale is
assumed.
transform : `subclass` of :map:`DL` and :map:`DX`, optional
A differential warp transform object, e.g.
:map:`DifferentiablePiecewiseAffine` or
:map:`DifferentiableThinPlateSplines`.
shape_model_cls : `subclass` of :map:`PDM`, optional
The class to be used for building the shape model. The most common
choice is :map:`OrthoPDM`.
max_shape_components : `int`, `float`, `list` of those or ``None``, optional
The number of shape components to keep. If `int`, then it sets the exact
number of components. If `float`, then it defines the variance
percentage that will be kept. If `list`, then it should
define a value per scale. If a single number, then this will be
applied to all scales. If ``None``, then all the components are kept.
Note that the unused components will be permanently trimmed.
max_appearance_components : `int`, `float`, `list` of those or ``None``, optional
The number of appearance components to keep. If `int`, then it sets the
exact number of components. If `float`, then it defines the variance
percentage that will be kept. If `list`, then it should define a value
per scale. If a single number, then this will be applied to all
scales. If ``None``, then all the components are kept. Note that the
unused components will be permanently trimmed.
verbose : `bool`, optional
If ``True``, then the progress of building the AAM will be printed.
batch_size : `int` or ``None``, optional
If an `int` is provided, then the training is performed in an
incremental fashion on image batches of size equal to the provided
value. If ``None``, then the training is performed directly on the
all the images.
"""
def __init__(self, images, group=None, holistic_features=no_op,
reference_shape=None, diagonal=None, scales=(0.5, 1.0),
transform=DifferentiableThinPlateSplines,
shape_model_cls=OrthoPDM, max_shape_components=None,
max_appearance_components=None, verbose=False,
batch_size=None):
super(LinearAAM, self).__init__(
images, group=group, verbose=verbose,
reference_shape=reference_shape,
holistic_features=holistic_features, transform=transform,
diagonal=diagonal, scales=scales,
max_shape_components=max_shape_components,
max_appearance_components=max_appearance_components,
shape_model_cls=shape_model_cls, batch_size=batch_size)
@property
def _str_title(self):
r"""
Returns a string containing name of the model.
:type: `string`
"""
return 'Linear Active Appearance Model'
def _build_shape_model(self, shapes, scale_index):
mean_aligned_shape = mean_pointcloud(align_shapes(shapes))
self.n_landmarks = mean_aligned_shape.n_points
self.reference_frame = build_reference_frame(mean_aligned_shape)
dense_shapes = densify_shapes(shapes, self.reference_frame,
self.transform)
# Build dense shape model
max_sc = self.max_shape_components[scale_index]
return self._shape_model_cls[scale_index](dense_shapes,
max_n_components=max_sc)
def _increment_shape_model(self, shapes, scale_index,
forgetting_factor=1.0):
aligned_shapes = align_shapes(shapes)
dense_shapes = densify_shapes(aligned_shapes, self.reference_frame,
self.transform)
# Increment shape model
self.shape_models[scale_index].increment(
dense_shapes, forgetting_factor=forgetting_factor,
max_n_components=self.max_shape_components[scale_index])
def _warp_images(self, images, shapes, reference_shape, scale_index,
prefix, verbose):
return warp_images(images, shapes, self.reference_frame,
self.transform, prefix=prefix,
verbose=verbose)
# TODO: implement me!
def _instance(self, scale_index, shape_instance, appearance_instance):
raise NotImplementedError()
# TODO: implement me!
# TODO: implement me!
[docs] def build_fitter_interfaces(self, sampling):
r"""
Method that builds the correct Lucas-Kanade fitting interface. It
only applies in case you wish to fit the AAM with a Lucas-Kanade
algorithm (i.e. :map:`LucasKanadeAAMFitter`).
Parameters
----------
sampling : `list` of `int` or `ndarray` or ``None``
It defines a sampling mask per scale. If `int`, then it
defines the sub-sampling step of the sampling mask. If `ndarray`,
then it explicitly defines the sampling mask. If ``None``, then no
sub-sampling is applied.
Returns
-------
fitter_interfaces : `list`
The `list` of Lucas-Kanade interface per scale.
"""
interfaces = []
for am, sm, s in zip(self.appearance_models, self.shape_models,
sampling):
template = am.mean()
# This is pretty hacky as we just steal the OrthoPDM's PCAModel
md_transform = LinearOrthoMDTransform(
sm.model, self.reference_shape)
interface = LucasKanadeLinearInterface(am, md_transform,
template, sampling=s)
interfaces.append(interface)
return interfaces
def __str__(self):
return _aam_str(self)
[docs]class LinearMaskedAAM(AAM):
r"""
Class for training a multi-scale Linear Masked Active Appearance Model.
Parameters
----------
images : `list` of `menpo.image.Image`
The `list` of training images.
group : `str` or ``None``, optional
The landmark group that will be used to train the AAM. If ``None`` and
the images only have a single landmark group, then that is the one
that will be used. Note that all the training images need to have the
specified landmark group.
holistic_features : `closure` or `list` of `closure`, optional
The features that will be extracted from the training images. Note
that the features are extracted before warping the images to the
reference shape. If `list`, then it must define a feature function per
scale. Please refer to `menpo.feature` for a list of potential features.
reference_shape : `menpo.shape.PointCloud` or ``None``, optional
The reference shape that will be used for building the AAM. The purpose
of the reference shape is to normalise the size of the training images.
The normalization is performed by rescaling all the training images
so that the scale of their ground truth shapes matches the scale of
the reference shape. Note that the reference shape is rescaled with
respect to the `diagonal` before performing the normalisation. If
``None``, then the mean shape will be used.
diagonal : `int` or ``None``, optional
This parameter is used to rescale the reference shape so that the
diagonal of its bounding box matches the provided value. In other
words, this parameter controls the size of the model at the highest
scale. If ``None``, then the reference shape does not get rescaled.
scales : `float` or `tuple` of `float`, optional
The scale value of each scale. They must provided in ascending order,
i.e. from lowest to highest scale. If `float`, then a single scale is
assumed.
patch_shape : (`int`, `int`), optional
The size of the patches of the mask that is used to sample the
appearance vectors.
shape_model_cls : `subclass` of :map:`PDM`, optional
The class to be used for building the shape model. The most common
choice is :map:`OrthoPDM`.
max_shape_components : `int`, `float`, `list` of those or ``None``, optional
The number of shape components to keep. If `int`, then it sets the exact
number of components. If `float`, then it defines the variance
percentage that will be kept. If `list`, then it should
define a value per scale. If a single number, then this will be
applied to all scales. If ``None``, then all the components are kept.
Note that the unused components will be permanently trimmed.
max_appearance_components : `int`, `float`, `list` of those or ``None``, optional
The number of appearance components to keep. If `int`, then it sets the
exact number of components. If `float`, then it defines the variance
percentage that will be kept. If `list`, then it should define a value
per scale. If a single number, then this will be applied to all
scales. If ``None``, then all the components are kept. Note that the
unused components will be permanently trimmed.
verbose : `bool`, optional
If ``True``, then the progress of building the AAM will be printed.
batch_size : `int` or ``None``, optional
If an `int` is provided, then the training is performed in an
incremental fashion on image batches of size equal to the provided
value. If ``None``, then the training is performed directly on the
all the images.
"""
def __init__(self, images, group=None, holistic_features=no_op,
reference_shape=None, diagonal=None, scales=(0.5, 1.0),
patch_shape=(17, 17), shape_model_cls=OrthoPDM,
max_shape_components=None, max_appearance_components=None,
verbose=False, batch_size=None):
# Check arguments
n_scales = len(checks.check_scales(scales))
self.patch_shape = checks.check_patch_shape(patch_shape, n_scales)
# Call superclass
super(LinearMaskedAAM, self).__init__(
images, group=group, verbose=verbose,
reference_shape=reference_shape,
holistic_features=holistic_features,
transform=DifferentiableThinPlateSplines, diagonal=diagonal,
scales=scales, max_shape_components=max_shape_components,
max_appearance_components=max_appearance_components,
shape_model_cls=shape_model_cls, batch_size=batch_size)
@property
def _str_title(self):
r"""
Returns a string containing name of the model.
:type: `string`
"""
return 'Linear Masked Active Appearance Model'
def _build_shape_model(self, shapes, scale_index):
mean_aligned_shape = mean_pointcloud(align_shapes(shapes))
self.n_landmarks = mean_aligned_shape.n_points
self.reference_frame = build_patch_reference_frame(
mean_aligned_shape, patch_shape=self.patch_shape[scale_index])
dense_shapes = densify_shapes(shapes, self.reference_frame,
self.transform)
# Build dense shape model
max_sc = self.max_shape_components[scale_index]
return self._shape_model_cls[scale_index](dense_shapes,
max_n_components=max_sc)
def _increment_shape_model(self, shapes, scale_index,
forgetting_factor=1.0):
aligned_shapes = align_shapes(shapes)
dense_shapes = densify_shapes(aligned_shapes, self.reference_frame,
self.transform)
# Increment shape model
self.shape_models[scale_index].increment(
dense_shapes, forgetting_factor=forgetting_factor,
max_n_components=self.max_shape_components[scale_index])
def _warp_images(self, images, shapes, reference_shape, scale_index,
prefix, verbose):
return warp_images(images, shapes, self.reference_frame,
self.transform, prefix=prefix,
verbose=verbose)
# TODO: implement me!
def _instance(self, scale_index, shape_instance, appearance_instance):
raise NotImplementedError()
# TODO: implement me!
# TODO: implement me!
[docs] def build_fitter_interfaces(self, sampling):
r"""
Method that builds the correct Lucas-Kanade fitting interface. It
only applies in case you wish to fit the AAM with a Lucas-Kanade
algorithm (i.e. :map:`LucasKanadeAAMFitter`).
Parameters
----------
sampling : `list` of `int` or `ndarray` or ``None``
It defines a sampling mask per scale. If `int`, then it
defines the sub-sampling step of the sampling mask. If `ndarray`,
then it explicitly defines the sampling mask. If ``None``, then no
sub-sampling is applied.
Returns
-------
fitter_interfaces : `list`
The `list` of Lucas-Kanade interface per scale.
"""
interfaces = []
for am, sm, s in zip(self.appearance_models, self.shape_models,
sampling):
template = am.mean()
# This is pretty hacky as we just steal the OrthoPDM's PCAModel
md_transform = LinearOrthoMDTransform(
sm.model, self.reference_shape)
interface = LucasKanadeLinearInterface(am, md_transform,
template, sampling=s)
interfaces.append(interface)
return interfaces
def __str__(self):
return _aam_str(self)
# TODO: implement offsets support?
[docs]class PatchAAM(AAM):
r"""
Class for training a multi-scale Patch-Based Active Appearance Model. The
appearance of this model is formulated by simply sampling patches around
the image's landmarks.
Parameters
----------
images : `list` of `menpo.image.Image`
The `list` of training images.
group : `str` or ``None``, optional
The landmark group that will be used to train the AAM. If ``None`` and
the images only have a single landmark group, then that is the one
that will be used. Note that all the training images need to have the
specified landmark group.
holistic_features : `closure` or `list` of `closure`, optional
The features that will be extracted from the training images. Note
that the features are extracted before warping the images to the
reference shape. If `list`, then it must define a feature function per
scale. Please refer to `menpo.feature` for a list of potential features.
reference_shape : `menpo.shape.PointCloud` or ``None``, optional
The reference shape that will be used for building the AAM. The purpose
of the reference shape is to normalise the size of the training images.
The normalization is performed by rescaling all the training images
so that the scale of their ground truth shapes matches the scale of
the reference shape. Note that the reference shape is rescaled with
respect to the `diagonal` before performing the normalisation. If
``None``, then the mean shape will be used.
diagonal : `int` or ``None``, optional
This parameter is used to rescale the reference shape so that the
diagonal of its bounding box matches the provided value. In other
words, this parameter controls the size of the model at the highest
scale. If ``None``, then the reference shape does not get rescaled.
scales : `float` or `tuple` of `float`, optional
The scale value of each scale. They must provided in ascending order,
i.e. from lowest to highest scale. If `float`, then a single scale is
assumed.
patch_shape : (`int`, `int`) or `list` of (`int`, `int`), optional
The shape of the patches to be extracted. If a `list` is provided,
then it defines a patch shape per scale.
patch_normalisation : `list` of `callable` or a single `callable`, optional
The normalisation function to be applied on the extracted patches. If
`list`, then it must have length equal to the number of scales. If a
single patch normalization `callable`, then this is the one applied to
all scales.
shape_model_cls : `subclass` of :map:`PDM`, optional
The class to be used for building the shape model. The most common
choice is :map:`OrthoPDM`.
max_shape_components : `int`, `float`, `list` of those or ``None``, optional
The number of shape components to keep. If `int`, then it sets the exact
number of components. If `float`, then it defines the variance
percentage that will be kept. If `list`, then it should
define a value per scale. If a single number, then this will be
applied to all scales. If ``None``, then all the components are kept.
Note that the unused components will be permanently trimmed.
max_appearance_components : `int`, `float`, `list` of those or ``None``, optional
The number of appearance components to keep. If `int`, then it sets the
exact number of components. If `float`, then it defines the variance
percentage that will be kept. If `list`, then it should define a value
per scale. If a single number, then this will be applied to all
scales. If ``None``, then all the components are kept. Note that the
unused components will be permanently trimmed.
verbose : `bool`, optional
If ``True``, then the progress of building the AAM will be printed.
batch_size : `int` or ``None``, optional
If an `int` is provided, then the training is performed in an
incremental fashion on image batches of size equal to the provided
value. If ``None``, then the training is performed directly on the
all the images.
"""
def __init__(self, images, group=None, holistic_features=no_op,
reference_shape=None, diagonal=None, scales=(0.5, 1.0),
patch_shape=(17, 17), patch_normalisation=no_op,
shape_model_cls=OrthoPDM, max_shape_components=None,
max_appearance_components=None, verbose=False,
batch_size=None):
n_scales = len(checks.check_scales(scales))
self.patch_shape = checks.check_patch_shape(patch_shape, n_scales)
self.patch_normalisation = checks.check_callable(patch_normalisation,
n_scales)
super(PatchAAM, self).__init__(
images, group=group, verbose=verbose,
reference_shape=reference_shape,
holistic_features=holistic_features, transform=None,
diagonal=diagonal, scales=scales,
max_shape_components=max_shape_components,
max_appearance_components=max_appearance_components,
shape_model_cls=shape_model_cls, batch_size=batch_size)
@property
def _str_title(self):
r"""
Returns a string containing name of the model.
:type: `string`
"""
return 'Patch-based Active Appearance Model'
def _warp_images(self, images, shapes, reference_shape, scale_index,
prefix, verbose):
return extract_patches(
images, shapes, self.patch_shape[scale_index],
normalise_function=self.patch_normalisation[scale_index],
prefix=prefix, verbose=verbose)
def _instance(self, scale_index, shape_instance, appearance_instance):
return shape_instance, appearance_instance
[docs] def build_fitter_interfaces(self, sampling):
r"""
Method that builds the correct Lucas-Kanade fitting interface. It
only applies in case you wish to fit the AAM with a Lucas-Kanade
algorithm (i.e. :map:`LucasKanadeAAMFitter`).
Parameters
----------
sampling : `list` of `int` or `ndarray` or ``None``
It defines a sampling mask per scale. If `int`, then it
defines the sub-sampling step of the sampling mask. If `ndarray`,
then it explicitly defines the sampling mask. If ``None``, then no
sub-sampling is applied.
Returns
-------
fitter_interfaces : `list`
The `list` of Lucas-Kanade interface per scale.
"""
interfaces = []
for j, (am, sm, s) in enumerate(zip(self.appearance_models,
self.shape_models,
sampling)):
template = am.mean()
interface = LucasKanadePatchInterface(
am, sm, template, sampling=s,
patch_shape=self.patch_shape[j],
patch_normalisation=self.patch_normalisation[j])
interfaces.append(interface)
return interfaces
[docs] def appearance_reconstructions(self, appearance_parameters,
n_iters_per_scale):
r"""
Method that generates the appearance reconstructions given a set of
appearance parameters. This is to be combined with a :map:`AAMResult`
object, in order to generate the appearance reconstructions of a
fitting procedure.
Parameters
----------
appearance_parameters : `list` of `ndarray`
A set of appearance parameters per fitting iteration. It can be
retrieved as a property of a :map:`AAMResult` object.
n_iters_per_scale : `list` of `int`
The number of iterations per scale. This is necessary in order to
figure out which appearance parameters correspond to the model of
each scale. It can be retrieved as a property of a :map:`AAMResult`
object.
Returns
-------
appearance_reconstructions : `list` of `ndarray`
List of the appearance reconstructions that correspond to the
provided parameters.
"""
appearance_reconstructions = []
previous = 0
for scale, n_iters in enumerate(n_iters_per_scale):
for c in appearance_parameters[previous:previous+n_iters+1]:
instance = self.appearance_models[scale].instance(c).pixels
appearance_reconstructions.append(instance)
previous = n_iters + 1
return appearance_reconstructions
def __str__(self):
return _aam_str(self)
def _aam_str(aam):
if aam.diagonal is not None:
diagonal = aam.diagonal
else:
y, x = aam.reference_shape.range()
diagonal = np.sqrt(x ** 2 + y ** 2)
# Compute scale info strings
scales_info = []
lvl_str_tmplt = r""" - Scale {}
- Holistic feature: {}
- Appearance model class: {}
- {} appearance components
- Shape model class: {}
- {} shape components
- {} similarity transform parameters"""
for k, s in enumerate(aam.scales):
scales_info.append(lvl_str_tmplt.format(
s, name_of_callable(aam.holistic_features[k]),
name_of_callable(aam.appearance_models[k]),
aam.appearance_models[k].n_components,
name_of_callable(aam.shape_models[k]),
aam.shape_models[k].model.n_components,
aam.shape_models[k].n_global_parameters))
# Patch based AAM
if hasattr(aam, 'patch_shape'):
for k in range(len(scales_info)):
scales_info[k] += '\n - Patch shape: {}'.format(
aam.patch_shape[k])
scales_info = '\n'.join(scales_info)
if aam.transform is not None:
transform_str = 'Images warped with {} transform'.format(
name_of_callable(aam.transform))
else:
transform_str = 'No image warping performed'
cls_str = r"""{class_title}
- Images scaled to diagonal: {diagonal:.2f}
- {transform}
- Scales: {scales}
{scales_info}
""".format(class_title=aam._str_title,
transform=transform_str,
diagonal=diagonal,
scales=aam.scales,
scales_info=scales_info)
return cls_str
HolisticAAM = AAM