added deepsad base code

Deep-SAD-PyTorch/src/DeepSAD.py

@@ -0,0 +1,161 @@
+import json
+import torch
+
+from base.base_dataset import BaseADDataset
+from networks.main import build_network, build_autoencoder
+from optim.DeepSAD_trainer import DeepSADTrainer
+from optim.ae_trainer import AETrainer
+
+
+class DeepSAD(object):
+    """A class for the Deep SAD method.
+
+    Attributes:
+        eta: Deep SAD hyperparameter eta (must be 0 < eta).
+        c: Hypersphere center c.
+        net_name: A string indicating the name of the neural network to use.
+        net: The neural network phi.
+        trainer: DeepSADTrainer to train a Deep SAD model.
+        optimizer_name: A string indicating the optimizer to use for training the Deep SAD network.
+        ae_net: The autoencoder network corresponding to phi for network weights pretraining.
+        ae_trainer: AETrainer to train an autoencoder in pretraining.
+        ae_optimizer_name: A string indicating the optimizer to use for pretraining the autoencoder.
+        results: A dictionary to save the results.
+        ae_results: A dictionary to save the autoencoder results.
+    """
+
+    def __init__(self, eta: float = 1.0):
+        """Inits DeepSAD with hyperparameter eta."""
+
+        self.eta = eta
+        self.c = None  # hypersphere center c
+
+        self.net_name = None
+        self.net = None  # neural network phi
+
+        self.trainer = None
+        self.optimizer_name = None
+
+        self.ae_net = None  # autoencoder network for pretraining
+        self.ae_trainer = None
+        self.ae_optimizer_name = None
+
+        self.results = {
+            'train_time': None,
+            'test_auc': None,
+            'test_time': None,
+            'test_scores': None,
+        }
+
+        self.ae_results = {
+            'train_time': None,
+            'test_auc': None,
+            'test_time': None
+        }
+
+    def set_network(self, net_name):
+        """Builds the neural network phi."""
+        self.net_name = net_name
+        self.net = build_network(net_name)
+
+    def train(self, dataset: BaseADDataset, optimizer_name: str = 'adam', lr: float = 0.001, n_epochs: int = 50,
+              lr_milestones: tuple = (), batch_size: int = 128, weight_decay: float = 1e-6, device: str = 'cuda',
+              n_jobs_dataloader: int = 0):
+        """Trains the Deep SAD model on the training data."""
+
+        self.optimizer_name = optimizer_name
+        self.trainer = DeepSADTrainer(self.c, self.eta, optimizer_name=optimizer_name, lr=lr, n_epochs=n_epochs,
+                                      lr_milestones=lr_milestones, batch_size=batch_size, weight_decay=weight_decay,
+                                      device=device, n_jobs_dataloader=n_jobs_dataloader)
+        # Get the model
+        self.net = self.trainer.train(dataset, self.net)
+        self.results['train_time'] = self.trainer.train_time
+        self.c = self.trainer.c.cpu().data.numpy().tolist()  # get as list
+
+    def test(self, dataset: BaseADDataset, device: str = 'cuda', n_jobs_dataloader: int = 0):
+        """Tests the Deep SAD model on the test data."""
+
+        if self.trainer is None:
+            self.trainer = DeepSADTrainer(self.c, self.eta, device=device, n_jobs_dataloader=n_jobs_dataloader)
+
+        self.trainer.test(dataset, self.net)
+
+        # Get results
+        self.results['test_auc'] = self.trainer.test_auc
+        self.results['test_time'] = self.trainer.test_time
+        self.results['test_scores'] = self.trainer.test_scores
+
+    def pretrain(self, dataset: BaseADDataset, optimizer_name: str = 'adam', lr: float = 0.001, n_epochs: int = 100,
+                 lr_milestones: tuple = (), batch_size: int = 128, weight_decay: float = 1e-6, device: str = 'cuda',
+                 n_jobs_dataloader: int = 0):
+        """Pretrains the weights for the Deep SAD network phi via autoencoder."""
+
+        # Set autoencoder network
+        self.ae_net = build_autoencoder(self.net_name)
+
+        # Train
+        self.ae_optimizer_name = optimizer_name
+        self.ae_trainer = AETrainer(optimizer_name, lr=lr, n_epochs=n_epochs, lr_milestones=lr_milestones,
+                                    batch_size=batch_size, weight_decay=weight_decay, device=device,
+                                    n_jobs_dataloader=n_jobs_dataloader)
+        self.ae_net = self.ae_trainer.train(dataset, self.ae_net)
+
+        # Get train results
+        self.ae_results['train_time'] = self.ae_trainer.train_time
+
+        # Test
+        self.ae_trainer.test(dataset, self.ae_net)
+
+        # Get test results
+        self.ae_results['test_auc'] = self.ae_trainer.test_auc
+        self.ae_results['test_time'] = self.ae_trainer.test_time
+
+        # Initialize Deep SAD network weights from pre-trained encoder
+        self.init_network_weights_from_pretraining()
+
+    def init_network_weights_from_pretraining(self):
+        """Initialize the Deep SAD network weights from the encoder weights of the pretraining autoencoder."""
+
+        net_dict = self.net.state_dict()
+        ae_net_dict = self.ae_net.state_dict()
+
+        # Filter out decoder network keys
+        ae_net_dict = {k: v for k, v in ae_net_dict.items() if k in net_dict}
+        # Overwrite values in the existing state_dict
+        net_dict.update(ae_net_dict)
+        # Load the new state_dict
+        self.net.load_state_dict(net_dict)
+
+    def save_model(self, export_model, save_ae=True):
+        """Save Deep SAD model to export_model."""
+
+        net_dict = self.net.state_dict()
+        ae_net_dict = self.ae_net.state_dict() if save_ae else None
+
+        torch.save({'c': self.c,
+                    'net_dict': net_dict,
+                    'ae_net_dict': ae_net_dict}, export_model)
+
+    def load_model(self, model_path, load_ae=False, map_location='cpu'):
+        """Load Deep SAD model from model_path."""
+
+        model_dict = torch.load(model_path, map_location=map_location)
+
+        self.c = model_dict['c']
+        self.net.load_state_dict(model_dict['net_dict'])
+
+        # load autoencoder parameters if specified
+        if load_ae:
+            if self.ae_net is None:
+                self.ae_net = build_autoencoder(self.net_name)
+            self.ae_net.load_state_dict(model_dict['ae_net_dict'])
+
+    def save_results(self, export_json):
+        """Save results dict to a JSON-file."""
+        with open(export_json, 'w') as fp:
+            json.dump(self.results, fp)
+
+    def save_ae_results(self, export_json):
+        """Save autoencoder results dict to a JSON-file."""
+        with open(export_json, 'w') as fp:
+            json.dump(self.ae_results, fp)

Deep-SAD-PyTorch/src/base/__init__.py

@@ -0,0 +1,5 @@
+from .base_dataset import *
+from .torchvision_dataset import *
+from .odds_dataset import *
+from .base_net import *
+from .base_trainer import *

Deep-SAD-PyTorch/src/base/base_dataset.py

@@ -0,0 +1,26 @@
+from abc import ABC, abstractmethod
+from torch.utils.data import DataLoader
+
+
+class BaseADDataset(ABC):
+    """Anomaly detection dataset base class."""
+
+    def __init__(self, root: str):
+        super().__init__()
+        self.root = root  # root path to data
+
+        self.n_classes = 2  # 0: normal, 1: outlier
+        self.normal_classes = None  # tuple with original class labels that define the normal class
+        self.outlier_classes = None  # tuple with original class labels that define the outlier class
+
+        self.train_set = None  # must be of type torch.utils.data.Dataset
+        self.test_set = None  # must be of type torch.utils.data.Dataset
+
+    @abstractmethod
+    def loaders(self, batch_size: int, shuffle_train=True, shuffle_test=False, num_workers: int = 0) -> (
+            DataLoader, DataLoader):
+        """Implement data loaders of type torch.utils.data.DataLoader for train_set and test_set."""
+        pass
+
+    def __repr__(self):
+        return self.__class__.__name__

Deep-SAD-PyTorch/src/base/base_net.py

@@ -0,0 +1,26 @@
+import logging
+import torch.nn as nn
+import numpy as np
+
+
+class BaseNet(nn.Module):
+    """Base class for all neural networks."""
+
+    def __init__(self):
+        super().__init__()
+        self.logger = logging.getLogger(self.__class__.__name__)
+        self.rep_dim = None  # representation dimensionality, i.e. dim of the code layer or last layer
+
+    def forward(self, *input):
+        """
+        Forward pass logic
+        :return: Network output
+        """
+        raise NotImplementedError
+
+    def summary(self):
+        """Network summary."""
+        net_parameters = filter(lambda p: p.requires_grad, self.parameters())
+        params = sum([np.prod(p.size()) for p in net_parameters])
+        self.logger.info('Trainable parameters: {}'.format(params))
+        self.logger.info(self)

Deep-SAD-PyTorch/src/base/base_trainer.py

@@ -0,0 +1,34 @@
+from abc import ABC, abstractmethod
+from .base_dataset import BaseADDataset
+from .base_net import BaseNet
+
+
+class BaseTrainer(ABC):
+    """Trainer base class."""
+
+    def __init__(self, optimizer_name: str, lr: float, n_epochs: int, lr_milestones: tuple, batch_size: int,
+                 weight_decay: float, device: str, n_jobs_dataloader: int):
+        super().__init__()
+        self.optimizer_name = optimizer_name
+        self.lr = lr
+        self.n_epochs = n_epochs
+        self.lr_milestones = lr_milestones
+        self.batch_size = batch_size
+        self.weight_decay = weight_decay
+        self.device = device
+        self.n_jobs_dataloader = n_jobs_dataloader
+
+    @abstractmethod
+    def train(self, dataset: BaseADDataset, net: BaseNet) -> BaseNet:
+        """
+        Implement train method that trains the given network using the train_set of dataset.
+        :return: Trained net
+        """
+        pass
+
+    @abstractmethod
+    def test(self, dataset: BaseADDataset, net: BaseNet):
+        """
+        Implement test method that evaluates the test_set of dataset on the given network.
+        """
+        pass

Deep-SAD-PyTorch/src/base/odds_dataset.py

@@ -0,0 +1,110 @@
+from pathlib import Path
+from torch.utils.data import Dataset
+from scipy.io import loadmat
+from sklearn.model_selection import train_test_split
+from sklearn.preprocessing import StandardScaler, MinMaxScaler
+from torchvision.datasets.utils import download_url
+
+import os
+import torch
+import numpy as np
+
+
+class ODDSDataset(Dataset):
+    """
+    ODDSDataset class for datasets from Outlier Detection DataSets (ODDS): http://odds.cs.stonybrook.edu/
+
+    Dataset class with additional targets for the semi-supervised setting and modification of __getitem__ method
+    to also return the semi-supervised target as well as the index of a data sample.
+    """
+
+    urls = {
+        'arrhythmia': 'https://www.dropbox.com/s/lmlwuspn1sey48r/arrhythmia.mat?dl=1',
+        'cardio': 'https://www.dropbox.com/s/galg3ihvxklf0qi/cardio.mat?dl=1',
+        'satellite': 'https://www.dropbox.com/s/dpzxp8jyr9h93k5/satellite.mat?dl=1',
+        'satimage-2': 'https://www.dropbox.com/s/hckgvu9m6fs441p/satimage-2.mat?dl=1',
+        'shuttle': 'https://www.dropbox.com/s/mk8ozgisimfn3dw/shuttle.mat?dl=1',
+        'thyroid': 'https://www.dropbox.com/s/bih0e15a0fukftb/thyroid.mat?dl=1'
+    }
+
+    def __init__(self, root: str, dataset_name: str, train=True, random_state=None, download=False):
+        super(Dataset, self).__init__()
+
+        self.classes = [0, 1]
+
+        if isinstance(root, torch._six.string_classes):
+            root = os.path.expanduser(root)
+        self.root = Path(root)
+        self.dataset_name = dataset_name
+        self.train = train  # training set or test set
+        self.file_name = self.dataset_name + '.mat'
+        self.data_file = self.root / self.file_name
+
+        if download:
+            self.download()
+
+        mat = loadmat(self.data_file)
+        X = mat['X']
+        y = mat['y'].ravel()
+        idx_norm = y == 0
+        idx_out = y == 1
+
+        # 60% data for training and 40% for testing; keep outlier ratio
+        X_train_norm, X_test_norm, y_train_norm, y_test_norm = train_test_split(X[idx_norm], y[idx_norm],
+                                                                                test_size=0.4,
+                                                                                random_state=random_state)
+        X_train_out, X_test_out, y_train_out, y_test_out = train_test_split(X[idx_out], y[idx_out],
+                                                                            test_size=0.4,
+                                                                            random_state=random_state)
+        X_train = np.concatenate((X_train_norm, X_train_out))
+        X_test = np.concatenate((X_test_norm, X_test_out))
+        y_train = np.concatenate((y_train_norm, y_train_out))
+        y_test = np.concatenate((y_test_norm, y_test_out))
+
+        # Standardize data (per feature Z-normalization, i.e. zero-mean and unit variance)
+        scaler = StandardScaler().fit(X_train)
+        X_train_stand = scaler.transform(X_train)
+        X_test_stand = scaler.transform(X_test)
+
+        # Scale to range [0,1]
+        minmax_scaler = MinMaxScaler().fit(X_train_stand)
+        X_train_scaled = minmax_scaler.transform(X_train_stand)
+        X_test_scaled = minmax_scaler.transform(X_test_stand)
+
+        if self.train:
+            self.data = torch.tensor(X_train_scaled, dtype=torch.float32)
+            self.targets = torch.tensor(y_train, dtype=torch.int64)
+        else:
+            self.data = torch.tensor(X_test_scaled, dtype=torch.float32)
+            self.targets = torch.tensor(y_test, dtype=torch.int64)
+
+        self.semi_targets = torch.zeros_like(self.targets)
+
+    def __getitem__(self, index):
+        """
+        Args:
+            index (int): Index
+
+        Returns:
+            tuple: (sample, target, semi_target, index)
+        """
+        sample, target, semi_target = self.data[index], int(self.targets[index]), int(self.semi_targets[index])
+
+        return sample, target, semi_target, index
+
+    def __len__(self):
+        return len(self.data)
+
+    def _check_exists(self):
+        return os.path.exists(self.data_file)
+
+    def download(self):
+        """Download the ODDS dataset if it doesn't exist in root already."""
+
+        if self._check_exists():
+            return
+
+        # download file
+        download_url(self.urls[self.dataset_name], self.root, self.file_name)
+
+        print('Done!')

Deep-SAD-PyTorch/src/base/torchvision_dataset.py

@@ -0,0 +1,17 @@
+from .base_dataset import BaseADDataset
+from torch.utils.data import DataLoader
+
+
+class TorchvisionDataset(BaseADDataset):
+    """TorchvisionDataset class for datasets already implemented in torchvision.datasets."""
+
+    def __init__(self, root: str):
+        super().__init__(root)
+
+    def loaders(self, batch_size: int, shuffle_train=True, shuffle_test=False, num_workers: int = 0) -> (
+            DataLoader, DataLoader):
+        train_loader = DataLoader(dataset=self.train_set, batch_size=batch_size, shuffle=shuffle_train,
+                                  num_workers=num_workers, drop_last=True)
+        test_loader = DataLoader(dataset=self.test_set, batch_size=batch_size, shuffle=shuffle_test,
+                                 num_workers=num_workers, drop_last=False)
+        return train_loader, test_loader

Deep-SAD-PyTorch/src/baseline_SemiDGM.py

@@ -0,0 +1,240 @@
+import click
+import torch
+import logging
+import random
+import numpy as np
+
+from utils.config import Config
+from utils.visualization.plot_images_grid import plot_images_grid
+from baselines.SemiDGM import SemiDeepGenerativeModel
+from datasets.main import load_dataset
+
+
+################################################################################
+# Settings
+################################################################################
+@click.command()
+@click.argument('dataset_name', type=click.Choice(['mnist', 'fmnist', 'cifar10', 'arrhythmia', 'cardio', 'satellite',
+                                                   'satimage-2', 'shuttle', 'thyroid']))
+@click.argument('net_name', type=click.Choice(['mnist_DGM_M2', 'mnist_DGM_M1M2', 'fmnist_DGM_M2', 'fmnist_DGM_M1M2',
+                                               'cifar10_DGM_M2', 'cifar10_DGM_M1M2',
+                                               'arrhythmia_DGM_M2', 'cardio_DGM_M2', 'satellite_DGM_M2',
+                                               'satimage-2_DGM_M2', 'shuttle_DGM_M2', 'thyroid_DGM_M2']))
+@click.argument('xp_path', type=click.Path(exists=True))
+@click.argument('data_path', type=click.Path(exists=True))
+@click.option('--load_config', type=click.Path(exists=True), default=None,
+              help='Config JSON-file path (default: None).')
+@click.option('--load_model', type=click.Path(exists=True), default=None,
+              help='Model file path (default: None).')
+@click.option('--ratio_known_normal', type=float, default=0.0,
+              help='Ratio of known (labeled) normal training examples.')
+@click.option('--ratio_known_outlier', type=float, default=0.0,
+              help='Ratio of known (labeled) anomalous training examples.')
+@click.option('--ratio_pollution', type=float, default=0.0,
+              help='Pollution ratio of unlabeled training data with unknown (unlabeled) anomalies.')
+@click.option('--device', type=str, default='cuda', help='Computation device to use ("cpu", "cuda", "cuda:2", etc.).')
+@click.option('--seed', type=int, default=-1, help='Set seed. If -1, use randomization.')
+@click.option('--optimizer_name', type=click.Choice(['adam']), default='adam',
+              help='Name of the optimizer to use for training the Semi-Supervised Deep Generative model.')
+@click.option('--lr', type=float, default=0.001,
+              help='Initial learning rate for training. Default=0.001')
+@click.option('--n_epochs', type=int, default=50, help='Number of epochs to train.')
+@click.option('--lr_milestone', type=int, default=0, multiple=True,
+              help='Lr scheduler milestones at which lr is multiplied by 0.1. Can be multiple and must be increasing.')
+@click.option('--batch_size', type=int, default=128, help='Batch size for mini-batch training.')
+@click.option('--weight_decay', type=float, default=1e-6,
+              help='Weight decay (L2 penalty) hyperparameter.')
+@click.option('--pretrain', type=bool, default=False, help='Pretrain a variational autoencoder.')
+@click.option('--vae_optimizer_name', type=click.Choice(['adam']), default='adam',
+              help='Name of the optimizer to use for variational autoencoder pretraining.')
+@click.option('--vae_lr', type=float, default=0.001,
+              help='Initial learning rate for pretraining. Default=0.001')
+@click.option('--vae_n_epochs', type=int, default=100, help='Number of epochs to train the variational autoencoder.')
+@click.option('--vae_lr_milestone', type=int, default=0, multiple=True,
+              help='Lr scheduler milestones at which lr is multiplied by 0.1. Can be multiple and must be increasing.')
+@click.option('--vae_batch_size', type=int, default=128, help='Batch size for variational autoencoder training.')
+@click.option('--vae_weight_decay', type=float, default=1e-6,
+              help='Weight decay (L2 penalty) hyperparameter for variational autoencoder.')
+@click.option('--num_threads', type=int, default=0,
+              help='Number of threads used for parallelizing CPU operations. 0 means that all resources are used.')
+@click.option('--n_jobs_dataloader', type=int, default=0,
+              help='Number of workers for data loading. 0 means that the data will be loaded in the main process.')
+@click.option('--normal_class', type=int, default=0,
+              help='Specify the normal class of the dataset (all other classes are considered anomalous).')
+@click.option('--known_outlier_class', type=int, default=1,
+              help='Specify the known outlier class of the dataset for semi-supervised anomaly detection.')
+@click.option('--n_known_outlier_classes', type=int, default=0,
+              help='Number of known outlier classes.'
+                   'If 0, no anomalies are known.'
+                   'If 1, outlier class as specified in --known_outlier_class option.'
+                   'If > 1, the specified number of outlier classes will be sampled at random.')
+def main(dataset_name, net_name, xp_path, data_path, load_config, load_model, ratio_known_normal, ratio_known_outlier,
+         ratio_pollution, device, seed, optimizer_name, lr, n_epochs, lr_milestone, batch_size, weight_decay, pretrain,
+         vae_optimizer_name, vae_lr, vae_n_epochs, vae_lr_milestone, vae_batch_size, vae_weight_decay,
+         num_threads, n_jobs_dataloader, normal_class, known_outlier_class, n_known_outlier_classes):
+    """
+    Semi-Supervised Deep Generative model (M1+M2 model) from Kingma et al. (2014)
+
+    :arg DATASET_NAME: Name of the dataset to load.
+    :arg NET_NAME: Name of the neural network to use.
+    :arg XP_PATH: Export path for logging the experiment.
+    :arg DATA_PATH: Root path of data.
+    """
+
+    # Get configuration
+    cfg = Config(locals().copy())
+
+    # Set up logging
+    logging.basicConfig(level=logging.INFO)
+    logger = logging.getLogger()
+    logger.setLevel(logging.INFO)
+    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+    log_file = xp_path + '/log.txt'
+    file_handler = logging.FileHandler(log_file)
+    file_handler.setLevel(logging.INFO)
+    file_handler.setFormatter(formatter)
+    logger.addHandler(file_handler)
+
+    # Print paths
+    logger.info('Log file is %s' % log_file)
+    logger.info('Data path is %s' % data_path)
+    logger.info('Export path is %s' % xp_path)
+
+    # Print experimental setup
+    logger.info('Dataset: %s' % dataset_name)
+    logger.info('Normal class: %d' % normal_class)
+    logger.info('Ratio of labeled normal train samples: %.2f' % ratio_known_normal)
+    logger.info('Ratio of labeled anomalous samples: %.2f' % ratio_known_outlier)
+    logger.info('Pollution ratio of unlabeled train data: %.2f' % ratio_pollution)
+    if n_known_outlier_classes == 1:
+        logger.info('Known anomaly class: %d' % known_outlier_class)
+    else:
+        logger.info('Number of known anomaly classes: %d' % n_known_outlier_classes)
+    logger.info('Network: %s' % net_name)
+
+    # If specified, load experiment config from JSON-file
+    if load_config:
+        cfg.load_config(import_json=load_config)
+        logger.info('Loaded configuration from %s.' % load_config)
+
+    # Set seed
+    if cfg.settings['seed'] != -1:
+        random.seed(cfg.settings['seed'])
+        np.random.seed(cfg.settings['seed'])
+        torch.manual_seed(cfg.settings['seed'])
+        torch.cuda.manual_seed(cfg.settings['seed'])
+        torch.backends.cudnn.deterministic = True
+        logger.info('Set seed to %d.' % cfg.settings['seed'])
+
+    # Default device to 'cpu' if cuda is not available
+    if not torch.cuda.is_available():
+        device = 'cpu'
+    # Set the number of threads used for parallelizing CPU operations
+    if num_threads > 0:
+        torch.set_num_threads(num_threads)
+    logger.info('Computation device: %s' % device)
+    logger.info('Number of threads: %d' % num_threads)
+    logger.info('Number of dataloader workers: %d' % n_jobs_dataloader)
+
+    # Load data
+    dataset = load_dataset(dataset_name, data_path, normal_class, known_outlier_class, n_known_outlier_classes,
+                           ratio_known_normal, ratio_known_outlier, ratio_pollution,
+                           random_state=np.random.RandomState(cfg.settings['seed']))
+    # Log random sample of known anomaly classes if more than 1 class
+    if n_known_outlier_classes > 1:
+        logger.info('Known anomaly classes: %s' % (dataset.known_outlier_classes,))
+
+    # Initialize semiDGM model and set neural network phi
+    alpha = 0.1 * (1 - ratio_known_normal - ratio_known_outlier) / (ratio_known_normal + ratio_known_outlier)
+    semiDGM = SemiDeepGenerativeModel(alpha=alpha)
+
+    # If specified, load model
+    if load_model:
+        # Initialize networks
+        semiDGM.set_vae(net_name)
+        semiDGM.set_network(net_name)
+        # Load model
+        semiDGM.load_model(model_path=load_model)
+        logger.info('Loading model from %s.' % load_model)
+
+    logger.info('Pretraining: %s' % pretrain)
+    if pretrain:
+        # Log pretraining details
+        logger.info('Pretraining optimizer: %s' % cfg.settings['vae_optimizer_name'])
+        logger.info('Pretraining learning rate: %g' % cfg.settings['vae_lr'])
+        logger.info('Pretraining epochs: %d' % cfg.settings['vae_n_epochs'])
+        logger.info('Pretraining learning rate scheduler milestones: %s' % (cfg.settings['vae_lr_milestone'],))
+        logger.info('Pretraining batch size: %d' % cfg.settings['vae_batch_size'])
+        logger.info('Pretraining weight decay: %g' % cfg.settings['vae_weight_decay'])
+
+        # Pretrain model on dataset (via variational autoencoder)
+        semiDGM.set_vae(net_name)
+        semiDGM.pretrain(dataset,
+                         optimizer_name=cfg.settings['vae_optimizer_name'],
+                         lr=cfg.settings['vae_lr'],
+                         n_epochs=cfg.settings['vae_n_epochs'],
+                         lr_milestones=cfg.settings['vae_lr_milestone'],
+                         batch_size=cfg.settings['vae_batch_size'],
+                         weight_decay=cfg.settings['vae_weight_decay'],
+                         device=device,
+                         n_jobs_dataloader=n_jobs_dataloader)
+
+        # Save pretraining results
+        semiDGM.save_vae_results(export_json=xp_path + '/vae_results.json')
+
+    # Log training details
+    logger.info('Training optimizer: %s' % cfg.settings['optimizer_name'])
+    logger.info('Training learning rate: %g' % cfg.settings['lr'])
+    logger.info('Training epochs: %d' % cfg.settings['n_epochs'])
+    logger.info('Training learning rate scheduler milestones: %s' % (cfg.settings['lr_milestone'],))
+    logger.info('Training batch size: %d' % cfg.settings['batch_size'])
+    logger.info('Training weight decay: %g' % cfg.settings['weight_decay'])
+
+    # Train model on dataset
+    semiDGM.set_network(net_name)
+    semiDGM.train(dataset,
+                  optimizer_name=cfg.settings['optimizer_name'],
+                  lr=cfg.settings['lr'],
+                  n_epochs=cfg.settings['n_epochs'],
+                  lr_milestones=cfg.settings['lr_milestone'],
+                  batch_size=cfg.settings['batch_size'],
+                  weight_decay=cfg.settings['weight_decay'],
+                  device=device,
+                  n_jobs_dataloader=n_jobs_dataloader)
+
+    # Test model
+    semiDGM.test(dataset, device=device, n_jobs_dataloader=n_jobs_dataloader)
+
+    # Save results, model, and configuration
+    semiDGM.save_results(export_json=xp_path + '/results.json')
+    semiDGM.save_model(export_model=xp_path + '/model.tar')
+    cfg.save_config(export_json=xp_path + '/config.json')
+
+    # Plot most anomalous and most normal test samples
+    indices, labels, scores = zip(*semiDGM.results['test_scores'])
+    indices, labels, scores = np.array(indices), np.array(labels), np.array(scores)
+    idx_all_sorted = indices[np.argsort(scores)]  # from lowest to highest score
+    idx_normal_sorted = indices[labels == 0][np.argsort(scores[labels == 0])]  # from lowest to highest score
+
+    if dataset_name in ('mnist', 'fmnist', 'cifar10'):
+
+        if dataset_name in ('mnist', 'fmnist'):
+            X_all_low = dataset.test_set.data[idx_all_sorted[:32], ...].unsqueeze(1)
+            X_all_high = dataset.test_set.data[idx_all_sorted[-32:], ...].unsqueeze(1)
+            X_normal_low = dataset.test_set.data[idx_normal_sorted[:32], ...].unsqueeze(1)
+            X_normal_high = dataset.test_set.data[idx_normal_sorted[-32:], ...].unsqueeze(1)
+
+        if dataset_name == 'cifar10':
+            X_all_low = torch.tensor(np.transpose(dataset.test_set.data[idx_all_sorted[:32], ...], (0,3,1,2)))
+            X_all_high = torch.tensor(np.transpose(dataset.test_set.data[idx_all_sorted[-32:], ...], (0,3,1,2)))
+            X_normal_low = torch.tensor(np.transpose(dataset.test_set.data[idx_normal_sorted[:32], ...], (0,3,1,2)))
+            X_normal_high = torch.tensor(np.transpose(dataset.test_set.data[idx_normal_sorted[-32:], ...], (0,3,1,2)))
+
+        plot_images_grid(X_all_low, export_img=xp_path + '/all_low', padding=2)
+        plot_images_grid(X_all_high, export_img=xp_path + '/all_high', padding=2)
+        plot_images_grid(X_normal_low, export_img=xp_path + '/normals_low', padding=2)
+        plot_images_grid(X_normal_high, export_img=xp_path + '/normals_high', padding=2)
+
+
+if __name__ == '__main__':
+    main()

Deep-SAD-PyTorch/src/baseline_isoforest.py

@@ -0,0 +1,183 @@
+import click
+import torch
+import logging
+import random
+import numpy as np
+
+from utils.config import Config
+from utils.visualization.plot_images_grid import plot_images_grid
+from baselines.isoforest import IsoForest
+from datasets.main import load_dataset
+
+
+################################################################################
+# Settings
+################################################################################
+@click.command()
+@click.argument('dataset_name', type=click.Choice(['mnist', 'fmnist', 'cifar10', 'arrhythmia', 'cardio', 'satellite',
+                                                   'satimage-2', 'shuttle', 'thyroid']))
+@click.argument('xp_path', type=click.Path(exists=True))
+@click.argument('data_path', type=click.Path(exists=True))
+@click.option('--load_config', type=click.Path(exists=True), default=None,
+              help='Config JSON-file path (default: None).')
+@click.option('--load_model', type=click.Path(exists=True), default=None,
+              help='Model file path (default: None).')
+@click.option('--ratio_known_normal', type=float, default=0.0,
+              help='Ratio of known (labeled) normal training examples.')
+@click.option('--ratio_known_outlier', type=float, default=0.0,
+              help='Ratio of known (labeled) anomalous training examples.')
+@click.option('--ratio_pollution', type=float, default=0.0,
+              help='Pollution ratio of unlabeled training data with unknown (unlabeled) anomalies.')
+@click.option('--seed', type=int, default=-1, help='Set seed. If -1, use randomization.')
+@click.option('--n_estimators', type=int, default=100,
+              help='Set the number of base estimators in the ensemble (default: 100).')
+@click.option('--max_samples', type=int, default=256,
+              help='Set the number of samples drawn to train each base estimator (default: 256).')
+@click.option('--contamination', type=float, default=0.1,
+              help='Expected fraction of anomalies in the training set. (default: 0.1).')
+@click.option('--n_jobs_model', type=int, default=-1, help='Number of jobs for model training.')
+@click.option('--hybrid', type=bool, default=False,
+              help='Train model on features extracted from an autoencoder. If True, load_ae must be specified.')
+@click.option('--load_ae', type=click.Path(exists=True), default=None,
+              help='Model file path to load autoencoder weights (default: None).')
+@click.option('--n_jobs_dataloader', type=int, default=0,
+              help='Number of workers for data loading. 0 means that the data will be loaded in the main process.')
+@click.option('--normal_class', type=int, default=0,
+              help='Specify the normal class of the dataset (all other classes are considered anomalous).')
+@click.option('--known_outlier_class', type=int, default=1,
+              help='Specify the known outlier class of the dataset for semi-supervised anomaly detection.')
+@click.option('--n_known_outlier_classes', type=int, default=0,
+              help='Number of known outlier classes.'
+                   'If 0, no anomalies are known.'
+                   'If 1, outlier class as specified in --known_outlier_class option.'
+                   'If > 1, the specified number of outlier classes will be sampled at random.')
+def main(dataset_name, xp_path, data_path, load_config, load_model, ratio_known_normal, ratio_known_outlier,
+         ratio_pollution, seed, n_estimators, max_samples, contamination, n_jobs_model, hybrid, load_ae,
+         n_jobs_dataloader, normal_class, known_outlier_class, n_known_outlier_classes):
+    """
+    (Hybrid) Isolation Forest model for anomaly detection.
+
+    :arg DATASET_NAME: Name of the dataset to load.
+    :arg XP_PATH: Export path for logging the experiment.
+    :arg DATA_PATH: Root path of data.
+    """
+
+    # Get configuration
+    cfg = Config(locals().copy())
+
+    # Set up logging
+    logging.basicConfig(level=logging.INFO)
+    logger = logging.getLogger()
+    logger.setLevel(logging.INFO)
+    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+    log_file = xp_path + '/log.txt'
+    file_handler = logging.FileHandler(log_file)
+    file_handler.setLevel(logging.INFO)
+    file_handler.setFormatter(formatter)
+    logger.addHandler(file_handler)
+
+    # Print paths
+    logger.info('Log file is %s.' % log_file)
+    logger.info('Data path is %s.' % data_path)
+    logger.info('Export path is %s.' % xp_path)
+
+    # Print experimental setup
+    logger.info('Dataset: %s' % dataset_name)
+    logger.info('Normal class: %d' % normal_class)
+    logger.info('Ratio of labeled normal train samples: %.2f' % ratio_known_normal)
+    logger.info('Ratio of labeled anomalous samples: %.2f' % ratio_known_outlier)
+    logger.info('Pollution ratio of unlabeled train data: %.2f' % ratio_pollution)
+    if n_known_outlier_classes == 1:
+        logger.info('Known anomaly class: %d' % known_outlier_class)
+    else:
+        logger.info('Number of known anomaly classes: %d' % n_known_outlier_classes)
+
+    # If specified, load experiment config from JSON-file
+    if load_config:
+        cfg.load_config(import_json=load_config)
+        logger.info('Loaded configuration from %s.' % load_config)
+
+    # Print Isolation Forest configuration
+    logger.info('Number of base estimators in the ensemble: %d' % cfg.settings['n_estimators'])
+    logger.info('Number of samples for training each base estimator: %d' % cfg.settings['max_samples'])
+    logger.info('Contamination parameter: %.2f' % cfg.settings['contamination'])
+    logger.info('Number of jobs for model training: %d' % n_jobs_model)
+    logger.info('Hybrid model: %s' % cfg.settings['hybrid'])
+
+    # Set seed
+    if cfg.settings['seed'] != -1:
+        random.seed(cfg.settings['seed'])
+        np.random.seed(cfg.settings['seed'])
+        torch.manual_seed(cfg.settings['seed'])
+        torch.cuda.manual_seed(cfg.settings['seed'])
+        torch.backends.cudnn.deterministic = True
+        logger.info('Set seed to %d.' % cfg.settings['seed'])
+
+    # Use 'cpu' as device for Isolation Forest
+    device = 'cpu'
+    torch.multiprocessing.set_sharing_strategy('file_system')  # fix multiprocessing issue for ubuntu
+    logger.info('Computation device: %s' % device)
+    logger.info('Number of dataloader workers: %d' % n_jobs_dataloader)
+
+    # Load data
+    dataset = load_dataset(dataset_name, data_path, normal_class, known_outlier_class, n_known_outlier_classes,
+                           ratio_known_normal, ratio_known_outlier, ratio_pollution,
+                           random_state=np.random.RandomState(cfg.settings['seed']))
+    # Log random sample of known anomaly classes if more than 1 class
+    if n_known_outlier_classes > 1:
+        logger.info('Known anomaly classes: %s' % (dataset.known_outlier_classes,))
+
+    # Initialize Isolation Forest model
+    Isoforest = IsoForest(hybrid=cfg.settings['hybrid'], n_estimators=cfg.settings['n_estimators'],
+                          max_samples=cfg.settings['max_samples'], contamination=cfg.settings['contamination'],
+                          n_jobs=n_jobs_model, seed=cfg.settings['seed'])
+
+    # If specified, load model parameters from already trained model
+    if load_model:
+        Isoforest.load_model(import_path=load_model, device=device)
+        logger.info('Loading model from %s.' % load_model)
+
+    # If specified, load model autoencoder weights for a hybrid approach
+    if hybrid and load_ae is not None:
+        Isoforest.load_ae(dataset_name, model_path=load_ae)
+        logger.info('Loaded pretrained autoencoder for features from %s.' % load_ae)
+
+    # Train model on dataset
+    Isoforest.train(dataset, device=device, n_jobs_dataloader=n_jobs_dataloader)
+
+    # Test model
+    Isoforest.test(dataset, device=device, n_jobs_dataloader=n_jobs_dataloader)
+
+    # Save results and configuration
+    Isoforest.save_results(export_json=xp_path + '/results.json')
+    cfg.save_config(export_json=xp_path + '/config.json')
+
+    # Plot most anomalous and most normal test samples
+    indices, labels, scores = zip(*Isoforest.results['test_scores'])
+    indices, labels, scores = np.array(indices), np.array(labels), np.array(scores)
+    idx_all_sorted = indices[np.argsort(scores)]  # from lowest to highest score
+    idx_normal_sorted = indices[labels == 0][np.argsort(scores[labels == 0])]  # from lowest to highest score
+
+    if dataset_name in ('mnist', 'fmnist', 'cifar10'):
+
+        if dataset_name in ('mnist', 'fmnist'):
+            X_all_low = dataset.test_set.data[idx_all_sorted[:32], ...].unsqueeze(1)
+            X_all_high = dataset.test_set.data[idx_all_sorted[-32:], ...].unsqueeze(1)
+            X_normal_low = dataset.test_set.data[idx_normal_sorted[:32], ...].unsqueeze(1)
+            X_normal_high = dataset.test_set.data[idx_normal_sorted[-32:], ...].unsqueeze(1)
+
+        if dataset_name == 'cifar10':
+            X_all_low = torch.tensor(np.transpose(dataset.test_set.data[idx_all_sorted[:32], ...], (0, 3, 1, 2)))
+            X_all_high = torch.tensor(np.transpose(dataset.test_set.data[idx_all_sorted[-32:], ...], (0, 3, 1, 2)))
+            X_normal_low = torch.tensor(np.transpose(dataset.test_set.data[idx_normal_sorted[:32], ...], (0, 3, 1, 2)))
+            X_normal_high = torch.tensor(
+                np.transpose(dataset.test_set.data[idx_normal_sorted[-32:], ...], (0, 3, 1, 2)))
+
+        plot_images_grid(X_all_low, export_img=xp_path + '/all_low', padding=2)
+        plot_images_grid(X_all_high, export_img=xp_path + '/all_high', padding=2)
+        plot_images_grid(X_normal_low, export_img=xp_path + '/normals_low', padding=2)
+        plot_images_grid(X_normal_high, export_img=xp_path + '/normals_high', padding=2)
+
+
+if __name__ == '__main__':
+    main()

Deep-SAD-PyTorch/src/baseline_kde.py

@@ -0,0 +1,180 @@
+import click
+import torch
+import logging
+import random
+import numpy as np
+
+from utils.config import Config
+from utils.visualization.plot_images_grid import plot_images_grid
+from baselines.kde import KDE
+from datasets.main import load_dataset
+
+
+################################################################################
+# Settings
+################################################################################
+@click.command()
+@click.argument('dataset_name', type=click.Choice(['mnist', 'fmnist', 'cifar10', 'arrhythmia', 'cardio', 'satellite',
+                                                   'satimage-2', 'shuttle', 'thyroid']))
+@click.argument('xp_path', type=click.Path(exists=True))
+@click.argument('data_path', type=click.Path(exists=True))
+@click.option('--load_config', type=click.Path(exists=True), default=None,
+              help='Config JSON-file path (default: None).')
+@click.option('--load_model', type=click.Path(exists=True), default=None,
+              help='Model file path (default: None).')
+@click.option('--ratio_known_normal', type=float, default=0.0,
+              help='Ratio of known (labeled) normal training examples.')
+@click.option('--ratio_known_outlier', type=float, default=0.0,
+              help='Ratio of known (labeled) anomalous training examples.')
+@click.option('--ratio_pollution', type=float, default=0.0,
+              help='Pollution ratio of unlabeled training data with unknown (unlabeled) anomalies.')
+@click.option('--seed', type=int, default=-1, help='Set seed. If -1, use randomization.')
+@click.option('--kernel', type=click.Choice(['gaussian', 'tophat', 'epanechnikov', 'exponential', 'linear', 'cosine']),
+              default='gaussian', help='Kernel for the KDE')
+@click.option('--grid_search_cv', type=bool, default=True,
+              help='Use sklearn GridSearchCV to determine optimal bandwidth')
+@click.option('--n_jobs_model', type=int, default=-1, help='Number of jobs for model training.')
+@click.option('--hybrid', type=bool, default=False,
+              help='Train KDE on features extracted from an autoencoder. If True, load_ae must be specified.')
+@click.option('--load_ae', type=click.Path(exists=True), default=None,
+              help='Model file path to load autoencoder weights (default: None).')
+@click.option('--n_jobs_dataloader', type=int, default=0,
+              help='Number of workers for data loading. 0 means that the data will be loaded in the main process.')
+@click.option('--normal_class', type=int, default=0,
+              help='Specify the normal class of the dataset (all other classes are considered anomalous).')
+@click.option('--known_outlier_class', type=int, default=1,
+              help='Specify the known outlier class of the dataset for semi-supervised anomaly detection.')
+@click.option('--n_known_outlier_classes', type=int, default=0,
+              help='Number of known outlier classes.'
+                   'If 0, no anomalies are known.'
+                   'If 1, outlier class as specified in --known_outlier_class option.'
+                   'If > 1, the specified number of outlier classes will be sampled at random.')
+def main(dataset_name, xp_path, data_path, load_config, load_model, ratio_known_normal, ratio_known_outlier,
+         ratio_pollution, seed, kernel, grid_search_cv, n_jobs_model, hybrid, load_ae, n_jobs_dataloader, normal_class,
+         known_outlier_class, n_known_outlier_classes):
+    """
+    (Hybrid) KDE for anomaly detection.
+
+    :arg DATASET_NAME: Name of the dataset to load.
+    :arg XP_PATH: Export path for logging the experiment.
+    :arg DATA_PATH: Root path of data.
+    """
+
+    # Get configuration
+    cfg = Config(locals().copy())
+
+    # Set up logging
+    logging.basicConfig(level=logging.INFO)
+    logger = logging.getLogger()
+    logger.setLevel(logging.INFO)
+    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+    log_file = xp_path + '/log.txt'
+    file_handler = logging.FileHandler(log_file)
+    file_handler.setLevel(logging.INFO)
+    file_handler.setFormatter(formatter)
+    logger.addHandler(file_handler)
+
+    # Print paths
+    logger.info('Log file is %s.' % log_file)
+    logger.info('Data path is %s.' % data_path)
+    logger.info('Export path is %s.' % xp_path)
+
+    # Print experimental setup
+    logger.info('Dataset: %s' % dataset_name)
+    logger.info('Normal class: %d' % normal_class)
+    logger.info('Ratio of labeled normal train samples: %.2f' % ratio_known_normal)
+    logger.info('Ratio of labeled anomalous samples: %.2f' % ratio_known_outlier)
+    logger.info('Pollution ratio of unlabeled train data: %.2f' % ratio_pollution)
+    if n_known_outlier_classes == 1:
+        logger.info('Known anomaly class: %d' % known_outlier_class)
+    else:
+        logger.info('Number of known anomaly classes: %d' % n_known_outlier_classes)
+
+    # If specified, load experiment config from JSON-file
+    if load_config:
+        cfg.load_config(import_json=load_config)
+        logger.info('Loaded configuration from %s.' % load_config)
+
+    # Print KDE configuration
+    logger.info('KDE kernel: %s' % cfg.settings['kernel'])
+    logger.info('Use GridSearchCV for bandwidth selection: %s' % cfg.settings['grid_search_cv'])
+    logger.info('Number of jobs for model training: %d' % n_jobs_model)
+    logger.info('Hybrid model: %s' % cfg.settings['hybrid'])
+
+    # Set seed
+    if cfg.settings['seed'] != -1:
+        random.seed(cfg.settings['seed'])
+        np.random.seed(cfg.settings['seed'])
+        torch.manual_seed(cfg.settings['seed'])
+        torch.cuda.manual_seed(cfg.settings['seed'])
+        torch.backends.cudnn.deterministic = True
+        logger.info('Set seed to %d.' % cfg.settings['seed'])
+
+    # Use 'cpu' as device for KDE
+    device = 'cpu'
+    torch.multiprocessing.set_sharing_strategy('file_system')  # fix multiprocessing issue for ubuntu
+    logger.info('Computation device: %s' % device)
+    logger.info('Number of dataloader workers: %d' % n_jobs_dataloader)
+
+    # Load data
+    dataset = load_dataset(dataset_name, data_path, normal_class, known_outlier_class, n_known_outlier_classes,
+                           ratio_known_normal, ratio_known_outlier, ratio_pollution,
+                           random_state=np.random.RandomState(cfg.settings['seed']))
+    # Log random sample of known anomaly classes if more than 1 class
+    if n_known_outlier_classes > 1:
+        logger.info('Known anomaly classes: %s' % (dataset.known_outlier_classes,))
+
+    # Initialize KDE model
+    kde = KDE(hybrid=cfg.settings['hybrid'], kernel=cfg.settings['kernel'], n_jobs=n_jobs_model,
+              seed=cfg.settings['seed'])
+
+    # If specified, load model parameters from already trained model
+    if load_model:
+        kde.load_model(import_path=load_model, device=device)
+        logger.info('Loading model from %s.' % load_model)
+
+    # If specified, load model autoencoder weights for a hybrid approach
+    if hybrid and load_ae is not None:
+        kde.load_ae(dataset_name, model_path=load_ae)
+        logger.info('Loaded pretrained autoencoder for features from %s.' % load_ae)
+
+    # Train model on dataset
+    kde.train(dataset, device=device, n_jobs_dataloader=n_jobs_dataloader,
+              bandwidth_GridSearchCV=cfg.settings['grid_search_cv'])
+
+    # Test model
+    kde.test(dataset, device=device, n_jobs_dataloader=n_jobs_dataloader)
+
+    # Save results and configuration
+    kde.save_results(export_json=xp_path + '/results.json')
+    cfg.save_config(export_json=xp_path + '/config.json')
+
+    # Plot most anomalous and most normal test samples
+    indices, labels, scores = zip(*kde.results['test_scores'])
+    indices, labels, scores = np.array(indices), np.array(labels), np.array(scores)
+    idx_all_sorted = indices[np.argsort(scores)]  # from lowest to highest score
+    idx_normal_sorted = indices[labels == 0][np.argsort(scores[labels == 0])]  # from lowest to highest score
+
+    if dataset_name in ('mnist', 'fmnist', 'cifar10'):
+
+        if dataset_name in ('mnist', 'fmnist'):
+            X_all_low = dataset.test_set.data[idx_all_sorted[:32], ...].unsqueeze(1)
+            X_all_high = dataset.test_set.data[idx_all_sorted[-32:], ...].unsqueeze(1)
+            X_normal_low = dataset.test_set.data[idx_normal_sorted[:32], ...].unsqueeze(1)
+            X_normal_high = dataset.test_set.data[idx_normal_sorted[-32:], ...].unsqueeze(1)
+
+        if dataset_name == 'cifar10':
+            X_all_low = torch.tensor(np.transpose(dataset.test_set.data[idx_all_sorted[:32], ...], (0, 3, 1, 2)))
+            X_all_high = torch.tensor(np.transpose(dataset.test_set.data[idx_all_sorted[-32:], ...], (0, 3, 1, 2)))
+            X_normal_low = torch.tensor(np.transpose(dataset.test_set.data[idx_normal_sorted[:32], ...], (0, 3, 1, 2)))
+            X_normal_high = torch.tensor(
+                np.transpose(dataset.test_set.data[idx_normal_sorted[-32:], ...], (0, 3, 1, 2)))
+
+        plot_images_grid(X_all_low, export_img=xp_path + '/all_low', padding=2)
+        plot_images_grid(X_all_high, export_img=xp_path + '/all_high', padding=2)
+        plot_images_grid(X_normal_low, export_img=xp_path + '/normals_low', padding=2)
+        plot_images_grid(X_normal_high, export_img=xp_path + '/normals_high', padding=2)
+
+
+if __name__ == '__main__':
+    main()

Deep-SAD-PyTorch/src/baseline_ocsvm.py

@@ -0,0 +1,174 @@
+import click
+import torch
+import logging
+import random
+import numpy as np
+
+from utils.config import Config
+from utils.visualization.plot_images_grid import plot_images_grid
+from baselines.ocsvm import OCSVM
+from datasets.main import load_dataset
+
+
+################################################################################
+# Settings
+################################################################################
+@click.command()
+@click.argument('dataset_name', type=click.Choice(['mnist', 'fmnist', 'cifar10', 'arrhythmia', 'cardio', 'satellite',
+                                                   'satimage-2', 'shuttle', 'thyroid']))
+@click.argument('xp_path', type=click.Path(exists=True))
+@click.argument('data_path', type=click.Path(exists=True))
+@click.option('--load_config', type=click.Path(exists=True), default=None,
+              help='Config JSON-file path (default: None).')
+@click.option('--load_model', type=click.Path(exists=True), default=None,
+              help='Model file path (default: None).')
+@click.option('--ratio_known_normal', type=float, default=0.0,
+              help='Ratio of known (labeled) normal training examples.')
+@click.option('--ratio_known_outlier', type=float, default=0.0,
+              help='Ratio of known (labeled) anomalous training examples.')
+@click.option('--ratio_pollution', type=float, default=0.0,
+              help='Pollution ratio of unlabeled training data with unknown (unlabeled) anomalies.')
+@click.option('--seed', type=int, default=-1, help='Set seed. If -1, use randomization.')
+@click.option('--kernel', type=click.Choice(['rbf', 'linear', 'poly']), default='rbf', help='Kernel for the OC-SVM')
+@click.option('--nu', type=float, default=0.1, help='OC-SVM hyperparameter nu (must be 0 < nu <= 1).')
+@click.option('--hybrid', type=bool, default=False,
+              help='Train OC-SVM on features extracted from an autoencoder. If True, load_ae must be specified.')
+@click.option('--load_ae', type=click.Path(exists=True), default=None,
+              help='Model file path to load autoencoder weights (default: None).')
+@click.option('--n_jobs_dataloader', type=int, default=0,
+              help='Number of workers for data loading. 0 means that the data will be loaded in the main process.')
+@click.option('--normal_class', type=int, default=0,
+              help='Specify the normal class of the dataset (all other classes are considered anomalous).')
+@click.option('--known_outlier_class', type=int, default=1,
+              help='Specify the known outlier class of the dataset for semi-supervised anomaly detection.')
+@click.option('--n_known_outlier_classes', type=int, default=0,
+              help='Number of known outlier classes.'
+                   'If 0, no anomalies are known.'
+                   'If 1, outlier class as specified in --known_outlier_class option.'
+                   'If > 1, the specified number of outlier classes will be sampled at random.')
+def main(dataset_name, xp_path, data_path, load_config, load_model, ratio_known_normal, ratio_known_outlier,
+         ratio_pollution, seed, kernel, nu, hybrid, load_ae, n_jobs_dataloader, normal_class, known_outlier_class,
+         n_known_outlier_classes):
+    """
+    (Hybrid) One-Class SVM for anomaly detection.
+
+    :arg DATASET_NAME: Name of the dataset to load.
+    :arg XP_PATH: Export path for logging the experiment.
+    :arg DATA_PATH: Root path of data.
+    """
+
+    # Get configuration
+    cfg = Config(locals().copy())
+
+    # Set up logging
+    logging.basicConfig(level=logging.INFO)
+    logger = logging.getLogger()
+    logger.setLevel(logging.INFO)
+    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+    log_file = xp_path + '/log.txt'
+    file_handler = logging.FileHandler(log_file)
+    file_handler.setLevel(logging.INFO)
+    file_handler.setFormatter(formatter)
+    logger.addHandler(file_handler)
+
+    # Print paths
+    logger.info('Log file is %s.' % log_file)
+    logger.info('Data path is %s.' % data_path)
+    logger.info('Export path is %s.' % xp_path)
+
+    # Print experimental setup
+    logger.info('Dataset: %s' % dataset_name)
+    logger.info('Normal class: %d' % normal_class)
+    logger.info('Ratio of labeled normal train samples: %.2f' % ratio_known_normal)
+    logger.info('Ratio of labeled anomalous samples: %.2f' % ratio_known_outlier)
+    logger.info('Pollution ratio of unlabeled train data: %.2f' % ratio_pollution)
+    if n_known_outlier_classes == 1:
+        logger.info('Known anomaly class: %d' % known_outlier_class)
+    else:
+        logger.info('Number of known anomaly classes: %d' % n_known_outlier_classes)
+
+    # If specified, load experiment config from JSON-file
+    if load_config:
+        cfg.load_config(import_json=load_config)
+        logger.info('Loaded configuration from %s.' % load_config)
+
+    # Print OC-SVM configuration
+    logger.info('OC-SVM kernel: %s' % cfg.settings['kernel'])
+    logger.info('Nu-paramerter: %.2f' % cfg.settings['nu'])
+    logger.info('Hybrid model: %s' % cfg.settings['hybrid'])
+
+    # Set seed
+    if cfg.settings['seed'] != -1:
+        random.seed(cfg.settings['seed'])
+        np.random.seed(cfg.settings['seed'])
+        torch.manual_seed(cfg.settings['seed'])
+        torch.cuda.manual_seed(cfg.settings['seed'])
+        torch.backends.cudnn.deterministic = True
+        logger.info('Set seed to %d.' % cfg.settings['seed'])
+
+    # Use 'cpu' as device for OC-SVM
+    device = 'cpu'
+    torch.multiprocessing.set_sharing_strategy('file_system')  # fix multiprocessing issue for ubuntu
+    logger.info('Computation device: %s' % device)
+    logger.info('Number of dataloader workers: %d' % n_jobs_dataloader)
+
+    # Load data
+    dataset = load_dataset(dataset_name, data_path, normal_class, known_outlier_class, n_known_outlier_classes,
+                           ratio_known_normal, ratio_known_outlier, ratio_pollution,
+                           random_state=np.random.RandomState(cfg.settings['seed']))
+    # Log random sample of known anomaly classes if more than 1 class
+    if n_known_outlier_classes > 1:
+        logger.info('Known anomaly classes: %s' % (dataset.known_outlier_classes,))
+
+    # Initialize OC-SVM model
+    ocsvm = OCSVM(cfg.settings['kernel'], cfg.settings['nu'], cfg.settings['hybrid'])
+
+    # If specified, load model parameters from already trained model
+    if load_model:
+        ocsvm.load_model(import_path=load_model, device=device)
+        logger.info('Loading model from %s.' % load_model)
+
+    # If specified, load model autoencoder weights for a hybrid approach
+    if hybrid and load_ae is not None:
+        ocsvm.load_ae(dataset_name, model_path=load_ae)
+        logger.info('Loaded pretrained autoencoder for features from %s.' % load_ae)
+
+    # Train model on dataset
+    ocsvm.train(dataset, device=device, n_jobs_dataloader=n_jobs_dataloader)
+
+    # Test model
+    ocsvm.test(dataset, device=device, n_jobs_dataloader=n_jobs_dataloader)
+
+    # Save results and configuration
+    ocsvm.save_results(export_json=xp_path + '/results.json')
+    cfg.save_config(export_json=xp_path + '/config.json')
+
+    # Plot most anomalous and most normal test samples
+    indices, labels, scores = zip(*ocsvm.results['test_scores'])
+    indices, labels, scores = np.array(indices), np.array(labels), np.array(scores)
+    idx_all_sorted = indices[np.argsort(scores)]  # from lowest to highest score
+    idx_normal_sorted = indices[labels == 0][np.argsort(scores[labels == 0])]  # from lowest to highest score
+
+    if dataset_name in ('mnist', 'fmnist', 'cifar10'):
+
+        if dataset_name in ('mnist', 'fmnist'):
+            X_all_low = dataset.test_set.data[idx_all_sorted[:32], ...].unsqueeze(1)
+            X_all_high = dataset.test_set.data[idx_all_sorted[-32:], ...].unsqueeze(1)
+            X_normal_low = dataset.test_set.data[idx_normal_sorted[:32], ...].unsqueeze(1)
+            X_normal_high = dataset.test_set.data[idx_normal_sorted[-32:], ...].unsqueeze(1)
+
+        if dataset_name == 'cifar10':
+            X_all_low = torch.tensor(np.transpose(dataset.test_set.data[idx_all_sorted[:32], ...], (0, 3, 1, 2)))
+            X_all_high = torch.tensor(np.transpose(dataset.test_set.data[idx_all_sorted[-32:], ...], (0, 3, 1, 2)))
+            X_normal_low = torch.tensor(np.transpose(dataset.test_set.data[idx_normal_sorted[:32], ...], (0, 3, 1, 2)))
+            X_normal_high = torch.tensor(
+                np.transpose(dataset.test_set.data[idx_normal_sorted[-32:], ...], (0, 3, 1, 2)))
+
+        plot_images_grid(X_all_low, export_img=xp_path + '/all_low', padding=2)
+        plot_images_grid(X_all_high, export_img=xp_path + '/all_high', padding=2)
+        plot_images_grid(X_normal_low, export_img=xp_path + '/normals_low', padding=2)
+        plot_images_grid(X_normal_high, export_img=xp_path + '/normals_high', padding=2)
+
+
+if __name__ == '__main__':
+    main()

Deep-SAD-PyTorch/src/baseline_ssad.py

@@ -0,0 +1,176 @@
+import click
+import torch
+import logging
+import random
+import numpy as np
+import cvxopt as co
+
+from utils.config import Config
+from utils.visualization.plot_images_grid import plot_images_grid
+from baselines.ssad import SSAD
+from datasets.main import load_dataset
+
+
+################################################################################
+# Settings
+################################################################################
+@click.command()
+@click.argument('dataset_name', type=click.Choice(['mnist', 'fmnist', 'cifar10', 'arrhythmia', 'cardio', 'satellite',
+                                                   'satimage-2', 'shuttle', 'thyroid']))
+@click.argument('xp_path', type=click.Path(exists=True))
+@click.argument('data_path', type=click.Path(exists=True))
+@click.option('--load_config', type=click.Path(exists=True), default=None,
+              help='Config JSON-file path (default: None).')
+@click.option('--load_model', type=click.Path(exists=True), default=None,
+              help='Model file path (default: None).')
+@click.option('--ratio_known_normal', type=float, default=0.0,
+              help='Ratio of known (labeled) normal training examples.')
+@click.option('--ratio_known_outlier', type=float, default=0.0,
+              help='Ratio of known (labeled) anomalous training examples.')
+@click.option('--ratio_pollution', type=float, default=0.0,
+              help='Pollution ratio of unlabeled training data with unknown (unlabeled) anomalies.')
+@click.option('--seed', type=int, default=-1, help='Set seed. If -1, use randomization.')
+@click.option('--kernel', type=click.Choice(['rbf']), default='rbf', help='Kernel for SSAD')
+@click.option('--kappa', type=float, default=1.0, help='SSAD hyperparameter kappa.')
+@click.option('--hybrid', type=bool, default=False,
+              help='Train SSAD on features extracted from an autoencoder. If True, load_ae must be specified')
+@click.option('--load_ae', type=click.Path(exists=True), default=None,
+              help='Model file path to load autoencoder weights (default: None).')
+@click.option('--n_jobs_dataloader', type=int, default=0,
+              help='Number of workers for data loading. 0 means that the data will be loaded in the main process.')
+@click.option('--normal_class', type=int, default=0,
+              help='Specify the normal class of the dataset (all other classes are considered anomalous).')
+@click.option('--known_outlier_class', type=int, default=1,
+              help='Specify the known outlier class of the dataset for semi-supervised anomaly detection.')
+@click.option('--n_known_outlier_classes', type=int, default=0,
+              help='Number of known outlier classes.'
+                   'If 0, no anomalies are known.'
+                   'If 1, outlier class as specified in --known_outlier_class option.'
+                   'If > 1, the specified number of outlier classes will be sampled at random.')
+def main(dataset_name, xp_path, data_path, load_config, load_model, ratio_known_normal, ratio_known_outlier,
+         ratio_pollution, seed, kernel, kappa, hybrid, load_ae, n_jobs_dataloader, normal_class, known_outlier_class,
+         n_known_outlier_classes):
+    """
+    (Hybrid) SSAD for anomaly detection as in Goernitz et al., Towards Supervised Anomaly Detection, JAIR, 2013.
+
+    :arg DATASET_NAME: Name of the dataset to load.
+    :arg XP_PATH: Export path for logging the experiment.
+    :arg DATA_PATH: Root path of data.
+    """
+
+    # Get configuration
+    cfg = Config(locals().copy())
+
+    # Set up logging
+    logging.basicConfig(level=logging.INFO)
+    logger = logging.getLogger()
+    logger.setLevel(logging.INFO)
+    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+    log_file = xp_path + '/log.txt'
+    file_handler = logging.FileHandler(log_file)
+    file_handler.setLevel(logging.INFO)
+    file_handler.setFormatter(formatter)
+    logger.addHandler(file_handler)
+
+    # Print paths
+    logger.info('Log file is %s.' % log_file)
+    logger.info('Data path is %s.' % data_path)
+    logger.info('Export path is %s.' % xp_path)
+
+    # Print experimental setup
+    logger.info('Dataset: %s' % dataset_name)
+    logger.info('Normal class: %d' % normal_class)
+    logger.info('Ratio of labeled normal train samples: %.2f' % ratio_known_normal)
+    logger.info('Ratio of labeled anomalous samples: %.2f' % ratio_known_outlier)
+    logger.info('Pollution ratio of unlabeled train data: %.2f' % ratio_pollution)
+    if n_known_outlier_classes == 1:
+        logger.info('Known anomaly class: %d' % known_outlier_class)
+    else:
+        logger.info('Number of known anomaly classes: %d' % n_known_outlier_classes)
+
+    # If specified, load experiment config from JSON-file
+    if load_config:
+        cfg.load_config(import_json=load_config)
+        logger.info('Loaded configuration from %s.' % load_config)
+
+    # Print SSAD configuration
+    logger.info('SSAD kernel: %s' % cfg.settings['kernel'])
+    logger.info('Kappa-paramerter: %.2f' % cfg.settings['kappa'])
+    logger.info('Hybrid model: %s' % cfg.settings['hybrid'])
+
+    # Set seed
+    if cfg.settings['seed'] != -1:
+        random.seed(cfg.settings['seed'])
+        np.random.seed(cfg.settings['seed'])
+        co.setseed(cfg.settings['seed'])
+        torch.manual_seed(cfg.settings['seed'])
+        torch.cuda.manual_seed(cfg.settings['seed'])
+        torch.backends.cudnn.deterministic = True
+        logger.info('Set seed to %d.' % cfg.settings['seed'])
+
+    # Use 'cpu' as device for SSAD
+    device = 'cpu'
+    torch.multiprocessing.set_sharing_strategy('file_system')  # fix multiprocessing issue for ubuntu
+    logger.info('Computation device: %s' % device)
+    logger.info('Number of dataloader workers: %d' % n_jobs_dataloader)
+
+    # Load data
+    dataset = load_dataset(dataset_name, data_path, normal_class, known_outlier_class, n_known_outlier_classes,
+                           ratio_known_normal, ratio_known_outlier, ratio_pollution,
+                           random_state=np.random.RandomState(cfg.settings['seed']))
+    # Log random sample of known anomaly classes if more than 1 class
+    if n_known_outlier_classes > 1:
+        logger.info('Known anomaly classes: %s' % (dataset.known_outlier_classes,))
+
+    # Initialize SSAD model
+    ssad = SSAD(kernel=cfg.settings['kernel'], kappa=cfg.settings['kappa'], hybrid=cfg.settings['hybrid'])
+
+    # If specified, load model parameters from already trained model
+    if load_model:
+        ssad.load_model(import_path=load_model, device=device)
+        logger.info('Loading model from %s.' % load_model)
+
+    # If specified, load model autoencoder weights for a hybrid approach
+    if hybrid and load_ae is not None:
+        ssad.load_ae(dataset_name, model_path=load_ae)
+        logger.info('Loaded pretrained autoencoder for features from %s.' % load_ae)
+
+    # Train model on dataset
+    ssad.train(dataset, device=device, n_jobs_dataloader=n_jobs_dataloader)
+
+    # Test model
+    ssad.test(dataset, device=device, n_jobs_dataloader=n_jobs_dataloader)
+
+    # Save results and configuration
+    ssad.save_results(export_json=xp_path + '/results.json')
+    cfg.save_config(export_json=xp_path + '/config.json')
+
+    # Plot most anomalous and most normal test samples
+    indices, labels, scores = zip(*ssad.results['test_scores'])
+    indices, labels, scores = np.array(indices), np.array(labels), np.array(scores)
+    idx_all_sorted = indices[np.argsort(scores)]  # from lowest to highest score
+    idx_normal_sorted = indices[labels == 0][np.argsort(scores[labels == 0])]  # from lowest to highest score
+
+    if dataset_name in ('mnist', 'fmnist', 'cifar10'):
+
+        if dataset_name in ('mnist', 'fmnist'):
+            X_all_low = dataset.test_set.data[idx_all_sorted[:32], ...].unsqueeze(1)
+            X_all_high = dataset.test_set.data[idx_all_sorted[-32:], ...].unsqueeze(1)
+            X_normal_low = dataset.test_set.data[idx_normal_sorted[:32], ...].unsqueeze(1)
+            X_normal_high = dataset.test_set.data[idx_normal_sorted[-32:], ...].unsqueeze(1)
+
+        if dataset_name == 'cifar10':
+            X_all_low = torch.tensor(np.transpose(dataset.test_set.data[idx_all_sorted[:32], ...], (0, 3, 1, 2)))
+            X_all_high = torch.tensor(np.transpose(dataset.test_set.data[idx_all_sorted[-32:], ...], (0, 3, 1, 2)))
+            X_normal_low = torch.tensor(np.transpose(dataset.test_set.data[idx_normal_sorted[:32], ...], (0, 3, 1, 2)))
+            X_normal_high = torch.tensor(
+                np.transpose(dataset.test_set.data[idx_normal_sorted[-32:], ...], (0, 3, 1, 2)))
+
+        plot_images_grid(X_all_low, export_img=xp_path + '/all_low', padding=2)
+        plot_images_grid(X_all_high, export_img=xp_path + '/all_high', padding=2)
+        plot_images_grid(X_normal_low, export_img=xp_path + '/normals_low', padding=2)
+        plot_images_grid(X_normal_high, export_img=xp_path + '/normals_high', padding=2)
+
+
+if __name__ == '__main__':
+    main()

Deep-SAD-PyTorch/src/baselines/SemiDGM.py

@@ -0,0 +1,128 @@
+import json
+import torch
+
+from base.base_dataset import BaseADDataset
+from networks.main import build_network, build_autoencoder
+from optim import SemiDeepGenerativeTrainer, VAETrainer
+
+
+class SemiDeepGenerativeModel(object):
+    """A class for the Semi-Supervised Deep Generative model (M1+M2 model).
+
+    Paper: Kingma et al. (2014). Semi-supervised learning with deep generative models. In NIPS (pp. 3581-3589).
+    Link: https://papers.nips.cc/paper/5352-semi-supervised-learning-with-deep-generative-models.pdf
+
+    Attributes:
+        net_name: A string indicating the name of the neural network to use.
+        net: The neural network.
+        trainer: SemiDeepGenerativeTrainer to train a Semi-Supervised Deep Generative model.
+        optimizer_name: A string indicating the optimizer to use for training.
+        results: A dictionary to save the results.
+    """
+
+    def __init__(self, alpha: float = 0.1):
+        """Inits SemiDeepGenerativeModel."""
+
+        self.alpha = alpha
+
+        self.net_name = None
+        self.net = None
+
+        self.trainer = None
+        self.optimizer_name = None
+
+        self.vae_net = None  # variational autoencoder network for pretraining
+        self.vae_trainer = None
+        self.vae_optimizer_name = None
+
+        self.results = {
+            'train_time': None,
+            'test_auc': None,
+            'test_time': None,
+            'test_scores': None,
+        }
+
+        self.vae_results = {
+            'train_time': None,
+            'test_auc': None,
+            'test_time': None
+        }
+
+    def set_vae(self, net_name):
+        """Builds the variational autoencoder network for pretraining."""
+        self.net_name = net_name
+        self.vae_net = build_autoencoder(self.net_name)  # VAE for pretraining
+
+    def set_network(self, net_name):
+        """Builds the neural network."""
+        self.net_name = net_name
+        self.net = build_network(net_name, ae_net=self.vae_net)  # full M1+M2 model
+
+    def train(self, dataset: BaseADDataset, optimizer_name: str = 'adam', lr: float = 0.001, n_epochs: int = 50,
+              lr_milestones: tuple = (), batch_size: int = 128, weight_decay: float = 1e-6, device: str = 'cuda',
+              n_jobs_dataloader: int = 0):
+        """Trains the Semi-Supervised Deep Generative model on the training data."""
+
+        self.optimizer_name = optimizer_name
+
+        self.trainer = SemiDeepGenerativeTrainer(alpha=self.alpha, optimizer_name=optimizer_name, lr=lr,
+                                                 n_epochs=n_epochs, lr_milestones=lr_milestones, batch_size=batch_size,
+                                                 weight_decay=weight_decay, device=device,
+                                                 n_jobs_dataloader=n_jobs_dataloader)
+        self.net = self.trainer.train(dataset, self.net)
+        self.results['train_time'] = self.trainer.train_time
+
+    def test(self, dataset: BaseADDataset, device: str = 'cuda', n_jobs_dataloader: int = 0):
+        """Tests the Semi-Supervised Deep Generative model on the test data."""
+
+        if self.trainer is None:
+            self.trainer = SemiDeepGenerativeTrainer(alpha=self.alpha, device=device,
+                                                     n_jobs_dataloader=n_jobs_dataloader)
+
+        self.trainer.test(dataset, self.net)
+        # Get results
+        self.results['test_auc'] = self.trainer.test_auc
+        self.results['test_time'] = self.trainer.test_time
+        self.results['test_scores'] = self.trainer.test_scores
+
+    def pretrain(self, dataset: BaseADDataset, optimizer_name: str = 'adam', lr: float = 0.001, n_epochs: int = 100,
+                 lr_milestones: tuple = (), batch_size: int = 128, weight_decay: float = 1e-6, device: str = 'cuda',
+                 n_jobs_dataloader: int = 0):
+        """Pretrains a variational autoencoder (M1) for the Semi-Supervised Deep Generative model."""
+
+        # Train
+        self.vae_optimizer_name = optimizer_name
+        self.vae_trainer = VAETrainer(optimizer_name=optimizer_name, lr=lr, n_epochs=n_epochs,
+                                      lr_milestones=lr_milestones, batch_size=batch_size, weight_decay=weight_decay,
+                                      device=device, n_jobs_dataloader=n_jobs_dataloader)
+        self.vae_net = self.vae_trainer.train(dataset, self.vae_net)
+        # Get train results
+        self.vae_results['train_time'] = self.vae_trainer.train_time
+
+        # Test
+        self.vae_trainer.test(dataset, self.vae_net)
+        # Get test results
+        self.vae_results['test_auc'] = self.vae_trainer.test_auc
+        self.vae_results['test_time'] = self.vae_trainer.test_time
+
+    def save_model(self, export_model):
+        """Save a Semi-Supervised Deep Generative model to export_model."""
+
+        net_dict = self.net.state_dict()
+        torch.save({'net_dict': net_dict}, export_model)
+
+    def load_model(self, model_path):
+        """Load a Semi-Supervised Deep Generative model from model_path."""
+
+        model_dict = torch.load(model_path)
+        self.net.load_state_dict(model_dict['net_dict'])
+
+    def save_results(self, export_json):
+        """Save results dict to a JSON-file."""
+        with open(export_json, 'w') as fp:
+            json.dump(self.results, fp)
+
+    def save_vae_results(self, export_json):
+        """Save variational autoencoder results dict to a JSON-file."""
+        with open(export_json, 'w') as fp:
+            json.dump(self.vae_results, fp)

Deep-SAD-PyTorch/src/baselines/__init__.py

@@ -0,0 +1,6 @@
+from .SemiDGM import SemiDeepGenerativeModel
+from .ocsvm import OCSVM
+from .kde import KDE
+from .isoforest import IsoForest
+from .ssad import SSAD
+from .shallow_ssad.ssad_convex import ConvexSSAD

Deep-SAD-PyTorch/src/baselines/isoforest.py

@@ -0,0 +1,147 @@
+import json
+import logging
+import time
+import torch
+import numpy as np
+
+from torch.utils.data import DataLoader
+from sklearn.ensemble import IsolationForest
+from sklearn.metrics import roc_auc_score
+from base.base_dataset import BaseADDataset
+from networks.main import build_autoencoder
+
+
+class IsoForest(object):
+    """A class for Isolation Forest models."""
+
+    def __init__(self, hybrid=False, n_estimators=100, max_samples='auto', contamination=0.1, n_jobs=-1, seed=None,
+                 **kwargs):
+        """Init Isolation Forest instance."""
+        self.n_estimators = n_estimators
+        self.max_samples = max_samples
+        self.contamination = contamination
+        self.n_jobs = n_jobs
+        self.seed = seed
+
+        self.model = IsolationForest(n_estimators=n_estimators, max_samples=max_samples, contamination=contamination,
+                                     n_jobs=n_jobs, random_state=seed, **kwargs)
+
+        self.hybrid = hybrid
+        self.ae_net = None  # autoencoder network for the case of a hybrid model
+
+        self.results = {
+            'train_time': None,
+            'test_time': None,
+            'test_auc': None,
+            'test_scores': None
+        }
+
+    def train(self, dataset: BaseADDataset, device: str = 'cpu', n_jobs_dataloader: int = 0):
+        """Trains the Isolation Forest model on the training data."""
+        logger = logging.getLogger()
+
+        # do not drop last batch for non-SGD optimization shallow_ssad
+        train_loader = DataLoader(dataset=dataset.train_set, batch_size=128, shuffle=True,
+                                  num_workers=n_jobs_dataloader, drop_last=False)
+
+        # Get data from loader
+        X = ()
+        for data in train_loader:
+            inputs, _, _, _ = data
+            inputs = inputs.to(device)
+            if self.hybrid:
+                inputs = self.ae_net.encoder(inputs)  # in hybrid approach, take code representation of AE as features
+            X_batch = inputs.view(inputs.size(0), -1)  # X_batch.shape = (batch_size, n_channels * height * width)
+            X += (X_batch.cpu().data.numpy(),)
+        X = np.concatenate(X)
+
+        # Training
+        logger.info('Starting training...')
+        start_time = time.time()
+        self.model.fit(X)
+        train_time = time.time() - start_time
+        self.results['train_time'] = train_time
+
+        logger.info('Training Time: {:.3f}s'.format(self.results['train_time']))
+        logger.info('Finished training.')
+
+    def test(self, dataset: BaseADDataset, device: str = 'cpu', n_jobs_dataloader: int = 0):
+        """Tests the Isolation Forest model on the test data."""
+        logger = logging.getLogger()
+
+        _, test_loader = dataset.loaders(batch_size=128, num_workers=n_jobs_dataloader)
+
+        # Get data from loader
+        idx_label_score = []
+        X = ()
+        idxs = []
+        labels = []
+        for data in test_loader:
+            inputs, label_batch, _, idx = data
+            inputs, label_batch, idx = inputs.to(device), label_batch.to(device), idx.to(device)
+            if self.hybrid:
+                inputs = self.ae_net.encoder(inputs)  # in hybrid approach, take code representation of AE as features
+            X_batch = inputs.view(inputs.size(0), -1)  # X_batch.shape = (batch_size, n_channels * height * width)
+            X += (X_batch.cpu().data.numpy(),)
+            idxs += idx.cpu().data.numpy().astype(np.int64).tolist()
+            labels += label_batch.cpu().data.numpy().astype(np.int64).tolist()
+        X = np.concatenate(X)
+
+        # Testing
+        logger.info('Starting testing...')
+        start_time = time.time()
+        scores = (-1.0) * self.model.decision_function(X)
+        self.results['test_time'] = time.time() - start_time
+        scores = scores.flatten()
+
+        # Save triples of (idx, label, score) in a list
+        idx_label_score += list(zip(idxs, labels, scores.tolist()))
+        self.results['test_scores'] = idx_label_score
+
+        # Compute AUC
+        _, labels, scores = zip(*idx_label_score)
+        labels = np.array(labels)
+        scores = np.array(scores)
+        self.results['test_auc'] = roc_auc_score(labels, scores)
+
+        # Log results
+        logger.info('Test AUC: {:.2f}%'.format(100. * self.results['test_auc']))
+        logger.info('Test Time: {:.3f}s'.format(self.results['test_time']))
+        logger.info('Finished testing.')
+
+    def load_ae(self, dataset_name, model_path):
+        """Load pretrained autoencoder from model_path for feature extraction in a hybrid Isolation Forest model."""
+
+        model_dict = torch.load(model_path, map_location='cpu')
+        ae_net_dict = model_dict['ae_net_dict']
+        if dataset_name in ['mnist', 'fmnist', 'cifar10']:
+            net_name = dataset_name + '_LeNet'
+        else:
+            net_name = dataset_name + '_mlp'
+
+        if self.ae_net is None:
+            self.ae_net = build_autoencoder(net_name)
+
+        # update keys (since there was a change in network definition)
+        ae_keys = list(self.ae_net.state_dict().keys())
+        for i in range(len(ae_net_dict)):
+            k, v = ae_net_dict.popitem(False)
+            new_key = ae_keys[i]
+            ae_net_dict[new_key] = v
+            i += 1
+
+        self.ae_net.load_state_dict(ae_net_dict)
+        self.ae_net.eval()
+
+    def save_model(self, export_path):
+        """Save Isolation Forest model to export_path."""
+        pass
+
+    def load_model(self, import_path, device: str = 'cpu'):
+        """Load Isolation Forest model from import_path."""
+        pass
+
+    def save_results(self, export_json):
+        """Save results dict to a JSON-file."""
+        with open(export_json, 'w') as fp:
+            json.dump(self.results, fp)

Deep-SAD-PyTorch/src/baselines/kde.py

@@ -0,0 +1,164 @@
+import json
+import logging
+import time
+import torch
+import numpy as np
+
+from torch.utils.data import DataLoader
+from sklearn.neighbors import KernelDensity
+from sklearn.metrics import roc_auc_score
+from sklearn.metrics.pairwise import pairwise_distances
+from sklearn.model_selection import GridSearchCV
+from base.base_dataset import BaseADDataset
+from networks.main import build_autoencoder
+
+
+class KDE(object):
+    """A class for Kernel Density Estimation models."""
+
+    def __init__(self, hybrid=False, kernel='gaussian', n_jobs=-1, seed=None, **kwargs):
+        """Init Kernel Density Estimation instance."""
+        self.kernel = kernel
+        self.n_jobs = n_jobs
+        self.seed = seed
+
+        self.model = KernelDensity(kernel=kernel, **kwargs)
+        self.bandwidth = self.model.bandwidth
+
+        self.hybrid = hybrid
+        self.ae_net = None  # autoencoder network for the case of a hybrid model
+
+        self.results = {
+            'train_time': None,
+            'test_time': None,
+            'test_auc': None,
+            'test_scores': None
+        }
+
+    def train(self, dataset: BaseADDataset, device: str = 'cpu', n_jobs_dataloader: int = 0,
+              bandwidth_GridSearchCV: bool = True):
+        """Trains the Kernel Density Estimation model on the training data."""
+        logger = logging.getLogger()
+
+        # do not drop last batch for non-SGD optimization shallow_ssad
+        train_loader = DataLoader(dataset=dataset.train_set, batch_size=128, shuffle=True,
+                                  num_workers=n_jobs_dataloader, drop_last=False)
+
+        # Get data from loader
+        X = ()
+        for data in train_loader:
+            inputs, _, _, _ = data
+            inputs = inputs.to(device)
+            if self.hybrid:
+                inputs = self.ae_net.encoder(inputs)  # in hybrid approach, take code representation of AE as features
+            X_batch = inputs.view(inputs.size(0), -1)  # X_batch.shape = (batch_size, n_channels * height * width)
+            X += (X_batch.cpu().data.numpy(),)
+        X = np.concatenate(X)
+
+        # Training
+        logger.info('Starting training...')
+        start_time = time.time()
+
+        if bandwidth_GridSearchCV:
+            # use grid search cross-validation to select bandwidth
+            logger.info('Using GridSearchCV for bandwidth selection...')
+            params = {'bandwidth': np.logspace(0.5, 5, num=10, base=2)}
+            hyper_kde = GridSearchCV(KernelDensity(kernel=self.kernel), params, n_jobs=self.n_jobs, cv=5, verbose=0)
+            hyper_kde.fit(X)
+            self.bandwidth = hyper_kde.best_estimator_.bandwidth
+            logger.info('Best bandwidth: {:.8f}'.format(self.bandwidth))
+            self.model = hyper_kde.best_estimator_
+        else:
+            # if exponential kernel, re-initialize kde with bandwidth minimizing the numerical error
+            if self.kernel == 'exponential':
+                self.bandwidth = np.max(pairwise_distances(X)) ** 2
+                self.model = KernelDensity(kernel=self.kernel, bandwidth=self.bandwidth)
+
+            self.model.fit(X)
+
+        train_time = time.time() - start_time
+        self.results['train_time'] = train_time
+
+        logger.info('Training Time: {:.3f}s'.format(self.results['train_time']))
+        logger.info('Finished training.')
+
+    def test(self, dataset: BaseADDataset, device: str = 'cpu', n_jobs_dataloader: int = 0):
+        """Tests the Kernel Density Estimation model on the test data."""
+        logger = logging.getLogger()
+
+        _, test_loader = dataset.loaders(batch_size=128, num_workers=n_jobs_dataloader)
+
+        # Get data from loader
+        idx_label_score = []
+        X = ()
+        idxs = []
+        labels = []
+        for data in test_loader:
+            inputs, label_batch, _, idx = data
+            inputs, label_batch, idx = inputs.to(device), label_batch.to(device), idx.to(device)
+            if self.hybrid:
+                inputs = self.ae_net.encoder(inputs)  # in hybrid approach, take code representation of AE as features
+            X_batch = inputs.view(inputs.size(0), -1)  # X_batch.shape = (batch_size, n_channels * height * width)
+            X += (X_batch.cpu().data.numpy(),)
+            idxs += idx.cpu().data.numpy().astype(np.int64).tolist()
+            labels += label_batch.cpu().data.numpy().astype(np.int64).tolist()
+        X = np.concatenate(X)
+
+        # Testing
+        logger.info('Starting testing...')
+        start_time = time.time()
+        scores = (-1.0) * self.model.score_samples(X)
+        self.results['test_time'] = time.time() - start_time
+        scores = scores.flatten()
+
+        # Save triples of (idx, label, score) in a list
+        idx_label_score += list(zip(idxs, labels, scores.tolist()))
+        self.results['test_scores'] = idx_label_score
+
+        # Compute AUC
+        _, labels, scores = zip(*idx_label_score)
+        labels = np.array(labels)
+        scores = np.array(scores)
+        self.results['test_auc'] = roc_auc_score(labels, scores)
+
+        # Log results
+        logger.info('Test AUC: {:.2f}%'.format(100. * self.results['test_auc']))
+        logger.info('Test Time: {:.3f}s'.format(self.results['test_time']))
+        logger.info('Finished testing.')
+
+    def load_ae(self, dataset_name, model_path):
+        """Load pretrained autoencoder from model_path for feature extraction in a hybrid KDE model."""
+
+        model_dict = torch.load(model_path, map_location='cpu')
+        ae_net_dict = model_dict['ae_net_dict']
+        if dataset_name in ['mnist', 'fmnist', 'cifar10']:
+            net_name = dataset_name + '_LeNet'
+        else:
+            net_name = dataset_name + '_mlp'
+
+        if self.ae_net is None:
+            self.ae_net = build_autoencoder(net_name)
+
+        # update keys (since there was a change in network definition)
+        ae_keys = list(self.ae_net.state_dict().keys())
+        for i in range(len(ae_net_dict)):
+            k, v = ae_net_dict.popitem(False)
+            new_key = ae_keys[i]
+            ae_net_dict[new_key] = v
+            i += 1
+
+        self.ae_net.load_state_dict(ae_net_dict)
+        self.ae_net.eval()
+
+    def save_model(self, export_path):
+        """Save KDE model to export_path."""
+        pass
+
+    def load_model(self, import_path, device: str = 'cpu'):
+        """Load KDE model from import_path."""
+        pass
+
+    def save_results(self, export_json):
+        """Save results dict to a JSON-file."""
+        with open(export_json, 'w') as fp:
+            json.dump(self.results, fp)

Deep-SAD-PyTorch/src/baselines/ocsvm.py

@@ -0,0 +1,221 @@
+import json
+import logging
+import time
+import torch
+import numpy as np
+
+from torch.utils.data import DataLoader
+from sklearn.svm import OneClassSVM
+from sklearn.metrics import roc_auc_score
+from base.base_dataset import BaseADDataset
+from networks.main import build_autoencoder
+
+
+class OCSVM(object):
+    """A class for One-Class SVM models."""
+
+    def __init__(self, kernel='rbf', nu=0.1, hybrid=False):
+        """Init OCSVM instance."""
+        self.kernel = kernel
+        self.nu = nu
+        self.rho = None
+        self.gamma = None
+
+        self.model = OneClassSVM(kernel=kernel, nu=nu)
+
+        self.hybrid = hybrid
+        self.ae_net = None  # autoencoder network for the case of a hybrid model
+        self.linear_model = None  # also init a model with linear kernel if hybrid approach
+
+        self.results = {
+            'train_time': None,
+            'test_time': None,
+            'test_auc': None,
+            'test_scores': None,
+            'train_time_linear': None,
+            'test_time_linear': None,
+            'test_auc_linear': None
+        }
+
+    def train(self, dataset: BaseADDataset, device: str = 'cpu', n_jobs_dataloader: int = 0):
+        """Trains the OC-SVM model on the training data."""
+        logger = logging.getLogger()
+
+        # do not drop last batch for non-SGD optimization shallow_ssad
+        train_loader = DataLoader(dataset=dataset.train_set, batch_size=128, shuffle=True,
+                                  num_workers=n_jobs_dataloader, drop_last=False)
+
+        # Get data from loader
+        X = ()
+        for data in train_loader:
+            inputs, _, _, _ = data
+            inputs = inputs.to(device)
+            if self.hybrid:
+                inputs = self.ae_net.encoder(inputs)  # in hybrid approach, take code representation of AE as features
+            X_batch = inputs.view(inputs.size(0), -1)  # X_batch.shape = (batch_size, n_channels * height * width)
+            X += (X_batch.cpu().data.numpy(),)
+        X = np.concatenate(X)
+
+        # Training
+        logger.info('Starting training...')
+
+        # Select model via hold-out test set of 1000 samples
+        gammas = np.logspace(-7, 2, num=10, base=2)
+        best_auc = 0.0
+
+        # Sample hold-out set from test set
+        _, test_loader = dataset.loaders(batch_size=128, num_workers=n_jobs_dataloader)
+
+        X_test = ()
+        labels = []
+        for data in test_loader:
+            inputs, label_batch, _, _ = data
+            inputs, label_batch = inputs.to(device), label_batch.to(device)
+            if self.hybrid:
+                inputs = self.ae_net.encoder(inputs)  # in hybrid approach, take code representation of AE as features
+            X_batch = inputs.view(inputs.size(0), -1)  # X_batch.shape = (batch_size, n_channels * height * width)
+            X_test += (X_batch.cpu().data.numpy(),)
+            labels += label_batch.cpu().data.numpy().astype(np.int64).tolist()
+        X_test, labels = np.concatenate(X_test), np.array(labels)
+        n_test, n_normal, n_outlier = len(X_test), np.sum(labels == 0), np.sum(labels == 1)
+        n_val = int(0.1 * n_test)
+        n_val_normal, n_val_outlier = int(n_val * (n_normal/n_test)), int(n_val * (n_outlier/n_test))
+        perm = np.random.permutation(n_test)
+        X_val = np.concatenate((X_test[perm][labels[perm] == 0][:n_val_normal],
+                                X_test[perm][labels[perm] == 1][:n_val_outlier]))
+        labels = np.array([0] * n_val_normal + [1] * n_val_outlier)
+
+        i = 1
+        for gamma in gammas:
+
+            # Model candidate
+            model = OneClassSVM(kernel=self.kernel, nu=self.nu, gamma=gamma)
+
+            # Train
+            start_time = time.time()
+            model.fit(X)
+            train_time = time.time() - start_time
+
+            # Test on small hold-out set from test set
+            scores = (-1.0) * model.decision_function(X_val)
+            scores = scores.flatten()
+
+            # Compute AUC
+            auc = roc_auc_score(labels, scores)
+
+            logger.info(f'  | Model {i:02}/{len(gammas):02} | Gamma: {gamma:.8f} | Train Time: {train_time:.3f}s '
+                        f'| Val AUC: {100. * auc:.2f} |')
+
+            if auc > best_auc:
+                best_auc = auc
+                self.model = model
+                self.gamma = gamma
+                self.results['train_time'] = train_time
+
+            i += 1
+
+        # If hybrid, also train a model with linear kernel
+        if self.hybrid:
+            self.linear_model = OneClassSVM(kernel='linear', nu=self.nu)
+            start_time = time.time()
+            self.linear_model.fit(X)
+            train_time = time.time() - start_time
+            self.results['train_time_linear'] = train_time
+
+        logger.info(f'Best Model: | Gamma: {self.gamma:.8f} | AUC: {100. * best_auc:.2f}')
+        logger.info('Training Time: {:.3f}s'.format(self.results['train_time']))
+        logger.info('Finished training.')
+
+    def test(self, dataset: BaseADDataset, device: str = 'cpu', n_jobs_dataloader: int = 0):
+        """Tests the OC-SVM model on the test data."""
+        logger = logging.getLogger()
+
+        _, test_loader = dataset.loaders(batch_size=128, num_workers=n_jobs_dataloader)
+
+        # Get data from loader
+        idx_label_score = []
+        X = ()
+        idxs = []
+        labels = []
+        for data in test_loader:
+            inputs, label_batch, _, idx = data
+            inputs, label_batch, idx = inputs.to(device), label_batch.to(device), idx.to(device)
+            if self.hybrid:
+                inputs = self.ae_net.encoder(inputs)  # in hybrid approach, take code representation of AE as features
+            X_batch = inputs.view(inputs.size(0), -1)  # X_batch.shape = (batch_size, n_channels * height * width)
+            X += (X_batch.cpu().data.numpy(),)
+            idxs += idx.cpu().data.numpy().astype(np.int64).tolist()
+            labels += label_batch.cpu().data.numpy().astype(np.int64).tolist()
+        X = np.concatenate(X)
+
+        # Testing
+        logger.info('Starting testing...')
+        start_time = time.time()
+
+        scores = (-1.0) * self.model.decision_function(X)
+
+        self.results['test_time'] = time.time() - start_time
+        scores = scores.flatten()
+        self.rho = -self.model.intercept_[0]
+
+        # Save triples of (idx, label, score) in a list
+        idx_label_score += list(zip(idxs, labels, scores.tolist()))
+        self.results['test_scores'] = idx_label_score
+
+        # Compute AUC
+        _, labels, scores = zip(*idx_label_score)
+        labels = np.array(labels)
+        scores = np.array(scores)
+        self.results['test_auc'] = roc_auc_score(labels, scores)
+
+        # If hybrid, also test model with linear kernel
+        if self.hybrid:
+            start_time = time.time()
+            scores_linear = (-1.0) * self.linear_model.decision_function(X)
+            self.results['test_time_linear'] = time.time() - start_time
+            scores_linear = scores_linear.flatten()
+            self.results['test_auc_linear'] = roc_auc_score(labels, scores_linear)
+            logger.info('Test AUC linear model: {:.2f}%'.format(100. * self.results['test_auc_linear']))
+            logger.info('Test Time linear model: {:.3f}s'.format(self.results['test_time_linear']))
+
+        # Log results
+        logger.info('Test AUC: {:.2f}%'.format(100. * self.results['test_auc']))
+        logger.info('Test Time: {:.3f}s'.format(self.results['test_time']))
+        logger.info('Finished testing.')
+
+    def load_ae(self, dataset_name, model_path):
+        """Load pretrained autoencoder from model_path for feature extraction in a hybrid OC-SVM model."""
+
+        model_dict = torch.load(model_path, map_location='cpu')
+        ae_net_dict = model_dict['ae_net_dict']
+        if dataset_name in ['mnist', 'fmnist', 'cifar10']:
+            net_name = dataset_name + '_LeNet'
+        else:
+            net_name = dataset_name + '_mlp'
+
+        if self.ae_net is None:
+            self.ae_net = build_autoencoder(net_name)
+
+        # update keys (since there was a change in network definition)
+        ae_keys = list(self.ae_net.state_dict().keys())
+        for i in range(len(ae_net_dict)):
+            k, v = ae_net_dict.popitem(False)
+            new_key = ae_keys[i]
+            ae_net_dict[new_key] = v
+            i += 1
+
+        self.ae_net.load_state_dict(ae_net_dict)
+        self.ae_net.eval()
+
+    def save_model(self, export_path):
+        """Save OC-SVM model to export_path."""
+        pass
+
+    def load_model(self, import_path, device: str = 'cpu'):
+        """Load OC-SVM model from import_path."""
+        pass
+
+    def save_results(self, export_json):
+        """Save results dict to a JSON-file."""
+        with open(export_json, 'w') as fp:
+            json.dump(self.results, fp)

Deep-SAD-PyTorch/src/baselines/shallow_ssad/__init__.py

@@ -0,0 +1 @@
+from .ssad_convex import ConvexSSAD

Deep-SAD-PyTorch/src/baselines/shallow_ssad/ssad_convex.py

@@ -0,0 +1,186 @@
+########################################################################################################################
+# Acknowledgements: https://github.com/nicococo/tilitools
+########################################################################################################################
+import numpy as np
+
+from cvxopt import matrix, spmatrix, sparse, spdiag
+from cvxopt.solvers import qp
+
+
+class ConvexSSAD:
+    """ Convex semi-supervised anomaly detection with hinge-loss and L2 regularizer
+        as described in Goernitz et al., Towards Supervised Anomaly Detection, JAIR, 2013
+
+             minimize  0.5 ||w||^2_2 - rho - kappa*gamma + eta_u sum_i xi_i + eta_l sum_j xi_j
+        {w,rho,gamma>=0,xi>=0}
+        subject to   <w,phi(x_i)> >= rho - xi_i
+                  y_j<w,phi(x_j)> >= y_j*rho + gamma - xi_j
+
+        And the corresponding dual optimization problem:
+
+            maximize -0.5 sum_(i,j) alpha_i alpha_j y_i y_j k(x_i,x_j)
+        {0<=alpha_i<=eta_i}
+            subject to 	kappa <= sum_j alpha_j  (for all labeled examples)
+                        1 = sum_j y_i alpha_j  (for all examples)
+
+        We introduce labels y_i = +1 for all unlabeled examples which enables us to combine sums.
+
+        Note: Only dual solution is supported.
+
+        Written by: Nico Goernitz, TU Berlin, 2013/14
+    """
+    PRECISION = 1e-9  # important: effects the threshold, support vectors and speed!
+
+    def __init__(self, kernel, y, kappa=1.0, Cp=1.0, Cu=1.0, Cn=1.0):
+        assert(len(y.shape) == 1)
+        self.kernel = kernel
+        self.y = y  # (vector) corresponding labels (+1,-1 and 0 for unlabeled)
+        self.kappa = kappa  # (scalar) regularizer for importance of the margin
+        self.Cp = Cp  # (scalar) the regularization constant for positively labeled samples > 0
+        self.Cu = Cu  # (scalar) the regularization constant for unlabeled samples > 0
+        self.Cn = Cn  # (scalar) the regularization constant for outliers > 0
+        self.samples = y.size
+        self.labeled = np.sum(np.abs(y))
+
+        # cy: (vector) converted label vector (+1 for pos and unlabeled, -1 for outliers)
+        self.cy = y.copy().reshape((y.size, 1))
+        self.cy[y == 0] = 1  # cy=+1.0 (unlabeled,pos) & cy=-1.0 (neg)
+
+        # cl: (vector) converted label vector (+1 for labeled examples, 0.0 for unlabeled)
+        self.cl = np.abs(y.copy())  # cl=+1.0 (labeled) cl=0.0 (unlabeled)
+
+        # (vector) converted upper bound box constraint for each example
+        self.cC = np.zeros(y.size)  # cC=Cu (unlabeled) cC=Cp (pos) cC=Cn (neg)
+        self.cC[y == 0] = Cu
+        self.cC[y == 1] = Cp
+        self.cC[y ==-1] = Cn
+
+        self.alphas = None
+        self.svs = None  # (vector) list of support vector (contains indices)
+        self.threshold = 0.0  # (scalar) the optimized threshold (rho)
+
+        # if there are no labeled examples, then set kappa to 0.0 otherwise
+        # the dual constraint kappa <= sum_{i \in labeled} alpha_i = 0.0 will
+        # prohibit a solution
+        if self.labeled == 0:
+            print('There are no labeled examples hence, setting kappa=0.0')
+            self.kappa = 0.0
+        print('Convex semi-supervised anomaly detection with {0} samples ({1} labeled).'.format(self.samples, self.labeled))
+
+    def set_train_kernel(self, kernel):
+        dim1, dim2 = kernel.shape
+        print([dim1, dim2])
+        assert(dim1 == dim2 and dim1 == self.samples)
+        self.kernel = kernel
+
+    def fit(self, check_psd_eigs=False):
+        # number of training examples
+        N = self.samples
+
+        # generate the label kernel
+        Y = self.cy.dot(self.cy.T)
+
+        # generate the final PDS kernel
+        P = matrix(self.kernel*Y)
+
+        # check for PSD
+        if check_psd_eigs:
+            eigs = np.linalg.eigvalsh(np.array(P))
+            if eigs[0] < 0.0:
+                print('Smallest eigenvalue is {0}'.format(eigs[0]))
+                P += spdiag([-eigs[0] for i in range(N)])
+
+        # there is no linear part of the objective
+        q = matrix(0.0, (N, 1))
+
+        # sum_i y_i alpha_i = A alpha = b = 1.0
+        A = matrix(self.cy, (1, self.samples), 'd')
+        b = matrix(1.0, (1, 1))
+
+        # inequality constraints: G alpha <= h
+        # 1) alpha_i  <= C_i
+        # 2) -alpha_i <= 0
+        G12 = spmatrix(1.0, range(N), range(N))
+        h1 = matrix(self.cC)
+        h2 = matrix(0.0, (N, 1))
+        G = sparse([G12, -G12])
+        h = matrix([h1, h2])
+        if self.labeled > 0:
+            # 3) kappa <= \sum_i labeled_i alpha_i -> -cl' alpha <= -kappa
+            print('Labeled data found.')
+            G3 = -matrix(self.cl, (1, self.cl.size), 'd')
+            h3 = -matrix(self.kappa, (1, 1))
+            G = sparse([G12, -G12, G3])
+            h = matrix([h1, h2, h3])
+
+        # solve the quadratic programm
+        sol = qp(P, -q, G, h, A, b)
+
+        # store solution
+        self.alphas = np.array(sol['x'])
+
+        # 1. find all support vectors, i.e. 0 < alpha_i <= C
+        # 2. store all support vector with alpha_i < C in 'margins'
+        self.svs = np.where(self.alphas >= ConvexSSAD.PRECISION)[0]
+
+        # these should sum to one
+        print('Validate solution:')
+        print('- found {0} support vectors'.format(len(self.svs)))
+        print('0 <= alpha_i : {0} of {1}'.format(np.sum(0. <= self.alphas), N))
+        print('- sum_(i) alpha_i cy_i = {0} = 1.0'.format(np.sum(self.alphas*self.cy)))
+        print('- sum_(i in sv) alpha_i cy_i = {0} ~ 1.0 (approx error)'.format(np.sum(self.alphas[self.svs]*self.cy[self.svs])))
+        print('- sum_(i in labeled) alpha_i = {0} >= {1} = kappa'.format(np.sum(self.alphas[self.cl == 1]), self.kappa))
+        print('- sum_(i in unlabeled) alpha_i = {0}'.format(np.sum(self.alphas[self.y == 0])))
+        print('- sum_(i in positives) alpha_i = {0}'.format(np.sum(self.alphas[self.y == 1])))
+        print('- sum_(i in negatives) alpha_i = {0}'.format(np.sum(self.alphas[self.y ==-1])))
+
+        # infer threshold (rho)
+        psvs = np.where(self.y[self.svs] == 0)[0]
+        # case 1: unlabeled support vectors available
+        self.threshold = 0.
+        unl_threshold = -1e12
+        lbl_threshold = -1e12
+        if psvs.size > 0:
+            k = self.kernel[:, self.svs]
+            k = k[self.svs[psvs], :]
+            unl_threshold = np.max(self.apply(k))
+
+        if np.sum(self.cl) > 1e-12:
+        # case 2: only labeled examples available
+            k = self.kernel[:, self.svs]
+            k = k[self.svs, :]
+            thres = self.apply(k)
+            pinds = np.where(self.y[self.svs] == +1)[0]
+            ninds = np.where(self.y[self.svs] == -1)[0]
+            # only negatives is not possible
+            if ninds.size > 0 and pinds.size == 0:
+                print('ERROR: Check pre-defined PRECISION.')
+                lbl_threshold = np.max(thres[ninds])
+            elif ninds.size == 0:
+                lbl_threshold = np.max(thres[pinds])
+            else:
+                # smallest negative + largest positive
+                p = np.max(thres[pinds])
+                n = np.min(thres[ninds])
+                lbl_threshold = (n+p)/2.
+        self.threshold = np.max((unl_threshold, lbl_threshold))
+
+    def get_threshold(self):
+        return self.threshold
+
+    def get_support_dual(self):
+        return self.svs
+
+    def get_alphas(self):
+        return self.alphas
+
+    def apply(self, kernel):
+        """ Application of dual trained ssad.
+            kernel = get_kernel(Y, X[:, cssad.svs], kernel_type, kernel_param)
+        """
+        if kernel.shape[1] == self.samples:
+            # if kernel is not restricted to support vectors
+            ay = self.alphas * self.cy
+        else:
+            ay = self.alphas[self.svs] * self.cy[self.svs]
+        return ay.T.dot(kernel.T).T - self.threshold

Deep-SAD-PyTorch/src/baselines/ssad.py

@@ -0,0 +1,244 @@
+import json
+import logging
+import time
+import torch
+import numpy as np
+
+from torch.utils.data import DataLoader
+from .shallow_ssad.ssad_convex import ConvexSSAD
+from sklearn.metrics import roc_auc_score
+from sklearn.metrics.pairwise import pairwise_kernels
+from base.base_dataset import BaseADDataset
+from networks.main import build_autoencoder
+
+
+class SSAD(object):
+    """
+    A class for kernel SSAD models as described in Goernitz et al., Towards Supervised Anomaly Detection, JAIR, 2013.
+    """
+
+    def __init__(self, kernel='rbf', kappa=1.0, Cp=1.0, Cu=1.0, Cn=1.0, hybrid=False):
+        """Init SSAD instance."""
+        self.kernel = kernel
+        self.kappa = kappa
+        self.Cp = Cp
+        self.Cu = Cu
+        self.Cn = Cn
+        self.rho = None
+        self.gamma = None
+
+        self.model = None
+        self.X_svs = None
+
+        self.hybrid = hybrid
+        self.ae_net = None  # autoencoder network for the case of a hybrid model
+        self.linear_model = None  # also init a model with linear kernel if hybrid approach
+        self.linear_X_svs = None
+
+        self.results = {
+            'train_time': None,
+            'test_time': None,
+            'test_auc': None,
+            'test_scores': None,
+            'train_time_linear': None,
+            'test_time_linear': None,
+            'test_auc_linear': None
+        }
+
+    def train(self, dataset: BaseADDataset, device: str = 'cpu', n_jobs_dataloader: int = 0):
+        """Trains the SSAD model on the training data."""
+        logger = logging.getLogger()
+
+        # do not drop last batch for non-SGD optimization shallow_ssad
+        train_loader = DataLoader(dataset=dataset.train_set, batch_size=128, shuffle=True,
+                                  num_workers=n_jobs_dataloader, drop_last=False)
+
+        # Get data from loader
+        X = ()
+        semi_targets = []
+        for data in train_loader:
+            inputs, _, semi_targets_batch, _ = data
+            inputs, semi_targets_batch = inputs.to(device), semi_targets_batch.to(device)
+            if self.hybrid:
+                inputs = self.ae_net.encoder(inputs)  # in hybrid approach, take code representation of AE as features
+            X_batch = inputs.view(inputs.size(0), -1)  # X_batch.shape = (batch_size, n_channels * height * width)
+            X += (X_batch.cpu().data.numpy(),)
+            semi_targets += semi_targets_batch.cpu().data.numpy().astype(np.int).tolist()
+        X, semi_targets = np.concatenate(X), np.array(semi_targets)
+
+        # Training
+        logger.info('Starting training...')
+
+        # Select model via hold-out test set of 1000 samples
+        gammas = np.logspace(-7, 2, num=10, base=2)
+        best_auc = 0.0
+
+        # Sample hold-out set from test set
+        _, test_loader = dataset.loaders(batch_size=128, num_workers=n_jobs_dataloader)
+
+        X_test = ()
+        labels = []
+        for data in test_loader:
+            inputs, label_batch, _, _ = data
+            inputs, label_batch = inputs.to(device), label_batch.to(device)
+            if self.hybrid:
+                inputs = self.ae_net.encoder(inputs)  # in hybrid approach, take code representation of AE as features
+            X_batch = inputs.view(inputs.size(0), -1)  # X_batch.shape = (batch_size, n_channels * height * width)
+            X_test += (X_batch.cpu().data.numpy(),)
+            labels += label_batch.cpu().data.numpy().astype(np.int64).tolist()
+        X_test, labels = np.concatenate(X_test), np.array(labels)
+        n_test, n_normal, n_outlier = len(X_test), np.sum(labels == 0), np.sum(labels == 1)
+        n_val = int(0.1 * n_test)
+        n_val_normal, n_val_outlier = int(n_val * (n_normal/n_test)), int(n_val * (n_outlier/n_test))
+        perm = np.random.permutation(n_test)
+        X_val = np.concatenate((X_test[perm][labels[perm] == 0][:n_val_normal],
+                                X_test[perm][labels[perm] == 1][:n_val_outlier]))
+        labels = np.array([0] * n_val_normal + [1] * n_val_outlier)
+
+        i = 1
+        for gamma in gammas:
+
+            # Build the training kernel
+            kernel = pairwise_kernels(X, X, metric=self.kernel, gamma=gamma)
+
+            # Model candidate
+            model = ConvexSSAD(kernel, semi_targets, Cp=self.Cp, Cu=self.Cu, Cn=self.Cn)
+
+            # Train
+            start_time = time.time()
+            model.fit()
+            train_time = time.time() - start_time
+
+            # Test on small hold-out set from test set
+            kernel_val = pairwise_kernels(X_val, X[model.svs, :], metric=self.kernel, gamma=gamma)
+            scores = (-1.0) * model.apply(kernel_val)
+            scores = scores.flatten()
+
+            # Compute AUC
+            auc = roc_auc_score(labels, scores)
+
+            logger.info(f'  | Model {i:02}/{len(gammas):02} | Gamma: {gamma:.8f} | Train Time: {train_time:.3f}s '
+                        f'| Val AUC: {100. * auc:.2f} |')
+
+            if auc > best_auc:
+                best_auc = auc
+                self.model = model
+                self.gamma = gamma
+                self.results['train_time'] = train_time
+
+            i += 1
+
+        # Get support vectors for testing
+        self.X_svs = X[self.model.svs, :]
+
+        # If hybrid, also train a model with linear kernel
+        if self.hybrid:
+            linear_kernel = pairwise_kernels(X, X, metric='linear')
+            self.linear_model = ConvexSSAD(linear_kernel, semi_targets, Cp=self.Cp, Cu=self.Cu, Cn=self.Cn)
+            start_time = time.time()
+            self.linear_model.fit()
+            train_time = time.time() - start_time
+            self.results['train_time_linear'] = train_time
+            self.linear_X_svs = X[self.linear_model.svs, :]
+
+        logger.info(f'Best Model: | Gamma: {self.gamma:.8f} | AUC: {100. * best_auc:.2f}')
+        logger.info('Training Time: {:.3f}s'.format(self.results['train_time']))
+        logger.info('Finished training.')
+
+    def test(self, dataset: BaseADDataset, device: str = 'cpu', n_jobs_dataloader: int = 0):
+        """Tests the SSAD model on the test data."""
+        logger = logging.getLogger()
+
+        _, test_loader = dataset.loaders(batch_size=128, num_workers=n_jobs_dataloader)
+
+        # Get data from loader
+        idx_label_score = []
+        X = ()
+        idxs = []
+        labels = []
+        for data in test_loader:
+            inputs, label_batch, _, idx = data
+            inputs, label_batch, idx = inputs.to(device), label_batch.to(device), idx.to(device)
+            if self.hybrid:
+                inputs = self.ae_net.encoder(inputs)  # in hybrid approach, take code representation of AE as features
+            X_batch = inputs.view(inputs.size(0), -1)  # X_batch.shape = (batch_size, n_channels * height * width)
+            X += (X_batch.cpu().data.numpy(),)
+            idxs += idx.cpu().data.numpy().astype(np.int64).tolist()
+            labels += label_batch.cpu().data.numpy().astype(np.int64).tolist()
+        X = np.concatenate(X)
+
+        # Testing
+        logger.info('Starting testing...')
+        start_time = time.time()
+
+        # Build kernel
+        kernel = pairwise_kernels(X, self.X_svs, metric=self.kernel, gamma=self.gamma)
+
+        scores = (-1.0) * self.model.apply(kernel)
+
+        self.results['test_time'] = time.time() - start_time
+        scores = scores.flatten()
+        self.rho = -self.model.threshold
+
+        # Save triples of (idx, label, score) in a list
+        idx_label_score += list(zip(idxs, labels, scores.tolist()))
+        self.results['test_scores'] = idx_label_score
+
+        # Compute AUC
+        _, labels, scores = zip(*idx_label_score)
+        labels = np.array(labels)
+        scores = np.array(scores)
+        self.results['test_auc'] = roc_auc_score(labels, scores)
+
+        # If hybrid, also test model with linear kernel
+        if self.hybrid:
+            start_time = time.time()
+            linear_kernel = pairwise_kernels(X, self.linear_X_svs, metric='linear')
+            scores_linear = (-1.0) * self.linear_model.apply(linear_kernel)
+            self.results['test_time_linear'] = time.time() - start_time
+            scores_linear = scores_linear.flatten()
+            self.results['test_auc_linear'] = roc_auc_score(labels, scores_linear)
+            logger.info('Test AUC linear model: {:.2f}%'.format(100. * self.results['test_auc_linear']))
+            logger.info('Test Time linear model: {:.3f}s'.format(self.results['test_time_linear']))
+
+        # Log results
+        logger.info('Test AUC: {:.2f}%'.format(100. * self.results['test_auc']))
+        logger.info('Test Time: {:.3f}s'.format(self.results['test_time']))
+        logger.info('Finished testing.')
+
+    def load_ae(self, dataset_name, model_path):
+        """Load pretrained autoencoder from model_path for feature extraction in a hybrid SSAD model."""
+
+        model_dict = torch.load(model_path, map_location='cpu')
+        ae_net_dict = model_dict['ae_net_dict']
+        if dataset_name in ['mnist', 'fmnist', 'cifar10']:
+            net_name = dataset_name + '_LeNet'
+        else:
+            net_name = dataset_name + '_mlp'
+
+        if self.ae_net is None:
+            self.ae_net = build_autoencoder(net_name)
+
+        # update keys (since there was a change in network definition)
+        ae_keys = list(self.ae_net.state_dict().keys())
+        for i in range(len(ae_net_dict)):
+            k, v = ae_net_dict.popitem(False)
+            new_key = ae_keys[i]
+            ae_net_dict[new_key] = v
+            i += 1
+
+        self.ae_net.load_state_dict(ae_net_dict)
+        self.ae_net.eval()
+
+    def save_model(self, export_path):
+        """Save SSAD model to export_path."""
+        pass
+
+    def load_model(self, import_path, device: str = 'cpu'):
+        """Load SSAD model from import_path."""
+        pass
+
+    def save_results(self, export_json):
+        """Save results dict to a JSON-file."""
+        with open(export_json, 'w') as fp:
+            json.dump(self.results, fp)

Deep-SAD-PyTorch/src/datasets/__init__.py

@@ -0,0 +1,6 @@
+from .main import load_dataset
+from .mnist import MNIST_Dataset
+from .fmnist import FashionMNIST_Dataset
+from .cifar10 import CIFAR10_Dataset
+from .odds import ODDSADDataset
+from .preprocessing import *

Deep-SAD-PyTorch/src/datasets/cifar10.py

@@ -0,0 +1,86 @@
+from torch.utils.data import Subset
+from PIL import Image
+from torchvision.datasets import CIFAR10
+from base.torchvision_dataset import TorchvisionDataset
+from .preprocessing import create_semisupervised_setting
+
+import torch
+import torchvision.transforms as transforms
+import random
+import numpy as np
+
+
+class CIFAR10_Dataset(TorchvisionDataset):
+
+    def __init__(self, root: str, normal_class: int = 5, known_outlier_class: int = 3, n_known_outlier_classes: int = 0,
+                 ratio_known_normal: float = 0.0, ratio_known_outlier: float = 0.0, ratio_pollution: float = 0.0):
+        super().__init__(root)
+
+        # Define normal and outlier classes
+        self.n_classes = 2  # 0: normal, 1: outlier
+        self.normal_classes = tuple([normal_class])
+        self.outlier_classes = list(range(0, 10))
+        self.outlier_classes.remove(normal_class)
+        self.outlier_classes = tuple(self.outlier_classes)
+
+        if n_known_outlier_classes == 0:
+            self.known_outlier_classes = ()
+        elif n_known_outlier_classes == 1:
+            self.known_outlier_classes = tuple([known_outlier_class])
+        else:
+            self.known_outlier_classes = tuple(random.sample(self.outlier_classes, n_known_outlier_classes))
+
+        # CIFAR-10 preprocessing: feature scaling to [0, 1]
+        transform = transforms.ToTensor()
+        target_transform = transforms.Lambda(lambda x: int(x in self.outlier_classes))
+
+        # Get train set
+        train_set = MyCIFAR10(root=self.root, train=True, transform=transform, target_transform=target_transform,
+                              download=True)
+
+        # Create semi-supervised setting
+        idx, _, semi_targets = create_semisupervised_setting(np.array(train_set.targets), self.normal_classes,
+                                                             self.outlier_classes, self.known_outlier_classes,
+                                                             ratio_known_normal, ratio_known_outlier, ratio_pollution)
+        train_set.semi_targets[idx] = torch.tensor(semi_targets)  # set respective semi-supervised labels
+
+        # Subset train_set to semi-supervised setup
+        self.train_set = Subset(train_set, idx)
+
+        # Get test set
+        self.test_set = MyCIFAR10(root=self.root, train=False, transform=transform, target_transform=target_transform,
+                                  download=True)
+
+
+class MyCIFAR10(CIFAR10):
+    """
+    Torchvision CIFAR10 class with additional targets for the semi-supervised setting and patch of __getitem__ method
+    to also return the semi-supervised target as well as the index of a data sample.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super(MyCIFAR10, self).__init__(*args, **kwargs)
+
+        self.semi_targets = torch.zeros(len(self.targets), dtype=torch.int64)
+
+    def __getitem__(self, index):
+        """Override the original method of the CIFAR10 class.
+        Args:
+            index (int): Index
+
+        Returns:
+            tuple: (image, target, semi_target, index)
+        """
+        img, target, semi_target = self.data[index], self.targets[index], int(self.semi_targets[index])
+
+        # doing this so that it is consistent with all other datasets
+        # to return a PIL Image
+        img = Image.fromarray(img)
+
+        if self.transform is not None:
+            img = self.transform(img)
+
+        if self.target_transform is not None:
+            target = self.target_transform(target)
+
+        return img, target, semi_target, index

Deep-SAD-PyTorch/src/datasets/fmnist.py

@@ -0,0 +1,85 @@
+from torch.utils.data import Subset
+from PIL import Image
+from torchvision.datasets import FashionMNIST
+from base.torchvision_dataset import TorchvisionDataset
+from .preprocessing import create_semisupervised_setting
+
+import torch
+import torchvision.transforms as transforms
+import random
+
+
+class FashionMNIST_Dataset(TorchvisionDataset):
+
+    def __init__(self, root: str, normal_class: int = 0, known_outlier_class: int = 1, n_known_outlier_classes: int = 0,
+                 ratio_known_normal: float = 0.0, ratio_known_outlier: float = 0.0, ratio_pollution: float = 0.0):
+        super().__init__(root)
+
+        # Define normal and outlier classes
+        self.n_classes = 2  # 0: normal, 1: outlier
+        self.normal_classes = tuple([normal_class])
+        self.outlier_classes = list(range(0, 10))
+        self.outlier_classes.remove(normal_class)
+        self.outlier_classes = tuple(self.outlier_classes)
+
+        if n_known_outlier_classes == 0:
+            self.known_outlier_classes = ()
+        elif n_known_outlier_classes == 1:
+            self.known_outlier_classes = tuple([known_outlier_class])
+        else:
+            self.known_outlier_classes = tuple(random.sample(self.outlier_classes, n_known_outlier_classes))
+
+        # FashionMNIST preprocessing: feature scaling to [0, 1]
+        transform = transforms.ToTensor()
+        target_transform = transforms.Lambda(lambda x: int(x in self.outlier_classes))
+
+        # Get train set
+        train_set = MyFashionMNIST(root=self.root, train=True, transform=transform, target_transform=target_transform,
+                                   download=True)
+
+        # Create semi-supervised setting
+        idx, _, semi_targets = create_semisupervised_setting(train_set.targets.cpu().data.numpy(), self.normal_classes,
+                                                             self.outlier_classes, self.known_outlier_classes,
+                                                             ratio_known_normal, ratio_known_outlier, ratio_pollution)
+        train_set.semi_targets[idx] = torch.tensor(semi_targets)  # set respective semi-supervised labels
+
+        # Subset train_set to semi-supervised setup
+        self.train_set = Subset(train_set, idx)
+
+        # Get test set
+        self.test_set = MyFashionMNIST(root=self.root, train=False, transform=transform,
+                                       target_transform=target_transform, download=True)
+
+
+class MyFashionMNIST(FashionMNIST):
+    """
+    Torchvision FashionMNIST class with additional targets for the semi-supervised setting and patch of __getitem__
+    method to also return the semi-supervised target as well as the index of a data sample.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super(MyFashionMNIST, self).__init__(*args, **kwargs)
+
+        self.semi_targets = torch.zeros_like(self.targets)
+
+    def __getitem__(self, index):
+        """Override the original method of the MyFashionMNIST class.
+        Args:
+            index (int): Index
+
+        Returns:
+            tuple: (image, target, semi_target, index)
+        """
+        img, target, semi_target = self.data[index], int(self.targets[index]), int(self.semi_targets[index])
+
+        # doing this so that it is consistent with all other datasets
+        # to return a PIL Image
+        img = Image.fromarray(img.numpy(), mode='L')
+
+        if self.transform is not None:
+            img = self.transform(img)
+
+        if self.target_transform is not None:
+            target = self.target_transform(target)
+
+        return img, target, semi_target, index

Deep-SAD-PyTorch/src/datasets/main.py

@@ -0,0 +1,54 @@
+from .mnist import MNIST_Dataset
+from .fmnist import FashionMNIST_Dataset
+from .cifar10 import CIFAR10_Dataset
+from .odds import ODDSADDataset
+
+
+def load_dataset(dataset_name, data_path, normal_class, known_outlier_class, n_known_outlier_classes: int = 0,
+                 ratio_known_normal: float = 0.0, ratio_known_outlier: float = 0.0, ratio_pollution: float = 0.0,
+                 random_state=None):
+    """Loads the dataset."""
+
+    implemented_datasets = ('mnist', 'fmnist', 'cifar10',
+                            'arrhythmia', 'cardio', 'satellite', 'satimage-2', 'shuttle', 'thyroid')
+    assert dataset_name in implemented_datasets
+
+    dataset = None
+
+    if dataset_name == 'mnist':
+        dataset = MNIST_Dataset(root=data_path,
+                                normal_class=normal_class,
+                                known_outlier_class=known_outlier_class,
+                                n_known_outlier_classes=n_known_outlier_classes,
+                                ratio_known_normal=ratio_known_normal,
+                                ratio_known_outlier=ratio_known_outlier,
+                                ratio_pollution=ratio_pollution)
+
+    if dataset_name == 'fmnist':
+        dataset = FashionMNIST_Dataset(root=data_path,
+                                       normal_class=normal_class,
+                                       known_outlier_class=known_outlier_class,
+                                       n_known_outlier_classes=n_known_outlier_classes,
+                                       ratio_known_normal=ratio_known_normal,
+                                       ratio_known_outlier=ratio_known_outlier,
+                                       ratio_pollution=ratio_pollution)
+
+    if dataset_name == 'cifar10':
+        dataset = CIFAR10_Dataset(root=data_path,
+                                  normal_class=normal_class,
+                                  known_outlier_class=known_outlier_class,
+                                  n_known_outlier_classes=n_known_outlier_classes,
+                                  ratio_known_normal=ratio_known_normal,
+                                  ratio_known_outlier=ratio_known_outlier,
+                                  ratio_pollution=ratio_pollution)
+
+    if dataset_name in ('arrhythmia', 'cardio', 'satellite', 'satimage-2', 'shuttle', 'thyroid'):
+        dataset = ODDSADDataset(root=data_path,
+                                dataset_name=dataset_name,
+                                n_known_outlier_classes=n_known_outlier_classes,
+                                ratio_known_normal=ratio_known_normal,
+                                ratio_known_outlier=ratio_known_outlier,
+                                ratio_pollution=ratio_pollution,
+                                random_state=random_state)
+
+    return dataset

Deep-SAD-PyTorch/src/datasets/mnist.py

@@ -0,0 +1,85 @@
+from torch.utils.data import Subset
+from PIL import Image
+from torchvision.datasets import MNIST
+from base.torchvision_dataset import TorchvisionDataset
+from .preprocessing import create_semisupervised_setting
+
+import torch
+import torchvision.transforms as transforms
+import random
+
+
+class MNIST_Dataset(TorchvisionDataset):
+
+    def __init__(self, root: str, normal_class: int = 0, known_outlier_class: int = 1, n_known_outlier_classes: int = 0,
+                 ratio_known_normal: float = 0.0, ratio_known_outlier: float = 0.0, ratio_pollution: float = 0.0):
+        super().__init__(root)
+
+        # Define normal and outlier classes
+        self.n_classes = 2  # 0: normal, 1: outlier
+        self.normal_classes = tuple([normal_class])
+        self.outlier_classes = list(range(0, 10))
+        self.outlier_classes.remove(normal_class)
+        self.outlier_classes = tuple(self.outlier_classes)
+
+        if n_known_outlier_classes == 0:
+            self.known_outlier_classes = ()
+        elif n_known_outlier_classes == 1:
+            self.known_outlier_classes = tuple([known_outlier_class])
+        else:
+            self.known_outlier_classes = tuple(random.sample(self.outlier_classes, n_known_outlier_classes))
+
+        # MNIST preprocessing: feature scaling to [0, 1]
+        transform = transforms.ToTensor()
+        target_transform = transforms.Lambda(lambda x: int(x in self.outlier_classes))
+
+        # Get train set
+        train_set = MyMNIST(root=self.root, train=True, transform=transform, target_transform=target_transform,
+                            download=True)
+
+        # Create semi-supervised setting
+        idx, _, semi_targets = create_semisupervised_setting(train_set.targets.cpu().data.numpy(), self.normal_classes,
+                                                             self.outlier_classes, self.known_outlier_classes,
+                                                             ratio_known_normal, ratio_known_outlier, ratio_pollution)
+        train_set.semi_targets[idx] = torch.tensor(semi_targets)  # set respective semi-supervised labels
+
+        # Subset train_set to semi-supervised setup
+        self.train_set = Subset(train_set, idx)
+
+        # Get test set
+        self.test_set = MyMNIST(root=self.root, train=False, transform=transform, target_transform=target_transform,
+                                download=True)
+
+
+class MyMNIST(MNIST):
+    """
+    Torchvision MNIST class with additional targets for the semi-supervised setting and patch of __getitem__ method
+    to also return the semi-supervised target as well as the index of a data sample.
+    """
+
+    def __init__(self, *args, **kwargs):
+        super(MyMNIST, self).__init__(*args, **kwargs)
+
+        self.semi_targets = torch.zeros_like(self.targets)
+
+    def __getitem__(self, index):
+        """Override the original method of the MNIST class.
+        Args:
+            index (int): Index
+
+        Returns:
+            tuple: (image, target, semi_target, index)
+        """
+        img, target, semi_target = self.data[index], int(self.targets[index]), int(self.semi_targets[index])
+
+        # doing this so that it is consistent with all other datasets
+        # to return a PIL Image
+        img = Image.fromarray(img.numpy(), mode='L')
+
+        if self.transform is not None:
+            img = self.transform(img)
+
+        if self.target_transform is not None:
+            target = self.target_transform(target)
+
+        return img, target, semi_target, index

Deep-SAD-PyTorch/src/datasets/odds.py

@@ -0,0 +1,47 @@
+from torch.utils.data import DataLoader, Subset
+from base.base_dataset import BaseADDataset
+from base.odds_dataset import ODDSDataset
+from .preprocessing import create_semisupervised_setting
+
+import torch
+
+
+class ODDSADDataset(BaseADDataset):
+
+    def __init__(self, root: str, dataset_name: str, n_known_outlier_classes: int = 0, ratio_known_normal: float = 0.0,
+                 ratio_known_outlier: float = 0.0, ratio_pollution: float = 0.0, random_state=None):
+        super().__init__(root)
+
+        # Define normal and outlier classes
+        self.n_classes = 2  # 0: normal, 1: outlier
+        self.normal_classes = (0,)
+        self.outlier_classes = (1,)
+
+        if n_known_outlier_classes == 0:
+            self.known_outlier_classes = ()
+        else:
+            self.known_outlier_classes = (1,)
+
+        # Get train set
+        train_set = ODDSDataset(root=self.root, dataset_name=dataset_name, train=True, random_state=random_state,
+                                download=True)
+
+        # Create semi-supervised setting
+        idx, _, semi_targets = create_semisupervised_setting(train_set.targets.cpu().data.numpy(), self.normal_classes,
+                                                             self.outlier_classes, self.known_outlier_classes,
+                                                             ratio_known_normal, ratio_known_outlier, ratio_pollution)
+        train_set.semi_targets[idx] = torch.tensor(semi_targets)  # set respective semi-supervised labels
+
+        # Subset train_set to semi-supervised setup
+        self.train_set = Subset(train_set, idx)
+
+        # Get test set
+        self.test_set = ODDSDataset(root=self.root, dataset_name=dataset_name, train=False, random_state=random_state)
+
+    def loaders(self, batch_size: int, shuffle_train=True, shuffle_test=False, num_workers: int = 0) -> (
+            DataLoader, DataLoader):
+        train_loader = DataLoader(dataset=self.train_set, batch_size=batch_size, shuffle=shuffle_train,
+                                  num_workers=num_workers, drop_last=True)
+        test_loader = DataLoader(dataset=self.test_set, batch_size=batch_size, shuffle=shuffle_test,
+                                 num_workers=num_workers, drop_last=False)
+        return train_loader, test_loader

Deep-SAD-PyTorch/src/datasets/preprocessing.py

@@ -0,0 +1,66 @@
+import torch
+import numpy as np
+
+
+def create_semisupervised_setting(labels, normal_classes, outlier_classes, known_outlier_classes,
+                                  ratio_known_normal, ratio_known_outlier, ratio_pollution):
+    """
+    Create a semi-supervised data setting. 
+    :param labels: np.array with labels of all dataset samples
+    :param normal_classes: tuple with normal class labels
+    :param outlier_classes: tuple with anomaly class labels
+    :param known_outlier_classes: tuple with known (labeled) anomaly class labels
+    :param ratio_known_normal: the desired ratio of known (labeled) normal samples
+    :param ratio_known_outlier: the desired ratio of known (labeled) anomalous samples
+    :param ratio_pollution: the desired pollution ratio of the unlabeled data with unknown (unlabeled) anomalies.
+    :return: tuple with list of sample indices, list of original labels, and list of semi-supervised labels
+    """
+    idx_normal = np.argwhere(np.isin(labels, normal_classes)).flatten()
+    idx_outlier = np.argwhere(np.isin(labels, outlier_classes)).flatten()
+    idx_known_outlier_candidates = np.argwhere(np.isin(labels, known_outlier_classes)).flatten()
+
+    n_normal = len(idx_normal)
+
+    # Solve system of linear equations to obtain respective number of samples
+    a = np.array([[1, 1, 0, 0],
+                  [(1-ratio_known_normal), -ratio_known_normal, -ratio_known_normal, -ratio_known_normal],
+                  [-ratio_known_outlier, -ratio_known_outlier, -ratio_known_outlier, (1-ratio_known_outlier)],
+                  [0, -ratio_pollution, (1-ratio_pollution), 0]])
+    b = np.array([n_normal, 0, 0, 0])
+    x = np.linalg.solve(a, b)
+
+    # Get number of samples
+    n_known_normal = int(x[0])
+    n_unlabeled_normal = int(x[1])
+    n_unlabeled_outlier = int(x[2])
+    n_known_outlier = int(x[3])
+
+    # Sample indices
+    perm_normal = np.random.permutation(n_normal)
+    perm_outlier = np.random.permutation(len(idx_outlier))
+    perm_known_outlier = np.random.permutation(len(idx_known_outlier_candidates))
+
+    idx_known_normal = idx_normal[perm_normal[:n_known_normal]].tolist()
+    idx_unlabeled_normal = idx_normal[perm_normal[n_known_normal:n_known_normal+n_unlabeled_normal]].tolist()
+    idx_unlabeled_outlier = idx_outlier[perm_outlier[:n_unlabeled_outlier]].tolist()
+    idx_known_outlier = idx_known_outlier_candidates[perm_known_outlier[:n_known_outlier]].tolist()
+
+    # Get original class labels
+    labels_known_normal = labels[idx_known_normal].tolist()
+    labels_unlabeled_normal = labels[idx_unlabeled_normal].tolist()
+    labels_unlabeled_outlier = labels[idx_unlabeled_outlier].tolist()
+    labels_known_outlier = labels[idx_known_outlier].tolist()
+
+    # Get semi-supervised setting labels
+    semi_labels_known_normal = np.ones(n_known_normal).astype(np.int32).tolist()
+    semi_labels_unlabeled_normal = np.zeros(n_unlabeled_normal).astype(np.int32).tolist()
+    semi_labels_unlabeled_outlier = np.zeros(n_unlabeled_outlier).astype(np.int32).tolist()
+    semi_labels_known_outlier = (-np.ones(n_known_outlier).astype(np.int32)).tolist()
+
+    # Create final lists
+    list_idx = idx_known_normal + idx_unlabeled_normal + idx_unlabeled_outlier + idx_known_outlier
+    list_labels = labels_known_normal + labels_unlabeled_normal + labels_unlabeled_outlier + labels_known_outlier
+    list_semi_labels = (semi_labels_known_normal + semi_labels_unlabeled_normal + semi_labels_unlabeled_outlier
+                        + semi_labels_known_outlier)
+
+    return list_idx, list_labels, list_semi_labels

Deep-SAD-PyTorch/src/main.py

@@ -0,0 +1,239 @@
+import click
+import torch
+import logging
+import random
+import numpy as np
+
+from utils.config import Config
+from utils.visualization.plot_images_grid import plot_images_grid
+from DeepSAD import DeepSAD
+from datasets.main import load_dataset
+
+
+################################################################################
+# Settings
+################################################################################
+@click.command()
+@click.argument('dataset_name', type=click.Choice(['mnist', 'fmnist', 'cifar10', 'arrhythmia', 'cardio', 'satellite',
+                                                   'satimage-2', 'shuttle', 'thyroid']))
+@click.argument('net_name', type=click.Choice(['mnist_LeNet', 'fmnist_LeNet', 'cifar10_LeNet', 'arrhythmia_mlp',
+                                               'cardio_mlp', 'satellite_mlp', 'satimage-2_mlp', 'shuttle_mlp',
+                                               'thyroid_mlp']))
+@click.argument('xp_path', type=click.Path(exists=True))
+@click.argument('data_path', type=click.Path(exists=True))
+@click.option('--load_config', type=click.Path(exists=True), default=None,
+              help='Config JSON-file path (default: None).')
+@click.option('--load_model', type=click.Path(exists=True), default=None,
+              help='Model file path (default: None).')
+@click.option('--eta', type=float, default=1.0, help='Deep SAD hyperparameter eta (must be 0 < eta).')
+@click.option('--ratio_known_normal', type=float, default=0.0,
+              help='Ratio of known (labeled) normal training examples.')
+@click.option('--ratio_known_outlier', type=float, default=0.0,
+              help='Ratio of known (labeled) anomalous training examples.')
+@click.option('--ratio_pollution', type=float, default=0.0,
+              help='Pollution ratio of unlabeled training data with unknown (unlabeled) anomalies.')
+@click.option('--device', type=str, default='cuda', help='Computation device to use ("cpu", "cuda", "cuda:2", etc.).')
+@click.option('--seed', type=int, default=-1, help='Set seed. If -1, use randomization.')
+@click.option('--optimizer_name', type=click.Choice(['adam']), default='adam',
+              help='Name of the optimizer to use for Deep SAD network training.')
+@click.option('--lr', type=float, default=0.001,
+              help='Initial learning rate for Deep SAD network training. Default=0.001')
+@click.option('--n_epochs', type=int, default=50, help='Number of epochs to train.')
+@click.option('--lr_milestone', type=int, default=0, multiple=True,
+              help='Lr scheduler milestones at which lr is multiplied by 0.1. Can be multiple and must be increasing.')
+@click.option('--batch_size', type=int, default=128, help='Batch size for mini-batch training.')
+@click.option('--weight_decay', type=float, default=1e-6,
+              help='Weight decay (L2 penalty) hyperparameter for Deep SAD objective.')
+@click.option('--pretrain', type=bool, default=True,
+              help='Pretrain neural network parameters via autoencoder.')
+@click.option('--ae_optimizer_name', type=click.Choice(['adam']), default='adam',
+              help='Name of the optimizer to use for autoencoder pretraining.')
+@click.option('--ae_lr', type=float, default=0.001,
+              help='Initial learning rate for autoencoder pretraining. Default=0.001')
+@click.option('--ae_n_epochs', type=int, default=100, help='Number of epochs to train autoencoder.')
+@click.option('--ae_lr_milestone', type=int, default=0, multiple=True,
+              help='Lr scheduler milestones at which lr is multiplied by 0.1. Can be multiple and must be increasing.')
+@click.option('--ae_batch_size', type=int, default=128, help='Batch size for mini-batch autoencoder training.')
+@click.option('--ae_weight_decay', type=float, default=1e-6,
+              help='Weight decay (L2 penalty) hyperparameter for autoencoder objective.')
+@click.option('--num_threads', type=int, default=0,
+              help='Number of threads used for parallelizing CPU operations. 0 means that all resources are used.')
+@click.option('--n_jobs_dataloader', type=int, default=0,
+              help='Number of workers for data loading. 0 means that the data will be loaded in the main process.')
+@click.option('--normal_class', type=int, default=0,
+              help='Specify the normal class of the dataset (all other classes are considered anomalous).')
+@click.option('--known_outlier_class', type=int, default=1,
+              help='Specify the known outlier class of the dataset for semi-supervised anomaly detection.')
+@click.option('--n_known_outlier_classes', type=int, default=0,
+              help='Number of known outlier classes.'
+                   'If 0, no anomalies are known.'
+                   'If 1, outlier class as specified in --known_outlier_class option.'
+                   'If > 1, the specified number of outlier classes will be sampled at random.')
+def main(dataset_name, net_name, xp_path, data_path, load_config, load_model, eta,
+         ratio_known_normal, ratio_known_outlier, ratio_pollution, device, seed,
+         optimizer_name, lr, n_epochs, lr_milestone, batch_size, weight_decay,
+         pretrain, ae_optimizer_name, ae_lr, ae_n_epochs, ae_lr_milestone, ae_batch_size, ae_weight_decay,
+         num_threads, n_jobs_dataloader, normal_class, known_outlier_class, n_known_outlier_classes):
+    """
+    Deep SAD, a method for deep semi-supervised anomaly detection.
+
+    :arg DATASET_NAME: Name of the dataset to load.
+    :arg NET_NAME: Name of the neural network to use.
+    :arg XP_PATH: Export path for logging the experiment.
+    :arg DATA_PATH: Root path of data.
+    """
+
+    # Get configuration
+    cfg = Config(locals().copy())
+
+    # Set up logging
+    logging.basicConfig(level=logging.INFO)
+    logger = logging.getLogger()
+    logger.setLevel(logging.INFO)
+    formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
+    log_file = xp_path + '/log.txt'
+    file_handler = logging.FileHandler(log_file)
+    file_handler.setLevel(logging.INFO)
+    file_handler.setFormatter(formatter)
+    logger.addHandler(file_handler)
+
+    # Print paths
+    logger.info('Log file is %s' % log_file)
+    logger.info('Data path is %s' % data_path)
+    logger.info('Export path is %s' % xp_path)
+
+    # Print experimental setup
+    logger.info('Dataset: %s' % dataset_name)
+    logger.info('Normal class: %d' % normal_class)
+    logger.info('Ratio of labeled normal train samples: %.2f' % ratio_known_normal)
+    logger.info('Ratio of labeled anomalous samples: %.2f' % ratio_known_outlier)
+    logger.info('Pollution ratio of unlabeled train data: %.2f' % ratio_pollution)
+    if n_known_outlier_classes == 1:
+        logger.info('Known anomaly class: %d' % known_outlier_class)
+    else:
+        logger.info('Number of known anomaly classes: %d' % n_known_outlier_classes)
+    logger.info('Network: %s' % net_name)
+
+    # If specified, load experiment config from JSON-file
+    if load_config:
+        cfg.load_config(import_json=load_config)
+        logger.info('Loaded configuration from %s.' % load_config)
+
+    # Print model configuration
+    logger.info('Eta-parameter: %.2f' % cfg.settings['eta'])
+
+    # Set seed
+    if cfg.settings['seed'] != -1:
+        random.seed(cfg.settings['seed'])
+        np.random.seed(cfg.settings['seed'])
+        torch.manual_seed(cfg.settings['seed'])
+        torch.cuda.manual_seed(cfg.settings['seed'])
+        torch.backends.cudnn.deterministic = True
+        logger.info('Set seed to %d.' % cfg.settings['seed'])
+
+    # Default device to 'cpu' if cuda is not available
+    if not torch.cuda.is_available():
+        device = 'cpu'
+    # Set the number of threads used for parallelizing CPU operations
+    if num_threads > 0:
+        torch.set_num_threads(num_threads)
+    logger.info('Computation device: %s' % device)
+    logger.info('Number of threads: %d' % num_threads)
+    logger.info('Number of dataloader workers: %d' % n_jobs_dataloader)
+
+    # Load data
+    dataset = load_dataset(dataset_name, data_path, normal_class, known_outlier_class, n_known_outlier_classes,
+                           ratio_known_normal, ratio_known_outlier, ratio_pollution,
+                           random_state=np.random.RandomState(cfg.settings['seed']))
+    # Log random sample of known anomaly classes if more than 1 class
+    if n_known_outlier_classes > 1:
+        logger.info('Known anomaly classes: %s' % (dataset.known_outlier_classes,))
+
+    # Initialize DeepSAD model and set neural network phi
+    deepSAD = DeepSAD(cfg.settings['eta'])
+    deepSAD.set_network(net_name)
+
+    # If specified, load Deep SAD model (center c, network weights, and possibly autoencoder weights)
+    if load_model:
+        deepSAD.load_model(model_path=load_model, load_ae=True, map_location=device)
+        logger.info('Loading model from %s.' % load_model)
+
+    logger.info('Pretraining: %s' % pretrain)
+    if pretrain:
+        # Log pretraining details
+        logger.info('Pretraining optimizer: %s' % cfg.settings['ae_optimizer_name'])
+        logger.info('Pretraining learning rate: %g' % cfg.settings['ae_lr'])
+        logger.info('Pretraining epochs: %d' % cfg.settings['ae_n_epochs'])
+        logger.info('Pretraining learning rate scheduler milestones: %s' % (cfg.settings['ae_lr_milestone'],))
+        logger.info('Pretraining batch size: %d' % cfg.settings['ae_batch_size'])
+        logger.info('Pretraining weight decay: %g' % cfg.settings['ae_weight_decay'])
+
+        # Pretrain model on dataset (via autoencoder)
+        deepSAD.pretrain(dataset,
+                         optimizer_name=cfg.settings['ae_optimizer_name'],
+                         lr=cfg.settings['ae_lr'],
+                         n_epochs=cfg.settings['ae_n_epochs'],
+                         lr_milestones=cfg.settings['ae_lr_milestone'],
+                         batch_size=cfg.settings['ae_batch_size'],
+                         weight_decay=cfg.settings['ae_weight_decay'],
+                         device=device,
+                         n_jobs_dataloader=n_jobs_dataloader)
+
+        # Save pretraining results
+        deepSAD.save_ae_results(export_json=xp_path + '/ae_results.json')
+
+    # Log training details
+    logger.info('Training optimizer: %s' % cfg.settings['optimizer_name'])
+    logger.info('Training learning rate: %g' % cfg.settings['lr'])
+    logger.info('Training epochs: %d' % cfg.settings['n_epochs'])
+    logger.info('Training learning rate scheduler milestones: %s' % (cfg.settings['lr_milestone'],))
+    logger.info('Training batch size: %d' % cfg.settings['batch_size'])
+    logger.info('Training weight decay: %g' % cfg.settings['weight_decay'])
+
+    # Train model on dataset
+    deepSAD.train(dataset,
+                  optimizer_name=cfg.settings['optimizer_name'],
+                  lr=cfg.settings['lr'],
+                  n_epochs=cfg.settings['n_epochs'],
+                  lr_milestones=cfg.settings['lr_milestone'],
+                  batch_size=cfg.settings['batch_size'],
+                  weight_decay=cfg.settings['weight_decay'],
+                  device=device,
+                  n_jobs_dataloader=n_jobs_dataloader)
+
+    # Test model
+    deepSAD.test(dataset, device=device, n_jobs_dataloader=n_jobs_dataloader)
+
+    # Save results, model, and configuration
+    deepSAD.save_results(export_json=xp_path + '/results.json')
+    deepSAD.save_model(export_model=xp_path + '/model.tar')
+    cfg.save_config(export_json=xp_path + '/config.json')
+
+    # Plot most anomalous and most normal test samples
+    indices, labels, scores = zip(*deepSAD.results['test_scores'])
+    indices, labels, scores = np.array(indices), np.array(labels), np.array(scores)
+    idx_all_sorted = indices[np.argsort(scores)]  # from lowest to highest score
+    idx_normal_sorted = indices[labels == 0][np.argsort(scores[labels == 0])]  # from lowest to highest score
+
+    if dataset_name in ('mnist', 'fmnist', 'cifar10'):
+
+        if dataset_name in ('mnist', 'fmnist'):
+            X_all_low = dataset.test_set.data[idx_all_sorted[:32], ...].unsqueeze(1)
+            X_all_high = dataset.test_set.data[idx_all_sorted[-32:], ...].unsqueeze(1)
+            X_normal_low = dataset.test_set.data[idx_normal_sorted[:32], ...].unsqueeze(1)
+            X_normal_high = dataset.test_set.data[idx_normal_sorted[-32:], ...].unsqueeze(1)
+
+        if dataset_name == 'cifar10':
+            X_all_low = torch.tensor(np.transpose(dataset.test_set.data[idx_all_sorted[:32], ...], (0,3,1,2)))
+            X_all_high = torch.tensor(np.transpose(dataset.test_set.data[idx_all_sorted[-32:], ...], (0,3,1,2)))
+            X_normal_low = torch.tensor(np.transpose(dataset.test_set.data[idx_normal_sorted[:32], ...], (0,3,1,2)))
+            X_normal_high = torch.tensor(np.transpose(dataset.test_set.data[idx_normal_sorted[-32:], ...], (0,3,1,2)))
+
+        plot_images_grid(X_all_low, export_img=xp_path + '/all_low', padding=2)
+        plot_images_grid(X_all_high, export_img=xp_path + '/all_high', padding=2)
+        plot_images_grid(X_normal_low, export_img=xp_path + '/normals_low', padding=2)
+        plot_images_grid(X_normal_high, export_img=xp_path + '/normals_high', padding=2)
+
+
+if __name__ == '__main__':
+    main()

Deep-SAD-PyTorch/src/networks/__init__.py

@@ -0,0 +1,10 @@
+from .main import build_network, build_autoencoder
+from .mnist_LeNet import MNIST_LeNet, MNIST_LeNet_Decoder, MNIST_LeNet_Autoencoder
+from .fmnist_LeNet import FashionMNIST_LeNet, FashionMNIST_LeNet_Decoder, FashionMNIST_LeNet_Autoencoder
+from .cifar10_LeNet import CIFAR10_LeNet, CIFAR10_LeNet_Decoder, CIFAR10_LeNet_Autoencoder
+from .mlp import MLP, MLP_Decoder, MLP_Autoencoder
+from .layers.stochastic import GaussianSample
+from .layers.standard import Standardize
+from .inference.distributions import log_standard_gaussian, log_gaussian, log_standard_categorical
+from .vae import VariationalAutoencoder, Encoder, Decoder
+from .dgm import DeepGenerativeModel, StackedDeepGenerativeModel

Deep-SAD-PyTorch/src/networks/cifar10_LeNet.py

@@ -0,0 +1,82 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from base.base_net import BaseNet
+
+
+class CIFAR10_LeNet(BaseNet):
+
+    def __init__(self, rep_dim=128):
+        super().__init__()
+
+        self.rep_dim = rep_dim
+        self.pool = nn.MaxPool2d(2, 2)
+
+        self.conv1 = nn.Conv2d(3, 32, 5, bias=False, padding=2)
+        self.bn2d1 = nn.BatchNorm2d(32, eps=1e-04, affine=False)
+        self.conv2 = nn.Conv2d(32, 64, 5, bias=False, padding=2)
+        self.bn2d2 = nn.BatchNorm2d(64, eps=1e-04, affine=False)
+        self.conv3 = nn.Conv2d(64, 128, 5, bias=False, padding=2)
+        self.bn2d3 = nn.BatchNorm2d(128, eps=1e-04, affine=False)
+        self.fc1 = nn.Linear(128 * 4 * 4, self.rep_dim, bias=False)
+
+    def forward(self, x):
+        x = x.view(-1, 3, 32, 32)
+        x = self.conv1(x)
+        x = self.pool(F.leaky_relu(self.bn2d1(x)))
+        x = self.conv2(x)
+        x = self.pool(F.leaky_relu(self.bn2d2(x)))
+        x = self.conv3(x)
+        x = self.pool(F.leaky_relu(self.bn2d3(x)))
+        x = x.view(int(x.size(0)), -1)
+        x = self.fc1(x)
+        return x
+
+
+class CIFAR10_LeNet_Decoder(BaseNet):
+
+    def __init__(self, rep_dim=128):
+        super().__init__()
+
+        self.rep_dim = rep_dim
+
+        self.deconv1 = nn.ConvTranspose2d(int(self.rep_dim / (4 * 4)), 128, 5, bias=False, padding=2)
+        nn.init.xavier_uniform_(self.deconv1.weight, gain=nn.init.calculate_gain('leaky_relu'))
+        self.bn2d4 = nn.BatchNorm2d(128, eps=1e-04, affine=False)
+        self.deconv2 = nn.ConvTranspose2d(128, 64, 5, bias=False, padding=2)
+        nn.init.xavier_uniform_(self.deconv2.weight, gain=nn.init.calculate_gain('leaky_relu'))
+        self.bn2d5 = nn.BatchNorm2d(64, eps=1e-04, affine=False)
+        self.deconv3 = nn.ConvTranspose2d(64, 32, 5, bias=False, padding=2)
+        nn.init.xavier_uniform_(self.deconv3.weight, gain=nn.init.calculate_gain('leaky_relu'))
+        self.bn2d6 = nn.BatchNorm2d(32, eps=1e-04, affine=False)
+        self.deconv4 = nn.ConvTranspose2d(32, 3, 5, bias=False, padding=2)
+        nn.init.xavier_uniform_(self.deconv4.weight, gain=nn.init.calculate_gain('leaky_relu'))
+
+    def forward(self, x):
+        x = x.view(int(x.size(0)), int(self.rep_dim / (4 * 4)), 4, 4)
+        x = F.leaky_relu(x)
+        x = self.deconv1(x)
+        x = F.interpolate(F.leaky_relu(self.bn2d4(x)), scale_factor=2)
+        x = self.deconv2(x)
+        x = F.interpolate(F.leaky_relu(self.bn2d5(x)), scale_factor=2)
+        x = self.deconv3(x)
+        x = F.interpolate(F.leaky_relu(self.bn2d6(x)), scale_factor=2)
+        x = self.deconv4(x)
+        x = torch.sigmoid(x)
+        return x
+
+
+class CIFAR10_LeNet_Autoencoder(BaseNet):
+
+    def __init__(self, rep_dim=128):
+        super().__init__()
+
+        self.rep_dim = rep_dim
+        self.encoder = CIFAR10_LeNet(rep_dim=rep_dim)
+        self.decoder = CIFAR10_LeNet_Decoder(rep_dim=rep_dim)
+
+    def forward(self, x):
+        x = self.encoder(x)
+        x = self.decoder(x)
+        return x

Deep-SAD-PyTorch/src/networks/dgm.py

@@ -0,0 +1,123 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from torch.nn import init
+from .vae import VariationalAutoencoder, Encoder, Decoder
+
+
+# Acknowledgements: https://github.com/wohlert/semi-supervised-pytorch
+class Classifier(nn.Module):
+    """
+    Classifier network, i.e. q(y|x), for two classes (0: normal, 1: outlier)
+
+    :param net: neural network class to use (as parameter to use the same network over different shallow_ssad)
+    """
+
+    def __init__(self, net, dims=None):
+        super(Classifier, self).__init__()
+        self.dims = dims
+        if dims is None:
+            self.net = net()
+            self.logits = nn.Linear(self.net.rep_dim, 2)
+        else:
+            [x_dim, h_dim, y_dim] = dims
+            self.dense = nn.Linear(x_dim, h_dim)
+            self.logits = nn.Linear(h_dim, y_dim)
+
+    def forward(self, x):
+        if self.dims is None:
+            x = self.net(x)
+        else:
+            x = F.relu(self.dense(x))
+        x = F.softmax(self.logits(x), dim=-1)
+        return x
+
+
+class DeepGenerativeModel(VariationalAutoencoder):
+    """
+    M2 model from the paper 'Semi-Supervised Learning with Deep Generative Models' (Kingma et al., 2014).
+
+    The 'Generative semi-supervised model' (M2) is a probabilistic model that incorporates label information in both
+    inference and generation.
+
+    :param dims: dimensions of the model given by [input_dim, label_dim, latent_dim, [hidden_dims]].
+    :param classifier_net: classifier network class to use.
+    """
+
+    def __init__(self, dims, classifier_net=None):
+        [x_dim, self.y_dim, z_dim, h_dim] = dims
+        super(DeepGenerativeModel, self).__init__([x_dim, z_dim, h_dim])
+
+        self.encoder = Encoder([x_dim + self.y_dim, h_dim, z_dim])
+        self.decoder = Decoder([z_dim + self.y_dim, list(reversed(h_dim)), x_dim])
+        if classifier_net is None:
+            self.classifier = Classifier(net=None, dims=[x_dim, h_dim[0], self.y_dim])
+        else:
+            self.classifier = Classifier(classifier_net)
+
+        # Init linear layers
+        for m in self.modules():
+            if isinstance(m, nn.Linear):
+                init.xavier_normal_(m.weight.data)
+                if m.bias is not None:
+                    m.bias.data.zero_()
+
+    def forward(self, x, y):
+        z, q_mu, q_log_var = self.encoder(torch.cat((x, y), dim=1))
+        self.kl_divergence = self._kld(z, (q_mu, q_log_var))
+        rec = self.decoder(torch.cat((z, y), dim=1))
+
+        return rec
+
+    def classify(self, x):
+        logits = self.classifier(x)
+        return logits
+
+    def sample(self, z, y):
+        """
+        Samples from the Decoder to generate an x.
+
+        :param z: latent normal variable
+        :param y: label (one-hot encoded)
+        :return: x
+        """
+        y = y.float()
+        x = self.decoder(torch.cat((z, y), dim=1))
+        return x
+
+
+class StackedDeepGenerativeModel(DeepGenerativeModel):
+    def __init__(self, dims, features):
+        """
+        M1+M2 model as described in (Kingma et al., 2014).
+
+        :param dims: dimensions of the model given by [input_dim, label_dim, latent_dim, [hidden_dims]].
+        :param classifier_net: classifier network class to use.
+        :param features: a pre-trained M1 model of class 'VariationalAutoencoder' trained on the same dataset.
+        """
+        [x_dim, y_dim, z_dim, h_dim] = dims
+        super(StackedDeepGenerativeModel, self).__init__([features.z_dim, y_dim, z_dim, h_dim])
+
+        # Be sure to reconstruct with the same dimensions
+        in_features = self.decoder.reconstruction.in_features
+        self.decoder.reconstruction = nn.Linear(in_features, x_dim)
+
+        # Make vae feature model untrainable by freezing parameters
+        self.features = features
+        self.features.train(False)
+
+        for param in self.features.parameters():
+            param.requires_grad = False
+
+    def forward(self, x, y):
+        # Sample a new latent x from the M1 model
+        x_sample, _, _ = self.features.encoder(x)
+
+        # Use the sample as new input to M2
+        return super(StackedDeepGenerativeModel, self).forward(x_sample, y)
+
+    def classify(self, x):
+        _, x, _ = self.features.encoder(x)
+        logits = self.classifier(x)
+        return logits

Deep-SAD-PyTorch/src/networks/fmnist_LeNet.py

@@ -0,0 +1,76 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from base.base_net import BaseNet
+
+
+class FashionMNIST_LeNet(BaseNet):
+
+    def __init__(self, rep_dim=64):
+        super().__init__()
+
+        self.rep_dim = rep_dim
+        self.pool = nn.MaxPool2d(2, 2)
+
+        self.conv1 = nn.Conv2d(1, 16, 5, bias=False, padding=2)
+        self.bn2d1 = nn.BatchNorm2d(16, eps=1e-04, affine=False)
+        self.conv2 = nn.Conv2d(16, 32, 5, bias=False, padding=2)
+        self.bn2d2 = nn.BatchNorm2d(32, eps=1e-04, affine=False)
+        self.fc1 = nn.Linear(32 * 7 * 7, 128, bias=False)
+        self.bn1d1 = nn.BatchNorm1d(128, eps=1e-04, affine=False)
+        self.fc2 = nn.Linear(128, self.rep_dim, bias=False)
+
+    def forward(self, x):
+        x = x.view(-1, 1, 28, 28)
+        x = self.conv1(x)
+        x = self.pool(F.leaky_relu(self.bn2d1(x)))
+        x = self.conv2(x)
+        x = self.pool(F.leaky_relu(self.bn2d2(x)))
+        x = x.view(int(x.size(0)), -1)
+        x = F.leaky_relu(self.bn1d1(self.fc1(x)))
+        x = self.fc2(x)
+        return x
+
+
+class FashionMNIST_LeNet_Decoder(BaseNet):
+
+    def __init__(self, rep_dim=64):
+        super().__init__()
+
+        self.rep_dim = rep_dim
+
+        self.fc3 = nn.Linear(self.rep_dim, 128, bias=False)
+        self.bn1d2 = nn.BatchNorm1d(128, eps=1e-04, affine=False)
+        self.deconv1 = nn.ConvTranspose2d(8, 32, 5, bias=False, padding=2)
+        self.bn2d3 = nn.BatchNorm2d(32, eps=1e-04, affine=False)
+        self.deconv2 = nn.ConvTranspose2d(32, 16, 5, bias=False, padding=3)
+        self.bn2d4 = nn.BatchNorm2d(16, eps=1e-04, affine=False)
+        self.deconv3 = nn.ConvTranspose2d(16, 1, 5, bias=False, padding=2)
+
+    def forward(self, x):
+        x = self.bn1d2(self.fc3(x))
+        x = x.view(int(x.size(0)), int(128 / 16), 4, 4)
+        x = F.interpolate(F.leaky_relu(x), scale_factor=2)
+        x = self.deconv1(x)
+        x = F.interpolate(F.leaky_relu(self.bn2d3(x)), scale_factor=2)
+        x = self.deconv2(x)
+        x = F.interpolate(F.leaky_relu(self.bn2d4(x)), scale_factor=2)
+        x = self.deconv3(x)
+        x = torch.sigmoid(x)
+        return x
+
+
+class FashionMNIST_LeNet_Autoencoder(BaseNet):
+
+    def __init__(self, rep_dim=64):
+        super().__init__()
+
+        self.rep_dim = rep_dim
+        self.encoder = FashionMNIST_LeNet(rep_dim=rep_dim)
+        self.decoder = FashionMNIST_LeNet_Decoder(rep_dim=rep_dim)
+
+    def forward(self, x):
+        x = self.encoder(x)
+        x = self.decoder(x)
+        return x

Deep-SAD-PyTorch/src/networks/inference/distributions.py

@@ -0,0 +1,41 @@
+import math
+import torch
+import torch.nn.functional as F
+
+
+# Acknowledgements: https://github.com/wohlert/semi-supervised-pytorch
+def log_standard_gaussian(x):
+    """
+    Evaluates the log pdf of a standard normal distribution at x.
+
+    :param x: point to evaluate
+    :return: log N(x|0,I)
+    """
+    return torch.sum(-0.5 * math.log(2 * math.pi) - x ** 2 / 2, dim=-1)
+
+
+def log_gaussian(x, mu, log_var):
+    """
+    Evaluates the log pdf of a normal distribution parametrized by mu and log_var at x.
+
+    :param x: point to evaluate
+    :param mu: mean
+    :param log_var: log variance
+    :return: log N(x|µ,σI)
+    """
+    log_pdf = -0.5 * math.log(2 * math.pi) - log_var / 2 - (x - mu)**2 / (2 * torch.exp(log_var))
+    return torch.sum(log_pdf, dim=-1)
+
+
+def log_standard_categorical(p):
+    """
+    Computes the cross-entropy between a (one-hot) categorical vector and a standard (uniform) categorical distribution.
+    :param p: one-hot categorical distribution
+    :return: H(p,u)
+    """
+    eps = 1e-8
+    prior = F.softmax(torch.ones_like(p), dim=1)  # Uniform prior over y
+    prior.requires_grad = False
+    cross_entropy = -torch.sum(p * torch.log(prior + eps), dim=1)
+
+    return cross_entropy

Deep-SAD-PyTorch/src/networks/layers/standard.py

@@ -0,0 +1,52 @@
+import torch
+
+from torch.nn import Module
+from torch.nn import init
+from torch.nn.parameter import Parameter
+
+
+# Acknowledgements: https://github.com/wohlert/semi-supervised-pytorch
+class Standardize(Module):
+    """
+    Applies (element-wise) standardization with trainable translation parameter μ and scale parameter σ, i.e. computes
+    (x - μ) / σ where '/' is applied element-wise.
+
+    Args:
+        in_features: size of each input sample
+        out_features: size of each output sample
+        bias: If set to False, the layer will not learn a translation parameter μ.
+            Default: ``True``
+
+    Attributes:
+        mu: the learnable translation parameter μ.
+        std: the learnable scale parameter σ.
+    """
+    __constants__ = ['mu']
+
+    def __init__(self, in_features, bias=True, eps=1e-6):
+        super(Standardize, self).__init__()
+        self.in_features = in_features
+        self.out_features = in_features
+        self.eps = eps
+        self.std = Parameter(torch.Tensor(in_features))
+        if bias:
+            self.mu = Parameter(torch.Tensor(in_features))
+        else:
+            self.register_parameter('mu', None)
+        self.reset_parameters()
+
+    def reset_parameters(self):
+        init.constant_(self.std, 1)
+        if self.mu is not None:
+            init.constant_(self.mu, 0)
+
+    def forward(self, x):
+        if self.mu is not None:
+            x -= self.mu
+        x = torch.div(x, self.std + self.eps)
+        return x
+
+    def extra_repr(self):
+        return 'in_features={}, out_features={}, bias={}'.format(
+            self.in_features, self.out_features, self.mu is not None
+        )

Deep-SAD-PyTorch/src/networks/layers/stochastic.py

@@ -0,0 +1,53 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from torch.autograd import Variable
+
+
+# Acknowledgements: https://github.com/wohlert/semi-supervised-pytorch
+class Stochastic(nn.Module):
+    """
+    Base stochastic layer that uses the reparametrization trick (Kingma and Welling, 2013) to draw a sample from a
+    distribution parametrized by mu and log_var.
+    """
+
+    def __init__(self):
+        super(Stochastic, self).__init__()
+
+    def reparametrize(self, mu, log_var):
+        epsilon = Variable(torch.randn(mu.size()), requires_grad=False)
+
+        if mu.is_cuda:
+            epsilon = epsilon.to(mu.device)
+
+        # log_std = 0.5 * log_var
+        # std = exp(log_std)
+        std = log_var.mul(0.5).exp_()
+
+        # z = std * epsilon + mu
+        z = mu.addcmul(std, epsilon)
+
+        return z
+
+    def forward(self, x):
+        raise NotImplementedError
+
+
+class GaussianSample(Stochastic):
+    """
+    Layer that represents a sample from a Gaussian distribution.
+    """
+
+    def __init__(self, in_features, out_features):
+        super(GaussianSample, self).__init__()
+        self.in_features = in_features
+        self.out_features = out_features
+
+        self.mu = nn.Linear(in_features, out_features)
+        self.log_var = nn.Linear(in_features, out_features)
+
+    def forward(self, x):
+        mu = self.mu(x)
+        log_var = F.softplus(self.log_var(x))
+        return self.reparametrize(mu, log_var), mu, log_var

Deep-SAD-PyTorch/src/networks/main.py

@@ -0,0 +1,138 @@
+from .mnist_LeNet import MNIST_LeNet, MNIST_LeNet_Autoencoder
+from .fmnist_LeNet import FashionMNIST_LeNet, FashionMNIST_LeNet_Autoencoder
+from .cifar10_LeNet import CIFAR10_LeNet, CIFAR10_LeNet_Autoencoder
+from .mlp import MLP, MLP_Autoencoder
+from .vae import VariationalAutoencoder
+from .dgm import DeepGenerativeModel, StackedDeepGenerativeModel
+
+
+def build_network(net_name, ae_net=None):
+    """Builds the neural network."""
+
+    implemented_networks = ('mnist_LeNet', 'mnist_DGM_M2', 'mnist_DGM_M1M2',
+                            'fmnist_LeNet', 'fmnist_DGM_M2', 'fmnist_DGM_M1M2',
+                            'cifar10_LeNet', 'cifar10_DGM_M2', 'cifar10_DGM_M1M2',
+                            'arrhythmia_mlp', 'cardio_mlp', 'satellite_mlp', 'satimage-2_mlp', 'shuttle_mlp',
+                            'thyroid_mlp',
+                            'arrhythmia_DGM_M2', 'cardio_DGM_M2', 'satellite_DGM_M2', 'satimage-2_DGM_M2',
+                            'shuttle_DGM_M2', 'thyroid_DGM_M2')
+    assert net_name in implemented_networks
+
+    net = None
+
+    if net_name == 'mnist_LeNet':
+        net = MNIST_LeNet()
+
+    if net_name == 'mnist_DGM_M2':
+        net = DeepGenerativeModel([1*28*28, 2, 32, [128, 64]], classifier_net=MNIST_LeNet)
+
+    if net_name == 'mnist_DGM_M1M2':
+        net = StackedDeepGenerativeModel([1*28*28, 2, 32, [128, 64]], features=ae_net)
+
+    if net_name == 'fmnist_LeNet':
+        net = FashionMNIST_LeNet()
+
+    if net_name == 'fmnist_DGM_M2':
+        net = DeepGenerativeModel([1*28*28, 2, 64, [256, 128]], classifier_net=FashionMNIST_LeNet)
+
+    if net_name == 'fmnist_DGM_M1M2':
+        net = StackedDeepGenerativeModel([1*28*28, 2, 64, [256, 128]], features=ae_net)
+
+    if net_name == 'cifar10_LeNet':
+        net = CIFAR10_LeNet()
+
+    if net_name == 'cifar10_DGM_M2':
+        net = DeepGenerativeModel([3*32*32, 2, 128, [512, 256]], classifier_net=CIFAR10_LeNet)
+
+    if net_name == 'cifar10_DGM_M1M2':
+        net = StackedDeepGenerativeModel([3*32*32, 2, 128, [512, 256]], features=ae_net)
+
+    if net_name == 'arrhythmia_mlp':
+        net = MLP(x_dim=274, h_dims=[128, 64], rep_dim=32, bias=False)
+
+    if net_name == 'cardio_mlp':
+        net = MLP(x_dim=21, h_dims=[32, 16], rep_dim=8, bias=False)
+
+    if net_name == 'satellite_mlp':
+        net = MLP(x_dim=36, h_dims=[32, 16], rep_dim=8, bias=False)
+
+    if net_name == 'satimage-2_mlp':
+        net = MLP(x_dim=36, h_dims=[32, 16], rep_dim=8, bias=False)
+
+    if net_name == 'shuttle_mlp':
+        net = MLP(x_dim=9, h_dims=[32, 16], rep_dim=8, bias=False)
+
+    if net_name == 'thyroid_mlp':
+        net = MLP(x_dim=6, h_dims=[32, 16], rep_dim=4, bias=False)
+
+    if net_name == 'arrhythmia_DGM_M2':
+        net = DeepGenerativeModel([274, 2, 32, [128, 64]])
+
+    if net_name == 'cardio_DGM_M2':
+        net = DeepGenerativeModel([21, 2, 8, [32, 16]])
+
+    if net_name == 'satellite_DGM_M2':
+        net = DeepGenerativeModel([36, 2, 8, [32, 16]])
+
+    if net_name == 'satimage-2_DGM_M2':
+        net = DeepGenerativeModel([36, 2, 8, [32, 16]])
+
+    if net_name == 'shuttle_DGM_M2':
+        net = DeepGenerativeModel([9, 2, 8, [32, 16]])
+
+    if net_name == 'thyroid_DGM_M2':
+        net = DeepGenerativeModel([6, 2, 4, [32, 16]])
+
+    return net
+
+
+def build_autoencoder(net_name):
+    """Builds the corresponding autoencoder network."""
+
+    implemented_networks = ('mnist_LeNet', 'mnist_DGM_M1M2',
+                            'fmnist_LeNet', 'fmnist_DGM_M1M2',
+                            'cifar10_LeNet', 'cifar10_DGM_M1M2',
+                            'arrhythmia_mlp', 'cardio_mlp', 'satellite_mlp', 'satimage-2_mlp', 'shuttle_mlp',
+                            'thyroid_mlp')
+
+    assert net_name in implemented_networks
+
+    ae_net = None
+
+    if net_name == 'mnist_LeNet':
+        ae_net = MNIST_LeNet_Autoencoder()
+
+    if net_name == 'mnist_DGM_M1M2':
+        ae_net = VariationalAutoencoder([1*28*28, 32, [128, 64]])
+
+    if net_name == 'fmnist_LeNet':
+        ae_net = FashionMNIST_LeNet_Autoencoder()
+
+    if net_name == 'fmnist_DGM_M1M2':
+        ae_net = VariationalAutoencoder([1*28*28, 64, [256, 128]])
+
+    if net_name == 'cifar10_LeNet':
+        ae_net = CIFAR10_LeNet_Autoencoder()
+
+    if net_name == 'cifar10_DGM_M1M2':
+        ae_net = VariationalAutoencoder([3*32*32, 128, [512, 256]])
+
+    if net_name == 'arrhythmia_mlp':
+        ae_net = MLP_Autoencoder(x_dim=274, h_dims=[128, 64], rep_dim=32, bias=False)
+
+    if net_name == 'cardio_mlp':
+        ae_net = MLP_Autoencoder(x_dim=21, h_dims=[32, 16], rep_dim=8, bias=False)
+
+    if net_name == 'satellite_mlp':
+        ae_net = MLP_Autoencoder(x_dim=36, h_dims=[32, 16], rep_dim=8, bias=False)
+
+    if net_name == 'satimage-2_mlp':
+        ae_net = MLP_Autoencoder(x_dim=36, h_dims=[32, 16], rep_dim=8, bias=False)
+
+    if net_name == 'shuttle_mlp':
+        ae_net = MLP_Autoencoder(x_dim=9, h_dims=[32, 16], rep_dim=8, bias=False)
+
+    if net_name == 'thyroid_mlp':
+        ae_net = MLP_Autoencoder(x_dim=6, h_dims=[32, 16], rep_dim=4, bias=False)
+
+    return ae_net

Deep-SAD-PyTorch/src/networks/mlp.py

@@ -0,0 +1,76 @@
+import torch.nn as nn
+import torch.nn.functional as F
+
+from base.base_net import BaseNet
+
+
+class MLP(BaseNet):
+
+    def __init__(self, x_dim, h_dims=[128, 64], rep_dim=32, bias=False):
+        super().__init__()
+
+        self.rep_dim = rep_dim
+
+        neurons = [x_dim, *h_dims]
+        layers = [Linear_BN_leakyReLU(neurons[i - 1], neurons[i], bias=bias) for i in range(1, len(neurons))]
+
+        self.hidden = nn.ModuleList(layers)
+        self.code = nn.Linear(h_dims[-1], rep_dim, bias=bias)
+
+    def forward(self, x):
+        x = x.view(int(x.size(0)), -1)
+        for layer in self.hidden:
+            x = layer(x)
+        return self.code(x)
+
+
+class MLP_Decoder(BaseNet):
+
+    def __init__(self, x_dim, h_dims=[64, 128], rep_dim=32, bias=False):
+        super().__init__()
+
+        self.rep_dim = rep_dim
+
+        neurons = [rep_dim, *h_dims]
+        layers = [Linear_BN_leakyReLU(neurons[i - 1], neurons[i], bias=bias) for i in range(1, len(neurons))]
+
+        self.hidden = nn.ModuleList(layers)
+        self.reconstruction = nn.Linear(h_dims[-1], x_dim, bias=bias)
+        self.output_activation = nn.Sigmoid()
+
+    def forward(self, x):
+        x = x.view(int(x.size(0)), -1)
+        for layer in self.hidden:
+            x = layer(x)
+        x = self.reconstruction(x)
+        return self.output_activation(x)
+
+
+class MLP_Autoencoder(BaseNet):
+
+    def __init__(self, x_dim, h_dims=[128, 64], rep_dim=32, bias=False):
+        super().__init__()
+
+        self.rep_dim = rep_dim
+        self.encoder = MLP(x_dim, h_dims, rep_dim, bias)
+        self.decoder = MLP_Decoder(x_dim, list(reversed(h_dims)), rep_dim, bias)
+
+    def forward(self, x):
+        x = self.encoder(x)
+        x = self.decoder(x)
+        return x
+
+
+class Linear_BN_leakyReLU(nn.Module):
+    """
+    A nn.Module that consists of a Linear layer followed by BatchNorm1d and a leaky ReLu activation
+    """
+
+    def __init__(self, in_features, out_features, bias=False, eps=1e-04):
+        super(Linear_BN_leakyReLU, self).__init__()
+
+        self.linear = nn.Linear(in_features, out_features, bias=bias)
+        self.bn = nn.BatchNorm1d(out_features, eps=eps, affine=bias)
+
+    def forward(self, x):
+        return F.leaky_relu(self.bn(self.linear(x)))

Deep-SAD-PyTorch/src/networks/mnist_LeNet.py

@@ -0,0 +1,71 @@
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from base.base_net import BaseNet
+
+
+class MNIST_LeNet(BaseNet):
+
+    def __init__(self, rep_dim=32):
+        super().__init__()
+
+        self.rep_dim = rep_dim
+        self.pool = nn.MaxPool2d(2, 2)
+
+        self.conv1 = nn.Conv2d(1, 8, 5, bias=False, padding=2)
+        self.bn1 = nn.BatchNorm2d(8, eps=1e-04, affine=False)
+        self.conv2 = nn.Conv2d(8, 4, 5, bias=False, padding=2)
+        self.bn2 = nn.BatchNorm2d(4, eps=1e-04, affine=False)
+        self.fc1 = nn.Linear(4 * 7 * 7, self.rep_dim, bias=False)
+
+    def forward(self, x):
+        x = x.view(-1, 1, 28, 28)
+        x = self.conv1(x)
+        x = self.pool(F.leaky_relu(self.bn1(x)))
+        x = self.conv2(x)
+        x = self.pool(F.leaky_relu(self.bn2(x)))
+        x = x.view(int(x.size(0)), -1)
+        x = self.fc1(x)
+        return x
+
+
+class MNIST_LeNet_Decoder(BaseNet):
+
+    def __init__(self, rep_dim=32):
+        super().__init__()
+
+        self.rep_dim = rep_dim
+
+        # Decoder network
+        self.deconv1 = nn.ConvTranspose2d(2, 4, 5, bias=False, padding=2)
+        self.bn3 = nn.BatchNorm2d(4, eps=1e-04, affine=False)
+        self.deconv2 = nn.ConvTranspose2d(4, 8, 5, bias=False, padding=3)
+        self.bn4 = nn.BatchNorm2d(8, eps=1e-04, affine=False)
+        self.deconv3 = nn.ConvTranspose2d(8, 1, 5, bias=False, padding=2)
+
+    def forward(self, x):
+        x = x.view(int(x.size(0)), int(self.rep_dim / 16), 4, 4)
+        x = F.interpolate(F.leaky_relu(x), scale_factor=2)
+        x = self.deconv1(x)
+        x = F.interpolate(F.leaky_relu(self.bn3(x)), scale_factor=2)
+        x = self.deconv2(x)
+        x = F.interpolate(F.leaky_relu(self.bn4(x)), scale_factor=2)
+        x = self.deconv3(x)
+        x = torch.sigmoid(x)
+        return x
+
+
+class MNIST_LeNet_Autoencoder(BaseNet):
+
+    def __init__(self, rep_dim=32):
+        super().__init__()
+
+        self.rep_dim = rep_dim
+        self.encoder = MNIST_LeNet(rep_dim=rep_dim)
+        self.decoder = MNIST_LeNet_Decoder(rep_dim=rep_dim)
+
+    def forward(self, x):
+        x = self.encoder(x)
+        x = self.decoder(x)
+        return x

Deep-SAD-PyTorch/src/networks/vae.py

@@ -0,0 +1,145 @@
+import torch.nn as nn
+import torch.nn.functional as F
+from torch.nn import init
+
+from .layers.stochastic import GaussianSample
+from .inference.distributions import log_standard_gaussian, log_gaussian
+
+
+# Acknowledgements: https://github.com/wohlert/semi-supervised-pytorch
+class Encoder(nn.Module):
+    """
+    Encoder, i.e. the inference network.
+
+    Attempts to infer the latent probability distribution p(z|x) from the data x by fitting a
+    variational distribution q_φ(z|x). Returns the two parameters of the distribution (µ, log σ²).
+
+    :param dims: dimensions of the network given by [input_dim, [hidden_dims], latent_dim].
+    """
+
+    def __init__(self, dims, sample_layer=GaussianSample):
+        super(Encoder, self).__init__()
+
+        [x_dim, h_dim, z_dim] = dims
+        neurons = [x_dim, *h_dim]
+        linear_layers = [nn.Linear(neurons[i-1], neurons[i]) for i in range(1, len(neurons))]
+
+        self.hidden = nn.ModuleList(linear_layers)
+        self.sample = sample_layer(h_dim[-1], z_dim)
+
+    def forward(self, x):
+        for layer in self.hidden:
+            x = F.relu(layer(x))
+        return self.sample(x)
+
+
+class Decoder(nn.Module):
+    """
+    Decoder, i.e. the generative network.
+
+    Generates samples from an approximation p_θ(x|z) of the original distribution p(x)
+    by transforming a latent representation z.
+
+    :param dims: dimensions of the network given by [latent_dim, [hidden_dims], input_dim].
+    """
+
+    def __init__(self, dims):
+        super(Decoder, self).__init__()
+
+        [z_dim, h_dim, x_dim] = dims
+        neurons = [z_dim, *h_dim]
+        linear_layers = [nn.Linear(neurons[i-1], neurons[i]) for i in range(1, len(neurons))]
+
+        self.hidden = nn.ModuleList(linear_layers)
+        self.reconstruction = nn.Linear(h_dim[-1], x_dim)
+        self.output_activation = nn.Sigmoid()
+
+    def forward(self, x):
+        for layer in self.hidden:
+            x = F.relu(layer(x))
+        return self.output_activation(self.reconstruction(x))
+
+
+class VariationalAutoencoder(nn.Module):
+    """
+    Variational Autoencoder (VAE) (Kingma and Welling, 2013) model consisting of an encoder-decoder pair for which
+    a variational distribution is fitted to the encoder.
+    Also known as the M1 model in (Kingma et al., 2014)
+
+    :param  dims: dimensions of the networks given by [input_dim, latent_dim, [hidden_dims]]. Encoder and decoder
+    are build symmetrically.
+    """
+
+    def __init__(self, dims):
+        super(VariationalAutoencoder, self).__init__()
+
+        [x_dim, z_dim, h_dim] = dims
+        self.z_dim = z_dim
+        self.flow = None
+
+        self.encoder = Encoder([x_dim, h_dim, z_dim])
+        self.decoder = Decoder([z_dim, list(reversed(h_dim)), x_dim])
+        self.kl_divergence = 0
+
+        # Init linear layers
+        for m in self.modules():
+            if isinstance(m, nn.Linear):
+                init.xavier_normal_(m.weight.data)
+                if m.bias is not None:
+                    m.bias.data.zero_()
+
+    def _kld(self, z, q_param, p_param=None):
+        """
+        Computes the KL-divergence of some latent variable z.
+
+        KL(q||p) = - ∫ q(z) log [ p(z) / q(z) ] = - E_q[ log p(z) - log q(z) ]
+
+        :param z: sample from q-distribuion
+        :param q_param: (mu, log_var) of the q-distribution
+        :param p_param: (mu, log_var) of the p-distribution
+        :return: KL(q||p)
+        """
+        (mu, log_var) = q_param
+
+        if self.flow is not None:
+            f_z, log_det_z = self.flow(z)
+            qz = log_gaussian(z, mu, log_var) - sum(log_det_z)
+            z = f_z
+        else:
+            qz = log_gaussian(z, mu, log_var)
+
+        if p_param is None:
+            pz = log_standard_gaussian(z)
+        else:
+            (mu, log_var) = p_param
+            pz = log_gaussian(z, mu, log_var)
+
+        kl = qz - pz
+
+        return kl
+
+    def add_flow(self, flow):
+        self.flow = flow
+
+    def forward(self, x, y=None):
+        """
+        Runs a forward pass on a data point through the VAE model to provide its reconstruction and the parameters of
+        the variational approximate distribution q.
+
+        :param x: input data
+        :return: reconstructed input
+        """
+        z, q_mu, q_log_var = self.encoder(x)
+        self.kl_divergence = self._kld(z, (q_mu, q_log_var))
+        rec = self.decoder(z)
+
+        return rec
+
+    def sample(self, z):
+        """
+        Given z ~ N(0, I) generates a sample from the learned distribution based on p_θ(x|z).
+
+        :param z: (torch.autograd.Variable) latent normal variable
+        :return: (torch.autograd.Variable) generated sample
+        """
+        return self.decoder(z)

Deep-SAD-PyTorch/src/optim/DeepSAD_trainer.py

@@ -0,0 +1,173 @@
+from base.base_trainer import BaseTrainer
+from base.base_dataset import BaseADDataset
+from base.base_net import BaseNet
+from torch.utils.data.dataloader import DataLoader
+from sklearn.metrics import roc_auc_score
+
+import logging
+import time
+import torch
+import torch.optim as optim
+import numpy as np
+
+
+class DeepSADTrainer(BaseTrainer):
+
+    def __init__(self, c, eta: float, optimizer_name: str = 'adam', lr: float = 0.001, n_epochs: int = 150,
+                 lr_milestones: tuple = (), batch_size: int = 128, weight_decay: float = 1e-6, device: str = 'cuda',
+                 n_jobs_dataloader: int = 0):
+        super().__init__(optimizer_name, lr, n_epochs, lr_milestones, batch_size, weight_decay, device,
+                         n_jobs_dataloader)
+
+        # Deep SAD parameters
+        self.c = torch.tensor(c, device=self.device) if c is not None else None
+        self.eta = eta
+
+        # Optimization parameters
+        self.eps = 1e-6
+
+        # Results
+        self.train_time = None
+        self.test_auc = None
+        self.test_time = None
+        self.test_scores = None
+
+    def train(self, dataset: BaseADDataset, net: BaseNet):
+        logger = logging.getLogger()
+
+        # Get train data loader
+        train_loader, _ = dataset.loaders(batch_size=self.batch_size, num_workers=self.n_jobs_dataloader)
+
+        # Set device for network
+        net = net.to(self.device)
+
+        # Set optimizer (Adam optimizer for now)
+        optimizer = optim.Adam(net.parameters(), lr=self.lr, weight_decay=self.weight_decay)
+
+        # Set learning rate scheduler
+        scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=self.lr_milestones, gamma=0.1)
+
+        # Initialize hypersphere center c (if c not loaded)
+        if self.c is None:
+            logger.info('Initializing center c...')
+            self.c = self.init_center_c(train_loader, net)
+            logger.info('Center c initialized.')
+
+        # Training
+        logger.info('Starting training...')
+        start_time = time.time()
+        net.train()
+        for epoch in range(self.n_epochs):
+
+            scheduler.step()
+            if epoch in self.lr_milestones:
+                logger.info('  LR scheduler: new learning rate is %g' % float(scheduler.get_lr()[0]))
+
+            epoch_loss = 0.0
+            n_batches = 0
+            epoch_start_time = time.time()
+            for data in train_loader:
+                inputs, _, semi_targets, _ = data
+                inputs, semi_targets = inputs.to(self.device), semi_targets.to(self.device)
+
+                # Zero the network parameter gradients
+                optimizer.zero_grad()
+
+                # Update network parameters via backpropagation: forward + backward + optimize
+                outputs = net(inputs)
+                dist = torch.sum((outputs - self.c) ** 2, dim=1)
+                losses = torch.where(semi_targets == 0, dist, self.eta * ((dist + self.eps) ** semi_targets.float()))
+                loss = torch.mean(losses)
+                loss.backward()
+                optimizer.step()
+
+                epoch_loss += loss.item()
+                n_batches += 1
+
+            # log epoch statistics
+            epoch_train_time = time.time() - epoch_start_time
+            logger.info(f'| Epoch: {epoch + 1:03}/{self.n_epochs:03} | Train Time: {epoch_train_time:.3f}s '
+                        f'| Train Loss: {epoch_loss / n_batches:.6f} |')
+
+        self.train_time = time.time() - start_time
+        logger.info('Training Time: {:.3f}s'.format(self.train_time))
+        logger.info('Finished training.')
+
+        return net
+
+    def test(self, dataset: BaseADDataset, net: BaseNet):
+        logger = logging.getLogger()
+
+        # Get test data loader
+        _, test_loader = dataset.loaders(batch_size=self.batch_size, num_workers=self.n_jobs_dataloader)
+
+        # Set device for network
+        net = net.to(self.device)
+
+        # Testing
+        logger.info('Starting testing...')
+        epoch_loss = 0.0
+        n_batches = 0
+        start_time = time.time()
+        idx_label_score = []
+        net.eval()
+        with torch.no_grad():
+            for data in test_loader:
+                inputs, labels, semi_targets, idx = data
+
+                inputs = inputs.to(self.device)
+                labels = labels.to(self.device)
+                semi_targets = semi_targets.to(self.device)
+                idx = idx.to(self.device)
+
+                outputs = net(inputs)
+                dist = torch.sum((outputs - self.c) ** 2, dim=1)
+                losses = torch.where(semi_targets == 0, dist, self.eta * ((dist + self.eps) ** semi_targets.float()))
+                loss = torch.mean(losses)
+                scores = dist
+
+                # Save triples of (idx, label, score) in a list
+                idx_label_score += list(zip(idx.cpu().data.numpy().tolist(),
+                                            labels.cpu().data.numpy().tolist(),
+                                            scores.cpu().data.numpy().tolist()))
+
+                epoch_loss += loss.item()
+                n_batches += 1
+
+        self.test_time = time.time() - start_time
+        self.test_scores = idx_label_score
+
+        # Compute AUC
+        _, labels, scores = zip(*idx_label_score)
+        labels = np.array(labels)
+        scores = np.array(scores)
+        self.test_auc = roc_auc_score(labels, scores)
+
+        # Log results
+        logger.info('Test Loss: {:.6f}'.format(epoch_loss / n_batches))
+        logger.info('Test AUC: {:.2f}%'.format(100. * self.test_auc))
+        logger.info('Test Time: {:.3f}s'.format(self.test_time))
+        logger.info('Finished testing.')
+
+    def init_center_c(self, train_loader: DataLoader, net: BaseNet, eps=0.1):
+        """Initialize hypersphere center c as the mean from an initial forward pass on the data."""
+        n_samples = 0
+        c = torch.zeros(net.rep_dim, device=self.device)
+
+        net.eval()
+        with torch.no_grad():
+            for data in train_loader:
+                # get the inputs of the batch
+                inputs, _, _, _ = data
+                inputs = inputs.to(self.device)
+                outputs = net(inputs)
+                n_samples += outputs.shape[0]
+                c += torch.sum(outputs, dim=0)
+
+        c /= n_samples
+
+        # If c_i is too close to 0, set to +-eps. Reason: a zero unit can be trivially matched with zero weights.
+        c[(abs(c) < eps) & (c < 0)] = -eps
+        c[(abs(c) < eps) & (c > 0)] = eps
+
+        return c

Deep-SAD-PyTorch/src/optim/SemiDGM_trainer.py

@@ -0,0 +1,188 @@
+from base.base_trainer import BaseTrainer
+from base.base_dataset import BaseADDataset
+from base.base_net import BaseNet
+from optim.variational import SVI, ImportanceWeightedSampler
+from utils.misc import binary_cross_entropy
+from sklearn.metrics import roc_auc_score
+
+import logging
+import time
+import torch
+import torch.optim as optim
+import numpy as np
+
+
+class SemiDeepGenerativeTrainer(BaseTrainer):
+
+    def __init__(self, alpha: float = 0.1, optimizer_name: str = 'adam', lr: float = 0.001, n_epochs: int = 150,
+                 lr_milestones: tuple = (), batch_size: int = 128, weight_decay: float = 1e-6, device: str = 'cuda',
+                 n_jobs_dataloader: int = 0):
+        super().__init__(optimizer_name, lr, n_epochs, lr_milestones, batch_size, weight_decay, device,
+                         n_jobs_dataloader)
+
+        self.alpha = alpha
+
+        # Results
+        self.train_time = None
+        self.test_auc = None
+        self.test_time = None
+        self.test_scores = None
+
+    def train(self, dataset: BaseADDataset, net: BaseNet):
+        logger = logging.getLogger()
+
+        # Get train data loader
+        train_loader, _ = dataset.loaders(batch_size=self.batch_size, num_workers=self.n_jobs_dataloader)
+
+        # Set device
+        net = net.to(self.device)
+
+        # Use importance weighted sampler (Burda et al., 2015) to get a better estimate on the log-likelihood.
+        sampler = ImportanceWeightedSampler(mc=1, iw=1)
+        elbo = SVI(net, likelihood=binary_cross_entropy, sampler=sampler)
+
+        # Set optimizer (Adam optimizer for now)
+        optimizer = optim.Adam(net.parameters(), lr=self.lr, weight_decay=self.weight_decay)
+
+        # Set learning rate scheduler
+        scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=self.lr_milestones, gamma=0.1)
+
+        # Training
+        logger.info('Starting training...')
+        start_time = time.time()
+        net.train()
+        for epoch in range(self.n_epochs):
+
+            scheduler.step()
+            if epoch in self.lr_milestones:
+                logger.info('  LR scheduler: new learning rate is %g' % float(scheduler.get_lr()[0]))
+
+            epoch_loss = 0.0
+            n_batches = 0
+            epoch_start_time = time.time()
+            for data in train_loader:
+                inputs, labels, semi_targets, _ = data
+
+                inputs = inputs.to(self.device)
+                labels = labels.to(self.device)
+                semi_targets = semi_targets.to(self.device)
+
+                # Get labeled and unlabeled data and make labels one-hot
+                inputs = inputs.view(inputs.size(0), -1)
+                x = inputs[semi_targets != 0]
+                u = inputs[semi_targets == 0]
+                y = labels[semi_targets != 0]
+                if y.nelement() > 1:
+                    y_onehot = torch.Tensor(y.size(0), 2).to(self.device)  # two labels: 0: normal, 1: outlier
+                    y_onehot.zero_()
+                    y_onehot.scatter_(1, y.view(-1, 1), 1)
+
+                # Zero the network parameter gradients
+                optimizer.zero_grad()
+
+                # Update network parameters via backpropagation: forward + backward + optimize
+                if y.nelement() < 2:
+                    L = torch.tensor(0.0).to(self.device)
+                else:
+                    L = -elbo(x, y_onehot)
+                U = -elbo(u)
+
+                # Regular cross entropy
+                if y.nelement() < 2:
+                    classication_loss = torch.tensor(0.0).to(self.device)
+                else:
+                    # Add auxiliary classification loss q(y|x)
+                    logits = net.classify(x)
+                    eps = 1e-8
+                    classication_loss = torch.sum(y_onehot * torch.log(logits + eps), dim=1).mean()
+
+                # Overall loss
+                loss = L - self.alpha * classication_loss + U  # J_alpha
+
+                loss.backward()
+                optimizer.step()
+
+                epoch_loss += loss.item()
+                n_batches += 1
+
+            # log epoch statistics
+            epoch_train_time = time.time() - epoch_start_time
+            logger.info(f'| Epoch: {epoch + 1:03}/{self.n_epochs:03} | Train Time: {epoch_train_time:.3f}s '
+                        f'| Train Loss: {epoch_loss / n_batches:.6f} |')
+
+        self.train_time = time.time() - start_time
+        logger.info('Training Time: {:.3f}s'.format(self.train_time))
+        logger.info('Finished training.')
+
+        return net
+
+    def test(self, dataset: BaseADDataset, net: BaseNet):
+        logger = logging.getLogger()
+
+        # Get test data loader
+        _, test_loader = dataset.loaders(batch_size=self.batch_size, num_workers=self.n_jobs_dataloader)
+
+        # Set device
+        net = net.to(self.device)
+
+        # Use importance weighted sampler (Burda et al., 2015) to get a better estimate on the log-likelihood.
+        sampler = ImportanceWeightedSampler(mc=1, iw=1)
+        elbo = SVI(net, likelihood=binary_cross_entropy, sampler=sampler)
+
+        # Testing
+        logger.info('Starting testing...')
+        epoch_loss = 0.0
+        n_batches = 0
+        start_time = time.time()
+        idx_label_score = []
+        net.eval()
+        with torch.no_grad():
+            for data in test_loader:
+                inputs, labels, _, idx = data
+                inputs = inputs.to(self.device)
+                labels = labels.to(self.device)
+                idx = idx.to(self.device)
+
+                # All test data is considered unlabeled
+                inputs = inputs.view(inputs.size(0), -1)
+                u = inputs
+                y = labels
+                y_onehot = torch.Tensor(y.size(0), 2).to(self.device)  # two labels: 0: normal, 1: outlier
+                y_onehot.zero_()
+                y_onehot.scatter_(1, y.view(-1, 1), 1)
+
+                # Compute loss
+                L = -elbo(u, y_onehot)
+                U = -elbo(u)
+
+                logits = net.classify(u)
+                eps = 1e-8
+                classication_loss = -torch.sum(y_onehot * torch.log(logits + eps), dim=1).mean()
+
+                loss = L + self.alpha * classication_loss + U  # J_alpha
+
+                # Compute scores
+                scores = logits[:, 1]  # likelihood/confidence for anomalous class as anomaly score
+
+                # Save triple of (idx, label, score) in a list
+                idx_label_score += list(zip(idx.cpu().data.numpy().tolist(),
+                                            labels.cpu().data.numpy().tolist(),
+                                            scores.cpu().data.numpy().tolist()))
+
+                epoch_loss += loss.item()
+                n_batches += 1
+
+        self.test_time = time.time() - start_time
+        self.test_scores = idx_label_score
+
+        # Compute AUC
+        _, labels, scores = zip(*idx_label_score)
+        labels = np.array(labels)
+        scores = np.array(scores)
+        self.test_auc = roc_auc_score(labels, scores)
+
+        # Log results
+        logger.info('Test Loss: {:.6f}'.format(epoch_loss / n_batches))
+        logger.info('Test AUC: {:.2f}%'.format(100. * self.test_auc))
+        logger.info('Test Time: {:.3f}s'.format(self.test_time))
+        logger.info('Finished testing.')

Deep-SAD-PyTorch/src/optim/__init__.py

@@ -0,0 +1,5 @@
+from .DeepSAD_trainer import DeepSADTrainer
+from .ae_trainer import AETrainer
+from .SemiDGM_trainer import SemiDeepGenerativeTrainer
+from .vae_trainer import VAETrainer
+from .variational import SVI, ImportanceWeightedSampler

Deep-SAD-PyTorch/src/optim/ae_trainer.py

@@ -0,0 +1,136 @@
+from base.base_trainer import BaseTrainer
+from base.base_dataset import BaseADDataset
+from base.base_net import BaseNet
+from sklearn.metrics import roc_auc_score
+
+import logging
+import time
+import torch
+import torch.nn as nn
+import torch.optim as optim
+import numpy as np
+
+
+class AETrainer(BaseTrainer):
+
+    def __init__(self, optimizer_name: str = 'adam', lr: float = 0.001, n_epochs: int = 150, lr_milestones: tuple = (),
+                 batch_size: int = 128, weight_decay: float = 1e-6, device: str = 'cuda', n_jobs_dataloader: int = 0):
+        super().__init__(optimizer_name, lr, n_epochs, lr_milestones, batch_size, weight_decay, device,
+                         n_jobs_dataloader)
+
+        # Results
+        self.train_time = None
+        self.test_auc = None
+        self.test_time = None
+
+    def train(self, dataset: BaseADDataset, ae_net: BaseNet):
+        logger = logging.getLogger()
+
+        # Get train data loader
+        train_loader, _ = dataset.loaders(batch_size=self.batch_size, num_workers=self.n_jobs_dataloader)
+
+        # Set loss
+        criterion = nn.MSELoss(reduction='none')
+
+        # Set device
+        ae_net = ae_net.to(self.device)
+        criterion = criterion.to(self.device)
+
+        # Set optimizer (Adam optimizer for now)
+        optimizer = optim.Adam(ae_net.parameters(), lr=self.lr, weight_decay=self.weight_decay)
+
+        # Set learning rate scheduler
+        scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=self.lr_milestones, gamma=0.1)
+
+        # Training
+        logger.info('Starting pretraining...')
+        start_time = time.time()
+        ae_net.train()
+        for epoch in range(self.n_epochs):
+
+            scheduler.step()
+            if epoch in self.lr_milestones:
+                logger.info('  LR scheduler: new learning rate is %g' % float(scheduler.get_lr()[0]))
+
+            epoch_loss = 0.0
+            n_batches = 0
+            epoch_start_time = time.time()
+            for data in train_loader:
+                inputs, _, _, _ = data
+                inputs = inputs.to(self.device)
+
+                # Zero the network parameter gradients
+                optimizer.zero_grad()
+
+                # Update network parameters via backpropagation: forward + backward + optimize
+                rec = ae_net(inputs)
+                rec_loss = criterion(rec, inputs)
+                loss = torch.mean(rec_loss)
+                loss.backward()
+                optimizer.step()
+
+                epoch_loss += loss.item()
+                n_batches += 1
+
+            # log epoch statistics
+            epoch_train_time = time.time() - epoch_start_time
+            logger.info(f'| Epoch: {epoch + 1:03}/{self.n_epochs:03} | Train Time: {epoch_train_time:.3f}s '
+                        f'| Train Loss: {epoch_loss / n_batches:.6f} |')
+
+        self.train_time = time.time() - start_time
+        logger.info('Pretraining Time: {:.3f}s'.format(self.train_time))
+        logger.info('Finished pretraining.')
+
+        return ae_net
+
+    def test(self, dataset: BaseADDataset, ae_net: BaseNet):
+        logger = logging.getLogger()
+
+        # Get test data loader
+        _, test_loader = dataset.loaders(batch_size=self.batch_size, num_workers=self.n_jobs_dataloader)
+
+        # Set loss
+        criterion = nn.MSELoss(reduction='none')
+
+        # Set device for network
+        ae_net = ae_net.to(self.device)
+        criterion = criterion.to(self.device)
+
+        # Testing
+        logger.info('Testing autoencoder...')
+        epoch_loss = 0.0
+        n_batches = 0
+        start_time = time.time()
+        idx_label_score = []
+        ae_net.eval()
+        with torch.no_grad():
+            for data in test_loader:
+                inputs, labels, _, idx = data
+                inputs, labels, idx = inputs.to(self.device), labels.to(self.device), idx.to(self.device)
+
+                rec = ae_net(inputs)
+                rec_loss = criterion(rec, inputs)
+                scores = torch.mean(rec_loss, dim=tuple(range(1, rec.dim())))
+
+                # Save triple of (idx, label, score) in a list
+                idx_label_score += list(zip(idx.cpu().data.numpy().tolist(),
+                                            labels.cpu().data.numpy().tolist(),
+                                            scores.cpu().data.numpy().tolist()))
+
+                loss = torch.mean(rec_loss)
+                epoch_loss += loss.item()
+                n_batches += 1
+
+        self.test_time = time.time() - start_time
+
+        # Compute AUC
+        _, labels, scores = zip(*idx_label_score)
+        labels = np.array(labels)
+        scores = np.array(scores)
+        self.test_auc = roc_auc_score(labels, scores)
+
+        # Log results
+        logger.info('Test Loss: {:.6f}'.format(epoch_loss / n_batches))
+        logger.info('Test AUC: {:.2f}%'.format(100. * self.test_auc))
+        logger.info('Test Time: {:.3f}s'.format(self.test_time))
+        logger.info('Finished testing autoencoder.')

Deep-SAD-PyTorch/src/optim/vae_trainer.py

@@ -0,0 +1,139 @@
+from base.base_trainer import BaseTrainer
+from base.base_dataset import BaseADDataset
+from base.base_net import BaseNet
+from utils.misc import binary_cross_entropy
+from sklearn.metrics import roc_auc_score
+
+import logging
+import time
+import torch
+import torch.optim as optim
+import numpy as np
+
+
+class VAETrainer(BaseTrainer):
+
+    def __init__(self, optimizer_name: str = 'adam', lr: float = 0.001, n_epochs: int = 150, lr_milestones: tuple = (),
+                 batch_size: int = 128, weight_decay: float = 1e-6, device: str = 'cuda', n_jobs_dataloader: int = 0):
+        super().__init__(optimizer_name, lr, n_epochs, lr_milestones, batch_size, weight_decay, device,
+                         n_jobs_dataloader)
+
+        # Results
+        self.train_time = None
+        self.test_auc = None
+        self.test_time = None
+
+    def train(self, dataset: BaseADDataset, vae: BaseNet):
+        logger = logging.getLogger()
+
+        # Get train data loader
+        train_loader, _ = dataset.loaders(batch_size=self.batch_size, num_workers=self.n_jobs_dataloader)
+
+        # Set device
+        vae = vae.to(self.device)
+
+        # Set optimizer (Adam optimizer for now)
+        optimizer = optim.Adam(vae.parameters(), lr=self.lr, weight_decay=self.weight_decay)
+
+        # Set learning rate scheduler
+        scheduler = optim.lr_scheduler.MultiStepLR(optimizer, milestones=self.lr_milestones, gamma=0.1)
+
+        # Training
+        logger.info('Starting pretraining...')
+        start_time = time.time()
+        vae.train()
+        for epoch in range(self.n_epochs):
+
+            scheduler.step()
+            if epoch in self.lr_milestones:
+                logger.info('  LR scheduler: new learning rate is %g' % float(scheduler.get_lr()[0]))
+
+            epoch_loss = 0.0
+            n_batches = 0
+            epoch_start_time = time.time()
+            for data in train_loader:
+                inputs, _, _, _ = data
+                inputs = inputs.to(self.device)
+                inputs = inputs.view(inputs.size(0), -1)
+
+                # Zero the network parameter gradients
+                optimizer.zero_grad()
+
+                # Update network parameters via backpropagation: forward + backward + optimize
+                rec = vae(inputs)
+
+                likelihood = -binary_cross_entropy(rec, inputs)
+                elbo = likelihood - vae.kl_divergence
+
+                # Overall loss
+                loss = -torch.mean(elbo)
+
+                loss.backward()
+                optimizer.step()
+
+                epoch_loss += loss.item()
+                n_batches += 1
+
+            # log epoch statistics
+            epoch_train_time = time.time() - epoch_start_time
+            logger.info(f'| Epoch: {epoch + 1:03}/{self.n_epochs:03} | Train Time: {epoch_train_time:.3f}s '
+                        f'| Train Loss: {epoch_loss / n_batches:.6f} |')
+
+        self.train_time = time.time() - start_time
+        logger.info('Pretraining Time: {:.3f}s'.format(self.train_time))
+        logger.info('Finished pretraining.')
+
+        return vae
+
+    def test(self, dataset: BaseADDataset, vae: BaseNet):
+        logger = logging.getLogger()
+
+        # Get test data loader
+        _, test_loader = dataset.loaders(batch_size=self.batch_size, num_workers=self.n_jobs_dataloader)
+
+        # Set device
+        vae = vae.to(self.device)
+
+        # Testing
+        logger.info('Starting testing...')
+        epoch_loss = 0.0
+        n_batches = 0
+        start_time = time.time()
+        idx_label_score = []
+        vae.eval()
+        with torch.no_grad():
+            for data in test_loader:
+                inputs, labels, _, idx = data
+                inputs, labels, idx = inputs.to(self.device), labels.to(self.device), idx.to(self.device)
+
+                inputs = inputs.view(inputs.size(0), -1)
+
+                rec = vae(inputs)
+                likelihood = -binary_cross_entropy(rec, inputs)
+                scores = -likelihood  # negative likelihood as anomaly score
+
+                # Save triple of (idx, label, score) in a list
+                idx_label_score += list(zip(idx.cpu().data.numpy().tolist(),
+                                            labels.cpu().data.numpy().tolist(),
+                                            scores.cpu().data.numpy().tolist()))
+
+                # Overall loss
+                elbo = likelihood - vae.kl_divergence
+                loss = -torch.mean(elbo)
+
+                epoch_loss += loss.item()
+                n_batches += 1
+
+        self.test_time = time.time() - start_time
+
+        # Compute AUC
+        _, labels, scores = zip(*idx_label_score)
+        labels = np.array(labels)
+        scores = np.array(scores)
+        self.test_auc = roc_auc_score(labels, scores)
+
+        # Log results
+        logger.info('Test Loss: {:.6f}'.format(epoch_loss / n_batches))
+        logger.info('Test AUC: {:.2f}%'.format(100. * self.test_auc))
+        logger.info('Test Time: {:.3f}s'.format(self.test_time))
+        logger.info('Finished testing variational autoencoder.')

Deep-SAD-PyTorch/src/optim/variational.py

@@ -0,0 +1,93 @@
+import torch
+import torch.nn.functional as F
+
+from torch import nn
+from itertools import repeat
+from utils import enumerate_discrete, log_sum_exp
+from networks import log_standard_categorical
+
+
+# Acknowledgements: https://github.com/wohlert/semi-supervised-pytorch
+class ImportanceWeightedSampler(object):
+    """
+    Importance weighted sampler (Burda et al., 2015) to be used together with SVI.
+
+    :param mc: number of Monte Carlo samples
+    :param iw: number of Importance Weighted samples
+    """
+
+    def __init__(self, mc=1, iw=1):
+        self.mc = mc
+        self.iw = iw
+
+    def resample(self, x):
+        return x.repeat(self.mc * self.iw, 1)
+
+    def __call__(self, elbo):
+        elbo = elbo.view(self.mc, self.iw, -1)
+        elbo = torch.mean(log_sum_exp(elbo, dim=1, sum_op=torch.mean), dim=0)
+        return elbo.view(-1)
+
+
+class SVI(nn.Module):
+    """
+    Stochastic variational inference (SVI) optimizer for semi-supervised learning.
+
+    :param model: semi-supervised model to evaluate
+    :param likelihood: p(x|y,z) for example BCE or MSE
+    :param beta: warm-up/scaling of KL-term
+    :param sampler: sampler for x and y, e.g. for Monte Carlo
+    """
+
+    base_sampler = ImportanceWeightedSampler(mc=1, iw=1)
+
+    def __init__(self, model, likelihood=F.binary_cross_entropy, beta=repeat(1), sampler=base_sampler):
+        super(SVI, self).__init__()
+        self.model = model
+        self.likelihood = likelihood
+        self.sampler = sampler
+        self.beta = beta
+
+    def forward(self, x, y=None):
+        is_labeled = False if y is None else True
+
+        # Prepare for sampling
+        xs, ys = (x, y)
+
+        # Enumerate choices of label
+        if not is_labeled:
+            ys = enumerate_discrete(xs, self.model.y_dim)
+            xs = xs.repeat(self.model.y_dim, 1)
+
+        # Increase sampling dimension
+        xs = self.sampler.resample(xs)
+        ys = self.sampler.resample(ys)
+
+        reconstruction = self.model(xs, ys)
+
+        # p(x|y,z)
+        likelihood = -self.likelihood(reconstruction, xs)
+
+        # p(y)
+        prior = -log_standard_categorical(ys)
+
+        # Equivalent to -L(x, y)
+        elbo = likelihood + prior - next(self.beta) * self.model.kl_divergence
+        L = self.sampler(elbo)
+
+        if is_labeled:
+            return torch.mean(L)
+
+        logits = self.model.classify(x)
+
+        L = L.view_as(logits.t()).t()
+
+        # Calculate entropy H(q(y|x)) and sum over all labels
+        eps = 1e-8
+        H = -torch.sum(torch.mul(logits, torch.log(logits + eps)), dim=-1)
+        L = torch.sum(torch.mul(logits, L), dim=-1)
+
+        # Equivalent to -U(x)
+        U = L + H
+
+        return torch.mean(U)

Deep-SAD-PyTorch/src/utils/__init__.py

@@ -0,0 +1,3 @@
+from .config import Config
+from .visualization.plot_images_grid import plot_images_grid
+from .misc import enumerate_discrete, log_sum_exp, binary_cross_entropy

Deep-SAD-PyTorch/src/utils/config.py

@@ -0,0 +1,23 @@
+import json
+
+
+class Config(object):
+    """Base class for experimental setting/configuration."""
+
+    def __init__(self, settings):
+        self.settings = settings
+
+    def load_config(self, import_json):
+        """Load settings dict from import_json (path/filename.json) JSON-file."""
+
+        with open(import_json, 'r') as fp:
+            settings = json.load(fp)
+
+        for key, value in settings.items():
+            self.settings[key] = value
+
+    def save_config(self, export_json):
+        """Save settings dict to export_json (path/filename.json) JSON-file."""
+
+        with open(export_json, 'w') as fp:
+            json.dump(self.settings, fp)

Deep-SAD-PyTorch/src/utils/misc.py

@@ -0,0 +1,46 @@
+import torch
+
+from torch.autograd import Variable
+
+
+# Acknowledgements: https://github.com/wohlert/semi-supervised-pytorch
+def enumerate_discrete(x, y_dim):
+    """
+    Generates a 'torch.Tensor' of size batch_size x n_labels of the given label.
+
+    :param x: tensor with batch size to mimic
+    :param y_dim: number of total labels
+    :return variable
+    """
+
+    def batch(batch_size, label):
+        labels = (torch.ones(batch_size, 1) * label).type(torch.LongTensor)
+        y = torch.zeros((batch_size, y_dim))
+        y.scatter_(1, labels, 1)
+        return y.type(torch.LongTensor)
+
+    batch_size = x.size(0)
+    generated = torch.cat([batch(batch_size, i) for i in range(y_dim)])
+
+    if x.is_cuda:
+        generated = generated.to(x.device)
+
+    return Variable(generated.float())
+
+
+def log_sum_exp(tensor, dim=-1, sum_op=torch.sum):
+    """
+    Uses the LogSumExp (LSE) as an approximation for the sum in a log-domain.
+
+    :param tensor: Tensor to compute LSE over
+    :param dim: dimension to perform operation over
+    :param sum_op: reductive operation to be applied, e.g. torch.sum or torch.mean
+    :return: LSE
+    """
+    max, _ = torch.max(tensor, dim=dim, keepdim=True)
+    return torch.log(sum_op(torch.exp(tensor - max), dim=dim, keepdim=True) + 1e-8) + max
+
+
+def binary_cross_entropy(x, y):
+    eps = 1e-8
+    return -torch.sum(y * torch.log(x + eps) + (1 - y) * torch.log(1 - x + eps), dim=-1)

Deep-SAD-PyTorch/src/utils/visualization/plot_images_grid.py

@@ -0,0 +1,26 @@
+import torch
+import matplotlib
+matplotlib.use('Agg')  # or 'PS', 'PDF', 'SVG'
+
+import matplotlib.pyplot as plt
+import numpy as np
+from torchvision.utils import make_grid
+
+
+def plot_images_grid(x: torch.tensor, export_img, title: str = '', nrow=8, padding=2, normalize=False, pad_value=0):
+    """Plot 4D Tensor of images of shape (B x C x H x W) as a grid."""
+
+    grid = make_grid(x, nrow=nrow, padding=padding, normalize=normalize, pad_value=pad_value)
+    npgrid = grid.cpu().numpy()
+
+    plt.imshow(np.transpose(npgrid, (1, 2, 0)), interpolation='nearest')
+
+    ax = plt.gca()
+    ax.xaxis.set_visible(False)
+    ax.yaxis.set_visible(False)
+
+    if not (title == ''):
+        plt.title(title)
+
+    plt.savefig(export_img, bbox_inches='tight', pad_inches=0.1)
+    plt.clf()