# Notebook from adriHei/deepflash2
Path: nbs/00_learner.ipynb

#default_exp learner
from nbdev.showdoc import show_doc_____no_output_____

# Ensemble Learner

> Implements functions necessary to build an  `EnsembleLearner` suitable for bioimgage segmentation_____no_output_____

#hide
import imageio
from scipy import ndimage
from fastcore.test import *_____no_output_____#export
import shutil, gc, joblib, json, zarr, numpy as np, pandas as pd
import torch, torch.nn as nn, torch.nn.functional as F
from torch.utils.data import DataLoader
from dataclasses import dataclass, field, asdict
from pathlib import Path

from sklearn import svm
from sklearn.model_selection import KFold
from sklearn.pipeline import Pipeline
from sklearn.preprocessing import StandardScaler
from scipy.ndimage.filters import gaussian_filter
import matplotlib.pyplot as plt

from fastprogress import progress_bar
from fastcore.basics import patch, GetAttr
from fastcore.foundation import add_docs, L
from fastai import optimizer
from fastai.torch_core import TensorImage
from fastai.learner import Learner
from fastai.callback.tracker import SaveModelCallback
from fastai.data.core import DataLoaders
from fastai.data.transforms import get_image_files, get_files
from fastai.vision.augment import Brightness, Contrast, Saturation
from fastai.losses import CrossEntropyLossFlat

from deepflash2.metrics import Dice, Iou
from deepflash2.losses import get_loss
from deepflash2.models import create_smp_model, save_smp_model, load_smp_model
from deepflash2.data import TileDataset, RandomTileDataset, _read_img, _read_msk
from deepflash2.utils import iou, plot_results, get_label_fn, calc_iterations, save_mask, save_unc, export_roi_set
from deepflash2.utils import compose_albumentations as _compose_albumentations
import deepflash2.tta as tta/media/data/anaconda3/envs/fastai2/lib/python3.8/site-packages/torch/cuda/__init__.py:52: UserWarning: CUDA initialization: Unexpected error from cudaGetDeviceCount(). Did you run some cuda functions before calling NumCudaDevices() that might have already set an error? Error 804: forward compatibility was attempted on non supported HW (Triggered internally at  /opt/conda/conda-bld/pytorch_1623448278899/work/c10/cuda/CUDAFunctions.cpp:115.)
  return torch._C._cuda_getDeviceCount() > 0

## Config_____no_output_____

#export
@dataclass
class Config:
    "Config class for settings."

    # Project
    proj_dir:str = 'deepflash2'

    # GT Estimation Settings
    staple_thres:float = 0.5
    staple_fval:int= 1
    mv_undec:int = 0

    # Train General Settings
    n:int = 5
    max_splits:int=5
    random_state:int = 42

    # Pytorch Segmentation Model Settings
    arch:str = 'Unet'
    encoder_name:str = 'resnet34'
    encoder_weights:str = 'imagenet'

    # Train Data Settings
    c:int = 2
    tile_shape:int = 512
    il:bool = False

    # Train Settings
    base_lr:float = 0.001
    bs:int = 4
    wd:float = 0.001
    mpt:bool = False
    optim:str = 'Adam'
    loss:str = 'CrossEntropyDiceLoss'
    n_iter:int = 2000
    sample_mult:int = 0

    # Validation and Prediction Settings
    tta:bool = True
    border_padding_factor:float = 0.25
    shift:float = 0.5

    # Train Data Augmentation
    gamma_limit_lower:int = 80
    gamma_limit_upper:int = 120
    CLAHE_clip_limit:float = 0.0
    brightness_limit:float = 0.0
    contrast_limit:float = 0.0
    flip:bool = True
    rot:int = 360
    distort_limit:float = 0

    # Loss Settings
    mode:str = 'multiclass' #currently only tested for multiclass
    loss_alpha:float = 0.5 # Twerksky/Focal loss
    loss_beta:float = 0.5 # Twerksy Loss
    loss_gamma:float = 2.0 # Focal loss
    loss_smooth_factor:float = 0. #SoftCrossEntropyLoss

    # Pred Settings
    pred_tta:bool = True
    min_pixel_export:int = 0

    # Folder Structure
    gt_dir:str = 'GT_Estimation'
    train_dir:str = 'Training'
    pred_dir:str = 'Prediction'
    ens_dir:str = 'ensemble'
    val_dir:str = 'valid'

    @property
    def albumentation_kwargs(self):
        kwargs = ['gamma_limit_lower', 'gamma_limit_upper', 'CLAHE_clip_limit',
                  'brightness_limit', 'contrast_limit', 'distort_limit']
        return dict(filter(lambda x: x[0] in kwargs, self.__dict__.items()))

    @property
    def svm_kwargs(self):
        svm_vars = ['kernel', 'nu', 'gamma']
        return dict(filter(lambda x: x[0] in svm_vars, self.__dict__.items()))

    def save(self, path):
        'Save configuration to path'
        path = Path(path)
        with open(path.with_suffix('.json'), 'w') as config_file:
            json.dump(asdict(self), config_file)
        print(f'Saved current configuration to {path}.json')

    def load(self, path):
        'Load configuration from path'
        path = Path(path)
        try:
            with open(path) as config_file: c = json.load(config_file)
            if not Path(c['proj_dir']).is_dir(): c['proj_dir']='deepflash2'
            for k,v in c.items(): setattr(self, k, v)
            print(f'Successsfully loaded configuration from {path}')
        except:
            print('Error! Select valid config file (.json)')_____no_output_____t1 = Config(n=3)
t1.save('test_config')
t2 = Config()
t2.load('test_config.json')
test_eq(t1, t2)Saved current configuration to test_config.json
Successsfully loaded configuration from test_config.json
#export
_optim_dict = {
    'ranger' : optimizer.ranger,
    'Adam' : optimizer.Adam,
    'RAdam' : optimizer.RAdam,
    'QHAdam' :optimizer.QHAdam,
    'Larc' : optimizer.Larc,
    'Lamb' : optimizer.Lamb,
    'SGD' : optimizer.SGD,
    'RMSProp' : optimizer.RMSProp,
}_____no_output_____

## Ensemble Prediction Class_____no_output_____

#export
# from https://github.com/MIC-DKFZ/nnUNet/blob/2fade8f32607220f8598544f0d5b5e5fa73768e5/nnunet/network_architecture/neural_network.py#L250
def _get_gaussian(patch_size, sigma_scale=1. / 8) -> np.ndarray:
    tmp = np.zeros(patch_size)
    center_coords = [i // 2 for i in patch_size]
    sigmas = [i * sigma_scale for i in patch_size]
    tmp[tuple(center_coords)] = 1
    gaussian_importance_map = gaussian_filter(tmp, sigmas, 0, mode='constant', cval=0)
    gaussian_importance_map = gaussian_importance_map / np.max(gaussian_importance_map) * 1
    gaussian_importance_map = gaussian_importance_map.astype(np.float32)

    # gaussian_importance_map cannot be 0, otherwise we may end up with nans!
    gaussian_importance_map[gaussian_importance_map == 0] = np.min(
        gaussian_importance_map[gaussian_importance_map != 0])

    return gaussian_importance_map_____no_output_____plt.imshow(_get_gaussian((512,512)));_____no_output_____#export
def energy_score(x, T=1, dim=1):
    'Return the energy score as proposed by  Liu, Weitang, et al. (2020).'
    return -(T*torch.logsumexp(x/T, dim=dim))_____no_output_____#export
class EnsemblePredict():
    'Class for prediction with multiple models'
    def __init__(self, models_paths, zarr_store=None):
        self.models_paths = models_paths
        self.device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')
        self.init_models()

        # Init zarr storage
        self.store = str(zarr_store) if zarr_store else zarr.storage.TempStore()
        self.root = zarr.group(store=self.store)
        self.g_smx = self.root.require_group('smx')
        self.g_std = self.root.require_group('std')
        self.g_eng = self.root.require_group('energy')

    def init_models(self):
        self.models = []
        self.stats = None
        for p in self.models_paths:
            model, stats = load_smp_model(p)
            if not self.stats: self.stats = stats
            assert np.array_equal(stats, self.stats), 'Only models trained on the same stats are allowed.'
            model.float()
            model.eval()
            model.to(self.device)
            self.models.append(model)

    def predict(self,
                ds,
                use_tta=True,
                bs=4,
                use_gaussian=True,
                sigma_scale=1./8,
                uncertainty_estimates=True,
                energy_T = 1.,
                verbose=0):

        if verbose>0: print('Ensemble prediction with models:', self.models_paths)

        tfms = [tta.HorizontalFlip(),tta.VerticalFlip()] if use_tta else []
        if verbose>0: print('Using Test-Time Augmentation with:', tfms)

        dl = DataLoader(ds, bs, num_workers=0, shuffle=False, pin_memory=True)

        # Create zero arrays
        data_shape = ds.image_shapes[0]
        softmax = np.zeros((*data_shape, ds.c), dtype='float32')
        merge_map = np.zeros(data_shape, dtype='float32')
        stdeviation = np.zeros(data_shape, dtype='float32') if uncertainty_estimates else None
        energy = np.zeros(data_shape, dtype='float32') if uncertainty_estimates else None

        # Define merge weights
        if use_gaussian:
            mw_numpy = _get_gaussian(ds.output_shape, sigma_scale)
        else:
            mw_numpy = np.ones(dl.output_shape)
        mw = torch.from_numpy(mw_numpy).to(self.device)

        # Loop over tiles (indices required!)
        for tiles, idxs in iter(dl):
            tiles = tiles.to(self.device)
            smx_merger = tta.Merger()
            if uncertainty_estimates:
                energy_merger = tta.Merger()

            # Loop over tt-augmentations
            for t in tta.Compose(tfms):
                aug_tiles = t.augment_image(tiles)
                model_merger = tta.Merger()
                if uncertainty_estimates: engergy_list = []

                # Loop over models
                for model in self.models:
                    with torch.no_grad():
                        logits = model(aug_tiles)
                    logits = t.deaugment_mask(logits)
                    smx_merger.append(F.softmax(logits, dim=1))
                    if uncertainty_estimates:
                        energy_merger.append(-energy_score(logits, energy_T)) #negative energy score

            out_list = []
            # Apply gaussian weigthing
            batch_smx = smx_merger.result()*mw.view(1,1,*mw.shape)
            # Reshape and append to list
            out_list.append([x for x in batch_smx.permute(0,2,3,1).cpu().numpy()])

            if uncertainty_estimates:
                batch_std = torch.mean(smx_merger.result('std'), dim=1)*mw.view(1,*mw.shape)
                out_list.append([x for x in batch_std.cpu().numpy()])

                batch_energy =  energy_merger.result()*mw.view(1,*mw.shape)
                out_list.append([x for x in batch_energy.cpu().numpy()])

            # Compose predictions
            for preds in zip(*out_list, idxs):
                if uncertainty_estimates: smx,std,eng,idx = preds
                else: smx, idx = preds
                out_slice = ds.out_slices[idx]
                in_slice = ds.in_slices[idx]
                softmax[out_slice] += smx[in_slice]
                merge_map[out_slice] += mw_numpy[in_slice]

                if uncertainty_estimates:
                    stdeviation[out_slice] += std[in_slice]
                    energy[out_slice] += eng[in_slice]

        # Normalize weighting
        softmax /= merge_map[..., np.newaxis]
        if uncertainty_estimates:
            energy /= merge_map
            stdeviation /= merge_map

        return softmax, stdeviation, energy

    def predict_images(self, image_list, ds_kwargs={}, verbose=1, **kwargs):
        "Predict images in 'image_list' with kwargs and save to zarr"
        for f in progress_bar(image_list, leave=False):
            if verbose>0: print(f'Predicting {f.name}')
            ds = TileDataset([f], stats=self.stats, return_index=True, **ds_kwargs)
            softmax, stdeviation, energy = self.predict(ds, **kwargs)

            # Save to zarr
            self.g_smx[f.name] = softmax
            if stdeviation is not None: self.g_std[f.name] = stdeviation
            if energy is not None: self.g_eng[f.name] = energy

        return self.g_smx, self.g_std, self.g_eng       _____no_output_____

## Ensemble Learner_____no_output_____

#export
class EnsembleLearner(GetAttr):
    _default = 'config'
    def __init__(self, image_dir='images', mask_dir=None, config=None, path=None, ensemble_dir=None, item_tfms=None,
                 label_fn=None, metrics=None, cbs=None, ds_kwargs={}, dl_kwargs={}, model_kwargs={}, stats=None, files=None):

        self.config = config or Config()
        self.stats = stats
        self.dl_kwargs = dl_kwargs
        self.model_kwargs = model_kwargs
        self.add_ds_kwargs = ds_kwargs
        self.item_tfms = item_tfms
        self.path = Path(path) if path is not None else Path('.')
        self.metrics = metrics or [Iou(), Dice()]
        self.loss_fn = self.get_loss()
        self.cbs = cbs or [SaveModelCallback(monitor='iou')] #ShowGraphCallback
        self.ensemble_dir = ensemble_dir or self.path/'ensemble'

        self.files = L(files) or get_image_files(self.path/image_dir, recurse=False)
        assert len(self.files)>0, f'Found {len(self.files)} images in "{image_dir}". Please check your images and image folder'
        if any([mask_dir, label_fn]):
            if label_fn: self.label_fn = label_fn
            else: self.label_fn = get_label_fn(self.files[0], self.path/mask_dir)
            #Check if corresponding masks exist
            mask_check = [self.label_fn(x).exists() for x in self.files]
            chk_str = f'Found {len(self.files)} images in "{image_dir}" and {sum(mask_check)} masks in "{mask_dir}".'
            assert len(self.files)==sum(mask_check) and len(self.files)>0, f'Please check your images and masks (and folders). {chk_str}'
            print(chk_str)

        else:
            self.label_fn = label_fn
        self.n_splits=min(len(self.files), self.max_splits)

        self.models = {}
        self.recorder = {}
        self._set_splits()
        self.ds = RandomTileDataset(self.files, label_fn=self.label_fn, stats=self.stats, verbose=0)
        self.stats = stats or self.ds.stats
        self.in_channels = self.ds.get_data(max_n=1)[0].shape[-1]
        self.df_val, self.df_ens, self.df_model, self.ood = None,None,None,None

    def _set_splits(self):
        if self.n_splits>1:
            kf = KFold(self.n_splits, shuffle=True, random_state=self.random_state)
            self.splits = {key:(self.files[idx[0]], self.files[idx[1]]) for key, idx in zip(range(1,self.n_splits+1), kf.split(self.files))}
        else:
            self.splits = {1: (self.files[0], self.files[0])}

    def _compose_albumentations(self, **kwargs):
        return _compose_albumentations(**kwargs)

    @property
    def pred_ds_kwargs(self):
        # Setting default shapes and padding
        ds_kwargs = self.add_ds_kwargs.copy()
        ds_kwargs['tile_shape']= (self.tile_shape,)*2
        ds_kwargs['n_classes']= self.c
        ds_kwargs['shift']= self.shift
        ds_kwargs['border_padding_factor']= self.border_padding_factor
        return ds_kwargs

    @property
    def train_ds_kwargs(self):
        # Setting default shapes and padding
        ds_kwargs = self.add_ds_kwargs.copy()
        # Settings from config
        ds_kwargs['stats']= self.stats
        ds_kwargs['tile_shape']= (self.tile_shape,)*2
        ds_kwargs['n_classes']= self.c
        ds_kwargs['shift']= 1.
        ds_kwargs['border_padding_factor']= 0.
        ds_kwargs['flip'] = self.flip
        ds_kwargs['albumentations_tfms'] = self._compose_albumentations(**self.albumentation_kwargs)
        ds_kwargs['sample_mult'] = self.sample_mult if self.sample_mult>0 else None
        return ds_kwargs

    @property
    def model_name(self):
        return f'{self.arch}_{self.encoder_name}_{self.c}classes'

    def get_loss(self):
        kwargs = {'mode':self.mode,
                  'classes':[x for x in range(1, self.c)],
                  'smooth_factor': self.loss_smooth_factor,
                  'alpha':self.loss_alpha,
                  'beta':self.loss_beta,
                  'gamma':self.loss_gamma}
        return get_loss(self.loss, **kwargs)

    def _get_dls(self, files, files_val=None):
        ds = []
        ds.append(RandomTileDataset(files, label_fn=self.label_fn, **self.train_ds_kwargs))
        if files_val:
            ds.append(TileDataset(files_val, label_fn=self.label_fn, **self.train_ds_kwargs))
        else:
            ds.append(ds[0])
        dls = DataLoaders.from_dsets(*ds, bs=self.bs, pin_memory=True, **self.dl_kwargs)
        if torch.cuda.is_available(): dls.cuda()
        return dls

    def _create_model(self):
        model = create_smp_model(arch=self.arch,
                                 encoder_name=self.encoder_name,
                                 encoder_weights=self.encoder_weights,
                                 in_channels=self.in_channels,
                                 classes=self.c,
                                 **self.model_kwargs)
        if torch.cuda.is_available(): model.cuda()
        return model

    def fit(self, i, n_iter=None, base_lr=None, **kwargs):
        n_iter = n_iter or self.n_iter
        base_lr = base_lr or self.base_lr
        name = self.ensemble_dir/f'{self.model_name}-fold{i}.pth'
        model = self._create_model()
        files_train, files_val = self.splits[i]
        dls = self._get_dls(files_train, files_val)
        self.learn = Learner(dls, model, metrics=self.metrics, wd=self.wd, loss_func=self.loss_fn, opt_func=_optim_dict[self.optim], cbs=self.cbs)
        self.learn.model_dir = self.ensemble_dir.parent/'.tmp'
        if self.mpt: self.learn.to_fp16()
        print(f'Starting training for {name.name}')
        epochs = calc_iterations(n_iter=n_iter,ds_length=len(dls.train_ds), bs=self.bs)
        #self.learn.fit_one_cycle(epochs, lr_max)
        self.learn.fine_tune(epochs, base_lr=base_lr)

        print(f'Saving model at {name}')
        name.parent.mkdir(exist_ok=True, parents=True)
        save_smp_model(self.learn.model, self.arch, name, stats=self.stats)
        self.models[i]=name
        self.recorder[i]=self.learn.recorder

    def fit_ensemble(self, n_iter, skip=False, **kwargs):
        for i in range(1, self.n+1):
            if skip and (i in self.models): continue
            self.fit(i, n_iter,  **kwargs)

    def set_n(self, n):
        for i in range(n, len(self.models)):
            self.models.pop(i+1, None)
        self.n = n

    def get_valid_results(self, model_no=None, export_dir=None, filetype='.png', **kwargs):
        res_list = []
        model_list = self.models if not model_no else [model_no]
        if export_dir:
            export_dir = Path(export_dir)
            pred_path = export_dir/'masks'
            pred_path.mkdir(parents=True, exist_ok=True)
            unc_path = export_dir/'uncertainties'
            unc_path.mkdir(parents=True, exist_ok=True)

        for i, model_path in model_list.items():
            ep = EnsemblePredict(models_paths=[model_path])
            _, files_val = self.splits[i]
            g_smx, g_std, g_eng = ep.predict_images(files_val, bs=self.bs, ds_kwargs=self.pred_ds_kwargs, **kwargs)

            chunk_store = g_smx.chunk_store.path
            for j, f in enumerate(files_val):
                msk = self.ds.get_data(f, mask=True)[0]
                pred = np.argmax(g_smx[f.name][:], axis=-1).astype('uint8')
                m_iou = iou(msk, pred)
                m_path = self.models[i].name
                df_tmp = pd.Series({'file' : f.name,
                        'model' :  m_path,
                        'model_no' : i,
                        'iou': m_iou,
                        'dice': 2*m_iou/(m_iou+1),
                        'mean_energy': np.mean(g_eng[f.name][:][pred>0]),
                        'mean_uncertainty': np.mean(g_std[f.name][:][pred>0]),
                        'image_path': f,
                        'mask_path': self.label_fn(f),
                        'softmax_path': f'{chunk_store}/{g_smx.path}/{f.name}',
                        'engergy_path': f'{chunk_store}/{g_eng.path}/{f.name}' if g_eng is not None else None,
                        'uncertainty_path': f'{chunk_store}/{g_std.path}/{f.name}'} if g_std is not None else None)
                res_list.append(df_tmp)
                if export_dir:
                    save_mask(pred, pred_path/f'{df_tmp.file}_{df_tmp.model}_mask', filetype)
                    if g_std is not None:
                        save_unc(g_std[f.name][:], unc_path/f'{df_tmp.file}_{df_tmp.model}_uncertainty', filetype)
                    if g_eng is not None:
                        save_unc(g_eng[f.name][:], unc_path/f'{df_tmp.file}_{df_tmp.model}_energy', filetype)
        self.df_val = pd.DataFrame(res_list)
        if export_dir:
            self.df_val.to_csv(export_dir/f'val_results.csv', index=False)
            self.df_val.to_excel(export_dir/f'val_results.xlsx')
        return self.df_val

    def show_valid_results(self, model_no=None, files=None, **kwargs):
        if self.df_val is None: self.get_valid_results(**kwargs)
        df = self.df_val
        if files is not None: df = df.set_index('file', drop=False).loc[files]
        if model_no is not None: df = df[df.model_no==model_no]
        for _, r in df.iterrows():
            img = self.ds.get_data(r.image_path)[0][:]
            msk = self.ds.get_data(r.image_path, mask=True)[0]
            pred = np.argmax(zarr.load(r.softmax_path), axis=-1).astype('uint8')
            std = zarr.load(r.uncertainty_path)
            _d_model = f'Model {r.model_no}'
            if self.tta: plot_results(img, msk, pred, std, df=r, model=_d_model)
            else: plot_results(img, msk, pred, np.zeros_like(pred), df=r, model=_d_model)

    def load_ensemble(self, path=None):
        path = path or self.ensemble_dir
        models = sorted(get_files(path, extensions='.pth', recurse=False))
        assert len(models)>0, f'No models found in {path}'
        self.models = {}
        for i, m in enumerate(models,1):
            self.models[i] = m
        print(f'Found {len(self.models)} models in folder {path}')
        print(self.models)

    def get_ensemble_results(self, files, zarr_store=None, export_dir=None, filetype='.png', **kwargs):
        ep = EnsemblePredict(models_paths=self.models.values(), zarr_store=zarr_store)
        g_smx, g_std, g_eng = ep.predict_images(files, bs=self.bs, ds_kwargs=self.pred_ds_kwargs, **kwargs)
        chunk_store = g_smx.chunk_store.path

        if export_dir:
            export_dir = Path(export_dir)
            pred_path = export_dir/'masks'
            pred_path.mkdir(parents=True, exist_ok=True)
            unc_path = export_dir/'uncertainties'
            unc_path.mkdir(parents=True, exist_ok=True)

        res_list = []
        for f in files:
            pred = np.argmax(g_smx[f.name][:], axis=-1).astype('uint8')
            df_tmp = pd.Series({'file' : f.name,
                                'ensemble' :  self.model_name,
                                'n_models' : len(self.models),
                                'mean_energy': np.mean(g_eng[f.name][:][pred>0]),
                                'mean_uncertainty': np.mean(g_std[f.name][:][pred>0]),
                                'image_path': f,
                                'softmax_path': f'{chunk_store}/{g_smx.path}/{f.name}',
                                'uncertainty_path': f'{chunk_store}/{g_std.path}/{f.name}' if g_std is not None else None,
                                'energy_path': f'{chunk_store}/{g_eng.path}/{f.name}'} if g_eng is not None else None)
            res_list.append(df_tmp)
            if export_dir:
                save_mask(pred, pred_path/f'{df_tmp.file}_{df_tmp.ensemble}_mask', filetype)
                if g_std is not None:
                    save_unc(g_std[f.name][:], unc_path/f'{df_tmp.file}_{df_tmp.ensemble}_unc', filetype)
                if g_eng is not None:
                        save_unc(g_eng[f.name][:], unc_path/f'{df_tmp.file}_{df_tmp.ensemble}_energy', filetype)

        self.df_ens  = pd.DataFrame(res_list)
        return g_smx, g_std, g_eng

    def score_ensemble_results(self, mask_dir=None, label_fn=None):
        if not label_fn:
            label_fn = get_label_fn(self.df_ens.image_path[0], self.path/mask_dir)
        for idx, r in self.df_ens.iterrows():
            msk_path = self.label_fn(r.image_path)
            msk = _read_msk(msk_path)
            self.df_ens.loc[idx, 'mask_path'] = msk_path
            pred = np.argmax(zarr.load(r.softmax_path), axis=-1).astype('uint8')
            self.df_ens.loc[idx, 'iou'] = iou(msk, pred)
        return self.df_ens

    def show_ensemble_results(self, files=None, model_no=None, unc=True, unc_metric=None):
        assert self.df_ens is not None, "Please run `get_ensemble_results` first."
        if model_no is None: df = self.df_ens
        else: df = self.df_models[df_models.model_no==model_no]
        if files is not None: df = df.set_index('file', drop=False).loc[files]
        for _, r in df.iterrows():
            imgs = []
            imgs.append(_read_img(r.image_path)[:])
            if 'iou' in r.index:
                imgs.append(_read_msk(r.mask_path))
                hastarget=True
            else:
                hastarget=False
            imgs.append(np.argmax(zarr.load(r.softmax_path), axis=-1).astype('uint8'))
            if unc: imgs.append(zarr.load(r.uncertainty_path))
            plot_results(*imgs, df=r, hastarget=hastarget, unc_metric=unc_metric)

    def lr_find(self, files=None, **kwargs):
        files = files or self.files
        dls = self._get_dls(files)
        model = self._create_model()
        learn = Learner(dls, model, metrics=self.metrics, wd=self.wd, loss_func=self.loss_fn, opt_func=_optim_dict[self.optim])
        if self.mpt: learn.to_fp16()
        sug_lrs = learn.lr_find(**kwargs)
        return sug_lrs, learn.recorder

    def export_imagej_rois(self, output_folder='ImageJ_ROIs', **kwargs):
        assert self.df_ens is not None, "Please run prediction first."

        output_folder = Path(output_folder)
        output_folder.mkdir(exist_ok=True, parents=True)
        for idx, r in progress_bar(self.df_ens.iterrows(), total=len(self.df_ens)):
            mask = np.argmax(zarr.load(r.softmax_path), axis=-1).astype('uint8')
            energy = zarr.load(r.energy_path)
            export_roi_set(mask, energy, name=r.file, path=output_folder, **kwargs)

    def clear_tmp(self):
        try:
            shutil.rmtree('/tmp/*', ignore_errors=True)
            shutil.rmtree(self.path/'.tmp')
            print(f'Deleted temporary files from {self.path/".tmp"}')
        except: print(f'No temporary files to delete at {self.path/".tmp"}')_____no_output_____#export
add_docs(EnsembleLearner, "Meta class to train and predict model ensembles with `n` models",
         #save_model= "Save `model` to `file` along with `arch`, `stats`, and `c` classes",
         #load_model="Load `model` from `file` along with `arch`, `stats`, and `c` classes",
         fit="Fit model number `i`",
         fit_ensemble="Fit `i` models and `skip` existing",
         #predict="Predict `files` with model at `model_path`",
         get_valid_results="Validate models on validation data and save results",
         show_valid_results="Plot results of all or `file` validation images",
         #ensemble_results="Merge single model results",
         get_ensemble_results="Get models and ensemble results",
         score_ensemble_results="Compare ensemble results (Intersection over the Union) to given segmentation masks.",
         show_ensemble_results="Show result of ensemble or `model_no`",
         load_ensemble="Get models saved at `path`",
         #compose_albumentations="Helper function to compose albumentations augmentations",
         #get_dls="Create datasets and dataloaders from files",
         #get_model="Get model architecture",
         get_loss="Get loss function from loss name (config)",
         set_n="Change to `n` models per ensemble",
         lr_find="Wrapper for learning rate finder",
         #show_mask_weights='Plot fn for masks and weights',
         #ood_train="Train SVM for OOD Detection",
         #ood_score="Get OOD score",
         #ood_save='Save OOD model to path',
         #ood_load='Load OOD model from path',
         export_imagej_rois='Export ImageJ ROI Sets to `ouput_folder`',
         clear_tmp="Clear directory with temporary files"
)_____no_output_____show_doc(EnsembleLearner)_____no_output_____

## Export -_____no_output_____

#hide
from nbdev.export import *
notebook2script()Converted 00_learner.ipynb.
Converted 01_models.ipynb.
Converted 02_data.ipynb.
Converted 03_metrics.ipynb.
Converted 05_losses.ipynb.
Converted 06_utils.ipynb.
Converted 07_tta.ipynb.
Converted 08_gui.ipynb.
Converted 09_gt.ipynb.
Converted add_information.ipynb.
Converted deepflash2.ipynb.
Converted gt_estimation.ipynb.
Converted index.ipynb.
Converted model_library.ipynb.
Converted predict.ipynb.
Converted train.ipynb.
Converted tutorial.ipynb.


# Notebook from zorrodong/bbknn
Path: examples/benchmark2.ipynb

import numpy as np
import pandas as pd
import scanpy.api as sc
import anndata
import bbknn
import matplotlib.pyplot as plt
import time
import os
import subprocess

sc.settings.verbosity = 3  # verbosity: errors (0), warnings (1), info (2), hints (3)
sc.settings.set_figure_params(dpi=80)  # low dpi (dots per inch) yields small inline figures
sc.logging.print_versions()

np.random.seed(0)/Users/kp9/anaconda3/envs/orig/lib/python3.5/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.
  from ._conv import register_converters as _register_converters

Part two of the benchmarking, BBKNN versus Scanorama only, but more cells. Same setup as before, with the `experiment` and `scanorama` directories and all the Scanorama-specific scripts detailed in part one. Moving back to the 4-core, 16 GB RAM MacBook Pro as we're just testing the two quickest methods.

Move to a 10,000 gene space, as the remaining methods are fast and 10,000 is representative of what Scanorama would filter down to as its HVGs. Evaluate Scanorama and both BBKNN neighbour detection algorithms (the exact `cKDTree` and the approximate annoy) for datasets ranging from 2^11 (~2000) to 2^18 (~250,000) cells._____no_output_____

celltypeA = np.random.random((1,10000))**10
celltypeB = np.random.random((1,10000))**10

batch0 = np.random.normal(size=(1,10000),loc=1,scale=0.5)
batch1 = np.random.normal(size=(1,10000),loc=1,scale=0.5)
batch2 = np.random.normal(size=(1,10000),loc=1,scale=0.5)
batch3 = np.random.normal(size=(1,10000),loc=1,scale=0.5)_____no_output_____ckd_time = []
annoy_time = []
for N in np.arange(8,16):
    expr_template = np.vstack((np.tile(celltypeA,(2**N,1)),np.tile(celltypeB,(2**N,1))))
    exprs0 = (expr_template + np.random.normal(size=(2**(N+1),10000),scale=0.1)) * batch0
    exprs1 = (expr_template + np.random.normal(size=(2**(N+1),10000),scale=0.1)) * batch1
    exprs2 = (expr_template + np.random.normal(size=(2**(N+1),10000),scale=0.1)) * batch2
    exprs3 = (expr_template + np.random.normal(size=(2**(N+1),10000),scale=0.1)) * batch3
    del expr_template
    adata = anndata.AnnData(X=np.vstack((exprs0,exprs1,exprs2,exprs3)))
    del exprs0
    del exprs1
    del exprs2
    del exprs3
    adata.obs['celltype'] = np.tile(np.concatenate((np.repeat('A',2**N),np.repeat('B',2**N))),4)
    adata.obs['batch'] = [str(i) for i in np.repeat(np.arange(4),2**(N+1))]
    sc.tl.pca(adata)
    t1 = time.time()
    bbknn.bbknn(adata)
    t2 = time.time()
    ckd_time.append(t2-t1)
    t1 = time.time()
    bbknn.bbknn(adata,approx=True)
    t2 = time.time()
    annoy_time.append(t2-t1)computing batch balanced neighbors
    finished (0:00:00.37) --> added to `.uns['neighbors']`
    'distances', weighted adjacency matrix
    'connectivities', weighted adjacency matrix
computing batch balanced neighbors
    finished (0:00:00.37) --> added to `.uns['neighbors']`
    'distances', weighted adjacency matrix
    'connectivities', weighted adjacency matrix
computing batch balanced neighbors
    finished (0:00:00.99) --> added to `.uns['neighbors']`
    'distances', weighted adjacency matrix
    'connectivities', weighted adjacency matrix
computing batch balanced neighbors
    finished (0:00:00.83) --> added to `.uns['neighbors']`
    'distances', weighted adjacency matrix
    'connectivities', weighted adjacency matrix
computing batch balanced neighbors
    finished (0:00:02.40) --> added to `.uns['neighbors']`
    'distances', weighted adjacency matrix
    'connectivities', weighted adjacency matrix
computing batch balanced neighbors
    finished (0:00:01.77) --> added to `.uns['neighbors']`
    'distances', weighted adjacency matrix
    'connectivities', weighted adjacency matrix
computing batch balanced neighbors
    finished (0:00:06.57) --> added to `.uns['neighbors']`
    'distances', weighted adjacency matrix
    'connectivities', weighted adjacency matrix
computing batch balanced neighbors
    finished (0:00:03.46) --> added to `.uns['neighbors']`
    'distances', weighted adjacency matrix
    'connectivities', weighted adjacency matrix
computing batch balanced neighbors
    finished (0:00:27.30) --> added to `.uns['neighbors']`
    'distances', weighted adjacency matrix
    'connectivities', weighted adjacency matrix
computing batch balanced neighbors
    finished (0:00:06.77) --> added to `.uns['neighbors']`
    'distances', weighted adjacency matrix
    'connectivities', weighted adjacency matrix
computing batch balanced neighbors
    finished (0:01:43.99) --> added to `.uns['neighbors']`
    'distances', weighted adjacency matrix
    'connectivities', weighted adjacency matrix
computing batch balanced neighbors
    finished (0:00:12.60) --> added to `.uns['neighbors']`
    'distances', weighted adjacency matrix
    'connectivities', weighted adjacency matrix
computing batch balanced neighbors
    finished (0:07:30.93) --> added to `.uns['neighbors']`
    'distances', weighted adjacency matrix
    'connectivities', weighted adjacency matrix
computing batch balanced neighbors
    finished (0:00:25.04) --> added to `.uns['neighbors']`
    'distances', weighted adjacency matrix
    'connectivities', weighted adjacency matrix
computing batch balanced neighbors
    finished (0:33:38.21) --> added to `.uns['neighbors']`
    'distances', weighted adjacency matrix
    'connectivities', weighted adjacency matrix
computing batch balanced neighbors
    finished (0:00:56.82) --> added to `.uns['neighbors']`
    'distances', weighted adjacency matrix
    'connectivities', weighted adjacency matrix

Scanorama is unable to process the 2^17 (~125,000) cell dataset and errors out due to resource constraints. As such, cut it off prematurely._____no_output_____

scanorama_time = []
os.chdir('../scanorama')
#this dies on N=14, so cut it early
for N in np.arange(8,14):
    expr_template = np.vstack((np.tile(celltypeA,(2**N,1)),np.tile(celltypeB,(2**N,1))))
    exprs0 = (expr_template + np.random.normal(size=(2**(N+1),10000),scale=0.1)) * batch0
    exprs1 = (expr_template + np.random.normal(size=(2**(N+1),10000),scale=0.1)) * batch1
    exprs2 = (expr_template + np.random.normal(size=(2**(N+1),10000),scale=0.1)) * batch2
    exprs3 = (expr_template + np.random.normal(size=(2**(N+1),10000),scale=0.1)) * batch3
    adata = anndata.AnnData(X=np.vstack((exprs0,exprs1,exprs2,exprs3)))
    del expr_template
    del exprs0
    del exprs1
    del exprs2
    del exprs3
    adata.obs['celltype'] = np.tile(np.concatenate((np.repeat('A',2**N),np.repeat('B',2**N))),4)
    adata.obs['batch'] = [str(i) for i in np.repeat(np.arange(4),2**(N+1))]
    for batch in [str(i) for i in np.arange(4)]:
        df=pd.DataFrame(data=adata[adata.obs['batch']==batch].X.transpose(),
                        index=adata[adata.obs['batch']==batch].var_names,
                        columns=np.arange(np.sum(adata.obs['batch']==batch)))
        df.to_csv('../scanorama/data/simul/'+batch+'.txt',sep='\t')
    del adata
    del df
    subprocess.run('python bin/process.py conf/simul.txt', shell=True)
    res = subprocess.run('python bin/simul.py', shell=True, stdout=subprocess.PIPE)
    scanorama_time.append(float(res.stdout.decode('utf-8').split('\n')[-2]))
    del res
    print(scanorama_time[-1])
os.chdir('../experiment')6.490329027175903
12.587922811508179
20.409188747406006
41.73166608810425
91.19898700714111
274.81899905204773

As cell count goes up, `cKDTree` tapers off in performance, and would likely meet the near-linearly scaling Scanorama at or near the largest evaluated dataset size, i.e. ~250,000 cells. However, annoy scales linearly as well, and manages to retain an order of magnitude of performance difference between itself and Scanorama. Also, Scanorama runs into the aforementioned resource constraints and aborts at ~125,000 cells, while BBKNN's lightweight nature means it still processes a dataset twice that size without any issue._____no_output_____

fig, ax = plt.subplots()
plt.plot(2**np.arange(11,19),ckd_time)
plt.plot(2**np.arange(11,17),scanorama_time)
plt.plot(2**np.arange(11,19),annoy_time)
ax.set_xscale('log', basex=2)
ax.set_yscale('log', basey=10)
plt.legend(['BBKNN (cKDTree)','Scanorama','BBKNN (annoy)'])
plt.xlabel('Total cells in dataset')
plt.ylabel('Run time (seconds)')

fig.savefig('bbknn_scanorama_10000genes.pdf',bbox_inches='tight')
plt.show()_____no_output_____


# Notebook from adroitashish/Python-for-Data-Science-and-Machine-Learning-Bootcamp
Path: 18.NLP Project.ipynb

# Natural Language Processing Project

We will use the [Yelp Review Data Set from Kaggle](https://www.kaggle.com/c/yelp-recsys-2013).

Each observation in this dataset is a review of a particular business by a particular user.

The "stars" column is the number of stars (1 through 5) assigned by the reviewer to the business. (Higher stars is better.) In other words, it is the rating of the business by the person who wrote the review.

The "cool" column is the number of "cool" votes this review received from other Yelp users.

All reviews start with 0 "cool" votes, and there is no limit to how many "cool" votes a review can receive. In other words, it is a rating of the review itself, not a rating of the business.

The "useful" and "funny" columns are similar to the "cool" column.

Let's get started!_____no_output_____## Imports
 **Import the usual suspects. :) **_____no_output_____

import numpy as np
import pandas as pd_____no_output_____

## The Data

**Read the yelp.csv file and set it as a dataframe called yelp.**_____no_output_____

yelp = pd.read_csv('yelp.csv')_____no_output_____

** Check the head, info , and describe methods on yelp.**_____no_output_____

yelp.head()_____no_output_____yelp.info()<class 'pandas.core.frame.DataFrame'>
RangeIndex: 10000 entries, 0 to 9999
Data columns (total 10 columns):
business_id    10000 non-null object
date           10000 non-null object
review_id      10000 non-null object
stars          10000 non-null int64
text           10000 non-null object
type           10000 non-null object
user_id        10000 non-null object
cool           10000 non-null int64
useful         10000 non-null int64
funny          10000 non-null int64
dtypes: int64(4), object(6)
memory usage: 781.3+ KB
yelp.describe()_____no_output_____

**Create a new column called "text length" which is the number of words in the text column.**_____no_output_____

yelp['text length'] = yelp['text'].apply(len)_____no_output_____

# EDA

Let's explore the data

## Imports

**Import the data visualization libraries if you haven't done so already.**_____no_output_____

import matplotlib.pyplot as plt
import seaborn as sns
sns.set_style('white')
%matplotlib inline_____no_output_____

**Use FacetGrid from the seaborn library to create a grid of 5 histograms of text length based off of the star ratings. Reference the seaborn documentation for hints on this**_____no_output_____

g = sns.FacetGrid(yelp,col='stars')
g.map(plt.hist,'text length')_____no_output_____

**Create a boxplot of text length for each star category.**_____no_output_____

sns.boxplot(x='stars',y='text length',data=yelp,palette='rainbow')_____no_output_____

**Create a countplot of the number of occurrences for each type of star rating.**_____no_output_____

sns.countplot(x='stars',data=yelp,palette='rainbow')_____no_output_____

** Use groupby to get the mean values of the numerical columns, you should be able to create this dataframe with the operation:**_____no_output_____

stars = yelp.groupby('stars').mean()
stars_____no_output_____

**Use the corr() method on that groupby dataframe to produce this dataframe:**_____no_output_____

stars.corr()_____no_output_____

**Then use seaborn to create a heatmap based off that .corr() dataframe:**_____no_output_____

sns.heatmap(stars.corr(),cmap='coolwarm',annot=True)_____no_output_____

## NLP Classification Task

Let's move on to the actual task. To make things a little easier, go ahead and only grab reviews that were either 1 star or 5 stars.

**Create a dataframe called yelp_class that contains the columns of yelp dataframe but for only the 1 or 5 star reviews.**_____no_output_____

yelp_class = yelp[(yelp.stars==1) | (yelp.stars==5)]_____no_output_____

** Create two objects X and y. X will be the 'text' column of yelp_class and y will be the 'stars' column of yelp_class. (Your features and target/labels)**_____no_output_____

X = yelp_class['text']
y = yelp_class['stars']_____no_output_____

**Import CountVectorizer and create a CountVectorizer object.**_____no_output_____

from sklearn.feature_extraction.text import CountVectorizer
cv = CountVectorizer()_____no_output_____

** Use the fit_transform method on the CountVectorizer object and pass in X (the 'text' column). Save this result by overwriting X.**_____no_output_____

X = cv.fit_transform(X)_____no_output_____

## Train Test Split

Let's split our data into training and testing data.

** Use train_test_split to split up the data into X_train, X_test, y_train, y_test. Use test_size=0.3 and random_state=101 **_____no_output_____

from sklearn.model_selection import train_test_split_____no_output_____X_train, X_test, y_train, y_test = train_test_split(X, y,test_size=0.3,random_state=101)_____no_output_____

## Training a Model

Time to train a model!

** Import MultinomialNB and create an instance of the estimator and call is nb **_____no_output_____

from sklearn.naive_bayes import MultinomialNB
nb = MultinomialNB()_____no_output_____

**Now fit nb using the training data.**_____no_output_____

nb.fit(X_train,y_train)_____no_output_____

## Predictions and Evaluations

Time to see how our model did!

**Use the predict method off of nb to predict labels from X_test.**_____no_output_____

predictions = nb.predict(X_test)_____no_output_____

** Create a confusion matrix and classification report using these predictions and y_test **_____no_output_____

from sklearn.metrics import confusion_matrix,classification_report_____no_output_____print(confusion_matrix(y_test,predictions))
print('\n')
print(classification_report(y_test,predictions))[[159  69]
 [ 22 976]]

             precision    recall  f1-score   support

          1       0.88      0.70      0.78       228
          5       0.93      0.98      0.96       998

avg / total       0.92      0.93      0.92      1226

**Great! Let's see what happens if we try to include TF-IDF to this process using a pipeline.**_____no_output_____# Using Text Processing

** Import TfidfTransformer from sklearn. **_____no_output_____

from sklearn.feature_extraction.text import  TfidfTransformer_____no_output_____

** Import Pipeline from sklearn. **_____no_output_____

from sklearn.pipeline import Pipeline_____no_output_____

** Now create a pipeline with the following steps:CountVectorizer(), TfidfTransformer(),MultinomialNB()**_____no_output_____

pipeline = Pipeline([
    ('bow', CountVectorizer()),  # strings to token integer counts
    ('tfidf', TfidfTransformer()),  # integer counts to weighted TF-IDF scores
    ('classifier', MultinomialNB()),  # train on TF-IDF vectors w/ Naive Bayes classifier
])_____no_output_____

## Using the Pipeline

**Time to use the pipeline! Remember this pipeline has all your pre-process steps in it already, meaning we'll need to re-split the original data (Remember that we overwrote X as the CountVectorized version. What we need is just the text**_____no_output_____### Train Test Split

**Redo the train test split on the yelp_class object.**_____no_output_____

X = yelp_class['text']
y = yelp_class['stars']
X_train, X_test, y_train, y_test = train_test_split(X, y,test_size=0.3,random_state=101)_____no_output_____

**Now fit the pipeline to the training data. Remember you can't use the same training data as last time because that data has already been vectorized. We need to pass in just the text and labels**_____no_output_____

# May take some time
pipeline.fit(X_train,y_train)_____no_output_____

### Predictions and Evaluation

** Now use the pipeline to predict from the X_test and create a classification report and confusion matrix. You should notice strange results.**_____no_output_____

predictions = pipeline.predict(X_test)_____no_output_____print(confusion_matrix(y_test,predictions))
print(classification_report(y_test,predictions))[[  0 228]
 [  0 998]]
             precision    recall  f1-score   support

          1       0.00      0.00      0.00       228
          5       0.81      1.00      0.90       998

avg / total       0.66      0.81      0.73      1226

Looks like Tf-Idf actually made things worse!_____no_output_____


# Notebook from Sirius2713/course-materials
Path: in-class-activities/08_Dask/8M_Intro/midway/Dask ML on Midway.ipynb

# Dask Machine Learning Workflows on the Midway Cluster_____no_output_____In this notebook, we are going to focus on two things:
1. Learning how to run interactive Dask jobs on the Midway Cluster.
2. Exploring Dask's machine learning functionality and its capacity to plug into existing scikit-learn workflows.

To accomplish these goals, we will continue to work with the Amazon Customer Reviews Dataset that we saw in the Spark unit. Recall that this dataset is available in the `/project2/macs30123/` directory on Midway2.

To run this notebook, you should log in to the Midway Cluster [via ThinLinc](https://midway2.rcc.uchicago.edu/main/). After you're logged in, run the following lines of code in a terminal window from a login node (to load the Python module we've been working with in the class and then install a couple of additional packages we'll be using):
```
module load python/anaconda-2019.03
pip install --user "dask[complete]" dask-jobqueue dask-ml --upgrade
```
Once you have installed these packages, start up a Jupyter Notebook from the login node (you'll be requesting resources via `dask-jobqueue`'s SLURM functions, on which you will run your code):
```
jupyter notebook
```

-----

*Credits: Thank you to [Yutao Chen](https://github.com/cytwill), who wrote the first version of this notebook (Fall 2020).*_____no_output_____## Launching a Dask Cluster via SLURM_____no_output_____To request Midway Cluster resources for our Dask Cluster, we have installed [dask-jobqueue](https://jobqueue.dask.org/en/latest/), which can be used to deploy Dask via common job queuing systems like SLURM (used in the Midway Cluster). Furthermore, it allows us to perform these interactions with SLURM an interactive context within a Jupyter Notebook.

Specifically, we'll be using the SLURMCluster function to request resources, using many of the same key words that we provide when write an `sbatch` script or provide arguments for `sinteractive` jobs. For instance, here, we're requesting 10 `broadwl` cores with 40GB of memory that are connected via an `ib0` (InfiniBand) interconnect (which we'll have available to us for 1 hour):_____no_output_____

from dask_jobqueue import SLURMCluster

# Compose SLURM script
cluster = SLURMCluster(queue='broadwl', cores=10, memory='40GB',
                       processes=10, walltime='01:00:00', interface='ib0',
                       job_extra=['--account=macs30123']
                      )

# Request resources
cluster.scale(jobs=1)/home/jclindaniel/.local/lib/python3.7/site-packages/distributed/node.py:164: UserWarning: Port 8787 is already in use.
Perhaps you already have a cluster running?
Hosting the HTTP server on port 35677 instead
  expected, actual

We'll need to wait a bit for our resources to be provisioned, but we can check on the progress of our resources via `squeue` as usual:_____no_output_____

! squeue -u jclindaniel             JOBID PARTITION     NAME     USER ST       TIME  NODES NODELIST(REASON)
          11235832   broadwl dask-wor jclindan  R       0:14      1 midway2-0172

Once our resources have been provisioned, we need to tell Dask that it should use them to run its computations. We can do this, by passing `cluster` into our `dask.distributed` client object._____no_output_____

from dask.distributed import Client

client = Client(cluster)
client_____no_output_____

That's all we need to do! If we want to see an interactive representation of what our workers are doing at any given time, we can click the link above, as in all the other Dask setups.

We're now ready to start using Dask. Let's start by reading our Customer Reviews Data (~17 GB in CSV form) into a Dask DataFrame:_____no_output_____

import dask.dataframe as dd

df = dd.read_csv('/project2/macs30123/AWS_book_reviews/*.csv')_____no_output_____df.head()_____no_output_____df.dtypes_____no_output_____

Note that if we look at the shape of the DataFrame, we can't actually see the number of rows immediately -- this is a `delayed` object that is only computed if we call the `.compute()` method. We need to do the same in order to describe our DataFrame as whole. Recall that, under the hood, Dask constructs an optimal task graph and only executes this series of tasks when we call the `.compute()` method:_____no_output_____

df.shape_____no_output_____df.shape[0].compute()_____no_output_____df.describe().compute()_____no_output_____

### Preprocessing_____no_output_____Next, let's perform some basic preprocessing for the book review dataset to obtain features for our machine learning demonstration later on. Intuitively, ID-like attributes are not likely to provide much useful information. Other attributes like total_votes and helpful_votes might contain more useful information about the ratings of books. We can make a mean calculation based on the groupby results to check our assumptions (here, across several different marketplaces):_____no_output_____

(df.groupby(by='marketplace')
   .mean()
   .compute()
)_____no_output_____

Not all of our data is numerical, though. We might want to encode categorial variables -- for instance, whether a user is a "Vine" member, or is a verified purchaser. These are both binary categories, so we'll just assign yes to "1" and no "0."_____no_output_____

df['vine_code'] = df['vine'].apply(lambda x:1 if x=='Y' else 0, meta=('vine', 'int64'))
df['verified_purchase_code'] = df['verified_purchase'].apply(lambda x:1 if x=='Y' else 0,
                                                                 meta=('verified_purchase', 'int64'))_____no_output_____df[['vine_code','verified_purchase_code']].head()_____no_output_____

We can also construct new columns in our DataFrame, derived from existing columns. To match our PySpark example from last week, let's create a column based on "Star Rating" that assigns any rating greater than 3 as a "good" review and any rating less than that as a "bad" one. We'll call this column `label`, as it will be the label we try to predict when fit our classifier later on:_____no_output_____

df['label'] = df['star_rating'].apply(lambda x: 1 if x > 3 else 0, meta=('star_rating', 'int64'))
df.head()_____no_output_____df.groupby('label').count().compute()['star_rating'] # not balanced_____no_output_____

OK, so we have some features and we have a label that we want to use them to predict (e.g. will a review be "bad" or "good?"). Let's arbitrarily pick some of these features and proceed with fitting our model (it's your job in Assignment 3 to construct better ones!):_____no_output_____

num_feature_df = df[['helpful_votes','total_votes','vine_code','verified_purchase_code']]_____no_output_____

Finally, before we fit our model, let's scale our data to decrease the effect of different scales for different attributes (for different scale methods, see [here](https://ml.dask.org/modules/api.html#module-dask_ml.preprocessing)):_____no_output_____

from dask_ml.preprocessing import MinMaxScaler

scaler = MinMaxScaler()
num_feature_df = scaler.fit_transform(num_feature_df)
num_feature_df.head()_____no_output_____

And we'll compute the overall size (and chunk size) of our datasets, so we can best decide how many samples we need for training and test data sets. Otherwise, we would not know the total sample size of our feature and label arrays (since they're distributed over multiple workers):_____no_output_____

import dask.array

label_array = (df['label'].to_dask_array()
                          .compute_chunk_sizes()
              )
num_feature_array = (num_feature_df.to_dask_array()
                                   .compute_chunk_sizes()
                    )_____no_output_____

### Machine Learning_____no_output_____Now that we know the size of our data (and how much of it is on different workers), we can split up our data into training and testing sets:_____no_output_____

from dask_ml.model_selection import train_test_split

X_train, X_test, y_train, y_test = train_test_split(num_feature_array,
                                            label_array,
                                            train_size=0.7,test_size=0.3,
                                            random_state=0)_____no_output_____X_train_____no_output_____y_test_____no_output_____

And let's recompute our class counts for our labels ("good" and "bad") so that we can use this to weight our classes in our classifier and try to deal with some of the class imbalance:_____no_output_____

class_counts = (df.groupby('label')
                  .count()
                  .compute()['star_rating']
               )_____no_output_____

Let's say that we wanted to fit a classifier to predict these labels based on our features using stochastic gradient descent (SGD). We might turn to Scikit-learn's [`SGDClassifier`](https://scikit-learn.org/stable/modules/generated/sklearn.linear_model.SGDClassifier.html#sklearn.linear_model.SGDClassifier) for this task, which fits a classifier using the stochastic gradient descent method via several different loss functions (e.g. hinge = SVM, log, = Logistic Regression, etc.), different penalties and so on.

Performing a search over this hyperparameter space would normally be challenging given the size of our data, but Dask conveniently allows us to perform such searches by providing a Scikit-learn classifier object to one of its "Search" functions. So, that's great! all we need to do is write a Scikit-learn classifier and Dask will automatically handle the parallelization of the hyperparameter search for us. Note that Dask-ML supports a similar workflow for Deep Learning packages like [PyTorch](https://ml.dask.org/pytorch.html), [Keras, and TensorFlow](https://ml.dask.org/keras.html).

To perform our hyperparameter search here, let's use an [incremental search](https://ml.dask.org/modules/generated/dask_ml.model_selection.IncrementalSearchCV.html):_____no_output_____

from dask_ml.model_selection import IncrementalSearchCV
from sklearn.linear_model import SGDClassifier
import numpy as np

clf = SGDClassifier(class_weight={0: 1, 1: class_counts[0] / class_counts[1]})
params = {'alpha': np.logspace(-4, 0, num=1000),
          'loss': ['hinge', 'log', 'modified_huber', 'squared_hinge'],
          'average': [True, False]}

search = IncrementalSearchCV(clf, params)
search.fit(X_train, y_train, classes=[0,1])/home/jclindaniel/.local/lib/python3.7/site-packages/dask_ml/model_selection/_incremental.py:1010: FutureWarning: decay_rate has been deprecated since Dask-ML v1.4.0.

    * Use InverseDecaySearchCV to use `decay_rate`
    * Specify decay_rate=None

  FutureWarning,
# check the best parameter
search.best_estimator______no_output_____# check the train accuracy
search.best_score______no_output_____# check the test accuracy
search.best_estimator_.score(X_test,y_test)_____no_output_____%matplotlib inline
# plot the confusion matrix
from sklearn.metrics import plot_confusion_matrix
import matplotlib.pyplot as plt

plot_confusion_matrix(search.best_estimator_, X_test, y_test)
plt.show()_____no_output_____

This model suffers a lot from data imbalance even after we specified weights (just as in the model we trained in PySpark, we're just predicting "Good" reviews regardless of what the true label is). Remember, though, we performed relatively naive feature selection here and you should be able to engineer better features in your assignments!

Regardless, from this short demo, you can see how seamless it is to perform Pandas DataFrame-like operations as well as Scikit-Learn machine learning tasks in Dask. If you're already working with these libraries in other capacities, this can be a great way to scale out your code, with very little additional effort on your part (e.g. writing new code in PySpark)!_____no_output_____


# Notebook from ka-rocha/LSSTC-DSFP-Sessions
Path: Sessions/Session11/Day2/FindingSources.ipynb

# Background Subtraction and Source Detection

**Version 0.1**

By Yusra AlSayyad (Princeton University)_____no_output_____Note: for portability, the examples in this notebook are one-dimensional and avoid using libraries. In practice on real astronomical images, I recommend using a library for astronomical image processing, e.g. AstroPy or the LSST Stack.

## Background Estimation

A prerequisite to this notebook is the `introductionToBasicStellarPhotometry.ipynb` notebook. We're going to use the same single stellar simulation, but with increasingly complex backgrounds.

First, setup the simulation and necessary imports_____no_output_____

import numpy as np
import matplotlib.pyplot as plt
from scipy.stats import norm
from matplotlib.ticker import MultipleLocator

%matplotlib notebook_____no_output_____def pixel_plot(pix, counts, fig=None, ax=None):
    '''Make a pixelated 1D plot'''
    if fig is None and ax is None:
        fig, ax = plt.subplots()

    ax.step(pix, counts,
            where='post')

    ax.set_xlabel('pixel number')
    ax.set_ylabel('relative counts')
    ax.xaxis.set_minor_locator(MultipleLocator(1))
    ax.xaxis.set_major_locator(MultipleLocator(5))
    fig.tight_layout()
    return fig, ax_____no_output_____# Define your PSF function phi()
# It is sufficient to copy and paste from
# your introductionToBasicStellarPhotometry noteboook

def phi(x, mu, fwhm):
    """Evalute the 1d PSF N(mu, sigma^2) along x

    Parameters
    ----------
    x : array-like of shape (n_pixels,)
        detector pixel number
    mu : float
        mean position of the 1D star
    fwhm : float
        Full-width half-maximum of the stellar profile on the detector

    Returns
    -------
    flux : array-like of shape (n_pixels,)
        Flux in each pixel of the input array
    """
    # complete

    return flux

# Define your image simulation function to
# It is sufficient to copy and paste from
# your introductionToBasicStellarPhotometry noteboook
# Note that the background S should now be supplied as
# an array of length (S) or a constant.

def simulate(x, mu, fwhm, S, F):
    """simulate a noisy stellar signal

    Parameters
    ----------
    x : array-like
        detector pixel number
    mu : float
        mean position of the 1D star
    fwhm : float
        Full-width half-maximum of the stellar profile on the detector
    S : float or array-like of len(x)
        Sky background for each pixel
    F : float
        Total stellar flux

    Returns
    -------
    noisy_counts : array-like (same shape as x)
        the (noisy) number of counts in each pixel
    """
    # complete

    return noisy_counts
_____no_output_____

## Problem 1) Simple 1-D Background Estimation

### Problem 1.1) Estimate a the background as a constant offset (order = 0)

For this problem we will use a simulated star with a constant background offset of $S=100$.

Background estimation is typically done by inspecting the distribution of counts in pixel bins. First inspect the distribution of counts, and pick an estimator for the background that is robust to the star (reduces biased fromthe star). Remember that we haven't done detection yet and don't know where the sources are.
_____no_output_____

# simulate the star
x = np.linspace(0, 100)
mu = 35
S = 100
fwhm = 5
F = 500

fig = plt.figure(figsize=(8,4))
ax = plt.subplot()
sim_star = # complete
pixel_plot(x, sim_star, fig=fig, ax=ax)
_____no_output_____# plot and inspect histogram

fig = plt.figure(figsize=(6,4))
# complete

S_estimate = # complete

print('My background estimate = {:.4f}'.format(S_estimate)
print('The mean pixel count = {:.4f}'.format( # complete_____no_output_____# plot your background model over the "image"

fig, ax = pixel_plot(x, sim_star)
pixel_plot(x, # complete, fig=fig, ax=ax)_____no_output_____

### Problem 1.2) Estimate a the background as a ramp/line (order = 1)

Now let's simulate a slightly more complicated background a linear ramp: $y = 3x + 100$. First simulate the same star with the new background. Then we're going to fit it using the following steps:
* Bin the image
* Use your robust estimator to estimate the background value per bin center
* Fit these bin center with a model

* A common simple model that astronomers use are Chebyshev polynomials. Chebyshevs have some very nice properties that prevent ringing at the edges of the fit window. Another popular way to "model" the bin centers is non-parametrically via interpolation._____no_output_____

# Double check that your simulate function can take S optionally as array-like_____no_output_____# Create and plot the image with S = 3*x + 100

sim_star = # complete
pixel_plot(x, sim_star)_____no_output_____# bin the image in 20-pixel bins

# complete
bin_centers = # complete
bin_values = # complete _____no_output_____# Fit the bin_values vs bin_centers with a 1st-order chebyshev polynomial
# Evaluate your model for the full image
# hint: look up np.polynomial.chebyshev.chebfit and np.polynomial.chebyshev.chebeval

# complete

# Replot the image:
fig, ax = pixel_plot(x, sim_star)
# binned values
ax.plot(bin_centers, bin_values, 'o')

# Overplot your background model:
# complete_____no_output_____# Finally plot your background subtracted image:

# complete_____no_output_____

### Problem 1.3) Estimate a more realistic background (still in 1D)

Now repeat the the exercise in problem 1.2 with a more complex background. _____no_output_____

SIGMA_PER_FWHM = 2*np.sqrt(2*np.log(2))

fwhm = 5
x = np.linspace(0, 100)
background = 1000*norm.pdf(x, 50, 18) + 100*norm.pdf(x, 20, fwhm/SIGMA_PER_FWHM) + 100*norm.pdf(x, 60, fwhm/SIGMA_PER_FWHM)

sim_star3 = simulate(x=x, mu=35, fwhm=fwhm, S=background, F=200)
fig, ax = pixel_plot(x, sim_star3)_____no_output_____

#### 1.3.1) Bin the image. Plot the bin centers. What bin size did you pick? _____no_output_____

bin_centers = # complete

bin_values = # complete_____no_output_____# overplot the binned esimtates:
ax.plot(bin_centers, bin_values, 'o')_____no_output_____

#### 1.3.2) Spatially model the binned estimates (bin_values vs bin_centers) as a chebyshev polynomial.

Evaluate your model on the image grid and overplot.(what degree/order did you pick?)_____no_output_____

# complete_____no_output_____

#### 1.3.3) Subtract off the model and plot the "background-subtracted image." _____no_output_____

# Plot the background subtracted image
_____no_output_____

## Problem 2) Finding Sources

Now that we have subtracted background image,  let’s look for sources.  In the lecture we focused on the matched filter interpretation.  Here we will go into the hypothesis testing and maximum likelihood interpretations.

### Maximum likelihood interpretation:

 Assume that we know there is a point source somewhere in this image.   We want to find  to pixel that has the maximum likelihood of having a point source centered on it.  recall from session 10, the probability for an  individual observation $I_i$ is:

$$P(X_i) = \frac{1}{\sqrt{2\pi\sigma_i^2}} \exp{-\frac{(X_i - y_i)^2}{2\sigma_i^2}}$$

Here: $X_i$ is the pixel value of pixel $i$ in the image and $y_i$ is the model prediction for that pixel.

The model in this case is your `simulate()` function from the `IntroductionToBasicStellarPhotometry.ipynb` notebook: the PSF evaluated at a distance from the center multiplied by the flux amplitude: $F * \phi(x - x_{center}) + S$ Where $F$ is the flux amplitude $\phi$ is the PSF profile (a function of position), and $S$ is the background.

Plug it in:
$$P(X_i) = \frac{1}{\sqrt{2\pi\sigma_i^2}} \exp{-\frac{(X_i - (F * \phi_i(x_{center}) + S))^2}{2\sigma_i^2}}$$

### Hypothesis test interpretation:

If I were teaching source detection to my non-scientist, college stats 101 students, I'd frame the problem like this:

Pretend you have an infinitely large population of pixels Say I know definitively, that the arbitrarily large population of pixels is drawn from $N(0,100)$ (has a variance of 10). I have another sample of 13 pixels. I want to tesst the hypothesis that those 13 pixels were drawn from the $N(0,100)$ pop too.

Test the hypothesis that your subsample of 13 pixels were drawn from the larger sample.
* $H_0$: $\mu = 0$
* $H_A$: $\mu > 0$

$$z = \frac{\bar{x} - \mu}{\sigma / \sqrt{n}} $$
$$z = \frac{\sum{x}/13 - 0}{10 /\sqrt{13}} $$

OK, if this is coming back now, let's replace this with our *real* estimator for PSF flux, which is a *weighted mean* of the pixels where the weights are the PSF $\phi_i$. Whenever I forget the formulas for weighted means, I consult the [wikipedia page](https://en.wikipedia.org/wiki/Weighted_arithmetic_mean).

Now tweak it for a weighted mean (PSF flux):

$$ z = \frac{\sum{\phi_i x_i} - \mu} {\sqrt{ \sum{\phi_i^2 \sigma_i^2}}} $$

Where the denominator is from the variance estimate of a weighted mean.  For constant $\sigma$ it reduces to $\sigma_{\bar{x}}^2 = \sigma^2 \sum{\phi^2_i}$, and for a constant $\phi$ this reduces to $\sigma_{\bar{x}}^2 = \sigma^2 /n$, the denomiator in the simple mean example above. Replace $\mu=0$ again.

$$ z = \frac{\sum{\phi_i x_i}} {\sqrt{ \sum{\phi_i^2 \sigma_i^2}}} $$
Our detection map is just the nominator for each pixel! We deal with the denominator later when choosing the thresholding, but we could just as easily divide the whole image by the denominator and have a z-score image!

### 2.0) Plot the problem image_____no_output_____

# set up simulation
x = np.linspace(0, 100)
mu = 35
S = 100
fwhm = 5
F = 300

fig = plt.figure(figsize=(8,4))
ax = plt.subplot()
sim_star = simulate(x, mu=mu, fwhm=fwhm, S=S, F=F)

# To simplify this pretend \sqrt{\sum \phi_i^2 \sigma_i^2we know for sure the background = 100
# Plots the backround subtracted image
image = sim_star - 100
pixel_plot(x, image, fig=fig, ax=ax)_____no_output_____

### 2.1) Make a kernel for the PSF.

Properties of kernels: They're centered at x=0 (which also means that they have an odd number of pixels) and sum up to 1. You can use your `phi()`._____no_output_____

xx = # complete
kernel = # complete

# plot your kernel_____no_output_____

### 2.2) Correlate the image with the PSF kernel,

and plot the result.

What are the tradeoffs when choosing the size of your PSF kernel? What happens if its too big? What happens if it's too small.?

**hint:** `scipy.signal.convolve`_____no_output_____

# detection_image = # complete

# plot your detection image
# Note: pay attention to how scipy.signal.convolve handles the edges. _____no_output_____

### 2.3) Detect pixels

for which the null hypothesis that there's no source centered there is ruled out at the 5$\sigma$ level._____no_output_____

# Using a robust estimator for the detection image standard deviation,
# Compute the 5 sigma threshold

threshold_value = # complete
print('5 sigma threshold value is = {:.4f}'.format(threshold_value))_____no_output_____

### 2.4) Dilate footprint to provide a window or region for the point source.
We will use this window to compute the centroid and total flux of the star in the next two lessons. In the meantime, compute the flux like we did in introductionToStellarPhotometry assuming the input center. _____no_output_____

# complete_____no_output_____

## Challenge problem A

Combine problem 1 and 2 to iterate background estimation and source detection, masking the pixels with detections. Use the more complex background from 1.3. _____no_output_____## Challenge problem B

Repeat challenge problem A in two dimensions_____no_output_____


# Notebook from lapisco/pdi-python-lectures
Path: notebooks/Complementar/Outros_01/T18/T18 - Code.ipynb

# Introdução_____no_output_____O desenvolvimentos de métodos de mofologia matemática baseiam-se em teoridas de conjuntos da matemática. Esta abordagem considera a imagem como o conjunto universo, objetos com conjuntos contidos no conjunto universo e estuda o resultado de operadores de deslocamento livre sobre a imagem. As operações morfológica visam promover melhoramentos na imagem, como remoção de ruído, junção de elementos separados, achar vales e picos em uma imagem. Alguns dos métodos são os de (i) erosão e (ii) dilatação, são ditos como elementares neste âmbito.

A operação de erosão consiste na alteração morfológica da imagem e sua operação computa o mínimo local dentro da área do elemento estruturante enquanto que a dilatação_____no_output_____# Discussões sobre os métodos_____no_output_____O métodos morfológicos são simples operações baseadas na forma do imagem, cujo objetiva obter melhorasmentos da mesma. A entrada para o método é a imagem propriamente dita (no exemplo a seguir utilza-se uma imagem binária para melhor entendimento) e o elemento estruturante, que irá operar sobre a imagem, bit a bit.

Os métodos implementados a seguir são os métodos da erosão e dilatação, utilizando três elementos estruturantes distintos._____no_output_____

import cv2
import numpy as np
import matplotlib.pyplot as plt
import pywt_____no_output_____

- Abrir imagem:_____no_output_____

img = np.array(cv2.imread('N-inv.png'))_____no_output_____plt.figure(1)
plt.imshow(img)
#plt.axis("off")
plt.title("Imagem original")
plt.show()_____no_output_____

### Erosão:_____no_output_____A erosão é um método que promove diminuição os limites do objeto e do seu fundo. _____no_output_____- Elementro Estruturante 01:_____no_output_____

def EE (size, type):
    if type == 1:
        # square
        kernel = np.ones(size, np.uint8)
        return kernel
    if type == 2:
        # star
        kernel = np.ones(size, np.uint8)
        center = np.uint8((size[0]/2))
        for k in range(0,size[0]):
            kernel[center,k] = 0
            kernel[k,center] = 0
        return kernel
    if type == 3:
        # something 01
        kernel = np.ones(size, np.uint8)
        center = np.uint8((size[0]/2))
        kernel[:,0] = 0
        kernel[center,1] = 0
        return kernel_____no_output_____size = np.array([3,3])
el_est = EE(size, 1)

img_erosion = cv2.erode(img, el_est,  iterations=2)

print(el_est)[[1 1 1]
 [1 1 1]
 [1 1 1]]
plt.figure(figsize=(9,9))
plt.subplot(131)
plt.imshow(img)
plt.axis("off")
plt.title("Imagem original")

plt.subplot(132)
plt.imshow(img_erosion)
plt.axis("off")
plt.title("Imagem erodida")

plt.subplot(133)
plt.imshow(el_est, 'gray', interpolation='nearest')
plt.axis("off")
plt.grid()
plt.title("El. Estruturante")
plt.show()_____no_output_____

- Elementos Estruturante 02:_____no_output_____

size = np.array([3,3])
el_est = EE(size, 2)

img_erosion = cv2.erode(img, el_est,  iterations=2)

print(el_est)[[1 0 1]
 [0 0 0]
 [1 0 1]]
plt.figure(figsize=(9,9))
plt.subplot(131)
plt.imshow(img)
plt.axis("off")
plt.title("Imagem original")

plt.subplot(132)
plt.imshow(img_erosion)
plt.axis("off")
plt.title("Imagem erodida")

plt.subplot(133)
plt.imshow(el_est, 'gray', interpolation='nearest')
plt.axis("off")
plt.grid()
plt.title("El. Estruturante")
plt.show()_____no_output_____

- Elementos Estruturante 03:_____no_output_____

size = np.array([3,3])
el_est = EE(size, 3)

img_erosion = cv2.erode(img, el_est,  iterations=2)

print(el_est)[[0 1 1]
 [0 0 1]
 [0 1 1]]
plt.figure(figsize=(9,9))
plt.subplot(131)
plt.imshow(img)
plt.axis("off")
plt.title("Imagem original")

plt.subplot(132)
plt.imshow(img_erosion)
plt.axis("off")
plt.title("Imagem erodida")

plt.subplot(133)
plt.imshow(el_est, 'gray', interpolation='nearest')
plt.axis("off")
plt.grid()
plt.title("El. Estruturante")
plt.show()_____no_output_____

Todos os elementos estruturantes operam da mesma forma na imagem, retraíndo o branco sobre o fundo preto e diminuindo as barreiras entre o objeto e o seu fundo, porém há diferenças:

- O elemento estruturante 1 promovre erosão em todos os sentidos;
- O elemento estruturante 2 promove erosão em apenas dis sentidos;
- O elemento estruturante 3 promove erosão de acordo com a sua forma;_____no_output_____### Dilatação:_____no_output_____A dilatação promove resultados análogos a erosão: aumento da região branca sobre a região preta, aumentod o objeto sobre o fundo._____no_output_____- Elementro Estruturante 01:_____no_output_____

size = np.array([3,3])
el_est = EE(size, 1)

img_erosion = cv2.dilate(img, el_est,  iterations=2)

print(el_est)[[1 1 1]
 [1 1 1]
 [1 1 1]]
plt.figure(figsize=(9,9))
plt.subplot(131)
plt.imshow(img)
plt.axis("off")
plt.title("Imagem original")

plt.subplot(132)
plt.imshow(img_erosion)
plt.axis("off")
plt.title("Imagem dilatada")

plt.subplot(133)
plt.imshow(el_est, 'gray', interpolation='nearest')
plt.axis("off")
plt.grid()
plt.title("El. Estruturante")
plt.show()_____no_output_____

- Elementos Estruturante 02:_____no_output_____

size = np.array([3,3])
el_est = EE(size, 2)

img_erosion = cv2.dilate(img, el_est,  iterations=2)

print(el_est)[[1 0 1]
 [0 0 0]
 [1 0 1]]
plt.figure(figsize=(9,9))
plt.subplot(131)
plt.imshow(img)
plt.axis("off")
plt.title("Imagem original")

plt.subplot(132)
plt.imshow(img_erosion)
plt.axis("off")
plt.title("Imagem dilatada")

plt.subplot(133)
plt.imshow(el_est, 'gray', interpolation='nearest')
plt.axis("off")
plt.grid()
plt.title("El. Estruturante")
plt.show()_____no_output_____

- Elementos Estruturante 03:_____no_output_____

size = np.array([3,3])
el_est = EE(size, 3)

img_erosion = cv2.dilate(img, el_est,  iterations=2)

print(el_est)[[0 1 1]
 [0 0 1]
 [0 1 1]]
plt.figure(figsize=(9,9))
plt.subplot(131)
plt.imshow(img)
plt.axis("off")
plt.title("Imagem original")

plt.subplot(132)
plt.imshow(img_erosion)
plt.axis("off")
plt.title("Imagem dilatada")

plt.subplot(133)
plt.imshow(el_est, 'gray', interpolation='nearest')
plt.axis("off")
plt.grid()
plt.title("El. Estruturante")
plt.show()_____no_output_____

Em todas as dilatações os resultados são parecidos, pois há uma sobreposição do branco sobre o fundo preto e aumento da área do objeto em branco. A aplicação dos elementos estruturantes promove as seguinte diferenças:

- O elemento estruturante 1 promovre dilatação em todos os sentidos;
- O elemento estruturante 2 promove dilatação em apenas dis sentidos;
- O elemento estruturante 3 promove dilatação de acordo com a sua forma;_____no_output_____## Dualidade entre erosão e dilatação_____no_output_____Utilizando a mesma imagem, porém com as cores invertidas é fácil visualizar a aplicação dos métodos:
- A erosão promove uma diminuição da região branca, logo um aumento do objeto
- A dilatação promove um aumento da região branca, uma diminuição do objeto._____no_output_____

img2 = np.array(cv2.imread('N.png'))

plt.figure(1)
plt.imshow(img2)
plt.axis("off")
#plt.title("Imagem original")
plt.show()_____no_output_____size = np.array([3,3])
el_est = EE(size, 3)

img2_erosion = cv2.erode(img2, el_est,  iterations=2)

size = np.array([3,3])
el_est = EE(size, 2)

img2_dilate = cv2.dilate(img2, el_est,  iterations=2)

plt.figure(figsize=(9,9))
plt.subplot(121)
plt.imshow(img2_erosion)
plt.axis("off")
plt.title("Imagem erodida")

plt.subplot(122)
plt.imshow(img2_dilate)
plt.axis("off")
plt.title("Imagem dilatada")

plt.show()_____no_output_____

# Conclusão_____no_output_____Os métodos morfológicos de erosão e dilatação são operadores elementares que funcionam de forma dual, dependendo da imagem em questão. Em imagens binárias promovem diminuição e crescimento de regiões brancas respectivamente, e espera-se que em imagens tons de cinza promovam efeitos semelhante a filtros passa-alta a e passa-baixa, removendo ruídos, ressaltando bordas e outras operações._____no_output_____