From 4974e7500ef9f5325eabcd940e05b9171d265788 Mon Sep 17 00:00:00 2001 From: nathanhubens Date: Wed, 1 Jul 2026 11:05:17 +0200 Subject: [PATCH] refactor!: move sensitivity analysis out of fasterai (BREAKING) analyze_sensitivity, SensitivityAnalyzer, SensitivityResult, LayerSensitivity moved out of fasterai into the higher-level FasterAI workflow package (decisions and allocation logic live there; fasterai holds compression mechanisms only). fasterai.analyze.sensitivity now raises a helpful ImportError via a PEP-562 __getattr__ stub. _modidx pruned; quarto sidebar entries and the sensitivity tutorial removed. Validated: nbdev-test stub green; import fasterai OK; stub raises with guidance. --- CHANGELOG.md | 5 + fasterai/_modidx.py | 72 +- fasterai/analyze/sensitivity.py | 819 +--------- nbs/_quarto.yml | 6 - nbs/analyze/sensitivity.ipynb | 1848 +---------------------- nbs/tutorials/analyze/sensitivity.ipynb | 1015 ------------- 6 files changed, 45 insertions(+), 3720 deletions(-) delete mode 100644 nbs/tutorials/analyze/sensitivity.ipynb diff --git a/CHANGELOG.md b/CHANGELOG.md index 3876d56..4faa615 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,11 @@ +## Unreleased + +### Breaking Changes +- **BREAKING**: Sensitivity analysis (`analyze_sensitivity`, `SensitivityAnalyzer`, `SensitivityResult`, `LayerSensitivity`) moved out of fasterai into the higher-level FasterAI workflow package; `fasterai.analyze.sensitivity` now raises `ImportError`. fasterai holds compression mechanisms only. See the FasterAI docs for the new import path. + ## 0.3.3 ### Bug Fixes diff --git a/fasterai/_modidx.py b/fasterai/_modidx.py index 9a396b6..1c02b2e 100644 --- a/fasterai/_modidx.py +++ b/fasterai/_modidx.py @@ -5,76 +5,8 @@ 'doc_host': 'https://FasterAI-Labs.github.io', 'git_url': 'https://github.com/FasterAI-Labs/fasterai', 'lib_path': 'fasterai'}, - 'syms': { 'fasterai.analyze.sensitivity': { 'fasterai.analyze.sensitivity.LayerSensitivity': ( 'analyze/sensitivity.html#layersensitivity', - 'fasterai/analyze/sensitivity.py'), - 'fasterai.analyze.sensitivity.LayerSensitivity.as_dict': ( 'analyze/sensitivity.html#layersensitivity.as_dict', - 'fasterai/analyze/sensitivity.py'), - 'fasterai.analyze.sensitivity.SensitivityAnalyzer': ( 'analyze/sensitivity.html#sensitivityanalyzer', - 'fasterai/analyze/sensitivity.py'), - 'fasterai.analyze.sensitivity.SensitivityAnalyzer.__init__': ( 'analyze/sensitivity.html#sensitivityanalyzer.__init__', - 'fasterai/analyze/sensitivity.py'), - 'fasterai.analyze.sensitivity.SensitivityAnalyzer._apply_sparsity': ( 'analyze/sensitivity.html#sensitivityanalyzer._apply_sparsity', - 'fasterai/analyze/sensitivity.py'), - 'fasterai.analyze.sensitivity.SensitivityAnalyzer._apply_structural_pruning': ( 'analyze/sensitivity.html#sensitivityanalyzer._apply_structural_pruning', - 'fasterai/analyze/sensitivity.py'), - 'fasterai.analyze.sensitivity.SensitivityAnalyzer._apply_weight_quantization': ( 'analyze/sensitivity.html#sensitivityanalyzer._apply_weight_quantization', - 'fasterai/analyze/sensitivity.py'), - 'fasterai.analyze.sensitivity.SensitivityAnalyzer._build_dependency_groups': ( 'analyze/sensitivity.html#sensitivityanalyzer._build_dependency_groups', - 'fasterai/analyze/sensitivity.py'), - 'fasterai.analyze.sensitivity.SensitivityAnalyzer._channel_signature': ( 'analyze/sensitivity.html#sensitivityanalyzer._channel_signature', - 'fasterai/analyze/sensitivity.py'), - 'fasterai.analyze.sensitivity.SensitivityAnalyzer._cleanup_sparsifier': ( 'analyze/sensitivity.html#sensitivityanalyzer._cleanup_sparsifier', - 'fasterai/analyze/sensitivity.py'), - 'fasterai.analyze.sensitivity.SensitivityAnalyzer._clone_model': ( 'analyze/sensitivity.html#sensitivityanalyzer._clone_model', - 'fasterai/analyze/sensitivity.py'), - 'fasterai.analyze.sensitivity.SensitivityAnalyzer._compute_qparams_asymmetric': ( 'analyze/sensitivity.html#sensitivityanalyzer._compute_qparams_asymmetric', - 'fasterai/analyze/sensitivity.py'), - 'fasterai.analyze.sensitivity.SensitivityAnalyzer._compute_qparams_symmetric': ( 'analyze/sensitivity.html#sensitivityanalyzer._compute_qparams_symmetric', - 'fasterai/analyze/sensitivity.py'), - 'fasterai.analyze.sensitivity.SensitivityAnalyzer._create_activation_quantize_hook': ( 'analyze/sensitivity.html#sensitivityanalyzer._create_activation_quantize_hook', - 'fasterai/analyze/sensitivity.py'), - 'fasterai.analyze.sensitivity.SensitivityAnalyzer._fake_quantize_per_channel': ( 'analyze/sensitivity.html#sensitivityanalyzer._fake_quantize_per_channel', - 'fasterai/analyze/sensitivity.py'), - 'fasterai.analyze.sensitivity.SensitivityAnalyzer._fake_quantize_per_tensor': ( 'analyze/sensitivity.html#sensitivityanalyzer._fake_quantize_per_tensor', - 'fasterai/analyze/sensitivity.py'), - 'fasterai.analyze.sensitivity.SensitivityAnalyzer._fresh_copy': ( 'analyze/sensitivity.html#sensitivityanalyzer._fresh_copy', - 'fasterai/analyze/sensitivity.py'), - 'fasterai.analyze.sensitivity.SensitivityAnalyzer._get_compressible_layers': ( 'analyze/sensitivity.html#sensitivityanalyzer._get_compressible_layers', - 'fasterai/analyze/sensitivity.py'), - 'fasterai.analyze.sensitivity.SensitivityAnalyzer._init_sparsifier': ( 'analyze/sensitivity.html#sensitivityanalyzer._init_sparsifier', - 'fasterai/analyze/sensitivity.py'), - 'fasterai.analyze.sensitivity.SensitivityAnalyzer._out_dim': ( 'analyze/sensitivity.html#sensitivityanalyzer._out_dim', - 'fasterai/analyze/sensitivity.py'), - 'fasterai.analyze.sensitivity.SensitivityAnalyzer._quiet': ( 'analyze/sensitivity.html#sensitivityanalyzer._quiet', - 'fasterai/analyze/sensitivity.py'), - 'fasterai.analyze.sensitivity.SensitivityAnalyzer._remove_activation_hooks': ( 'analyze/sensitivity.html#sensitivityanalyzer._remove_activation_hooks', - 'fasterai/analyze/sensitivity.py'), - 'fasterai.analyze.sensitivity.SensitivityAnalyzer._restore_layer': ( 'analyze/sensitivity.html#sensitivityanalyzer._restore_layer', - 'fasterai/analyze/sensitivity.py'), - 'fasterai.analyze.sensitivity.SensitivityAnalyzer._setup_activation_hooks': ( 'analyze/sensitivity.html#sensitivityanalyzer._setup_activation_hooks', - 'fasterai/analyze/sensitivity.py'), - 'fasterai.analyze.sensitivity.SensitivityAnalyzer.analyze': ( 'analyze/sensitivity.html#sensitivityanalyzer.analyze', - 'fasterai/analyze/sensitivity.py'), - 'fasterai.analyze.sensitivity.SensitivityAnalyzer.sweep': ( 'analyze/sensitivity.html#sensitivityanalyzer.sweep', - 'fasterai/analyze/sensitivity.py'), - 'fasterai.analyze.sensitivity.SensitivityResult': ( 'analyze/sensitivity.html#sensitivityresult', - 'fasterai/analyze/sensitivity.py'), - 'fasterai.analyze.sensitivity.SensitivityResult.__post_init__': ( 'analyze/sensitivity.html#sensitivityresult.__post_init__', - 'fasterai/analyze/sensitivity.py'), - 'fasterai.analyze.sensitivity.SensitivityResult.as_dict': ( 'analyze/sensitivity.html#sensitivityresult.as_dict', - 'fasterai/analyze/sensitivity.py'), - 'fasterai.analyze.sensitivity.SensitivityResult.plot': ( 'analyze/sensitivity.html#sensitivityresult.plot', - 'fasterai/analyze/sensitivity.py'), - 'fasterai.analyze.sensitivity.SensitivityResult.summary': ( 'analyze/sensitivity.html#sensitivityresult.summary', - 'fasterai/analyze/sensitivity.py'), - 'fasterai.analyze.sensitivity.SensitivityResult.to_dataframe': ( 'analyze/sensitivity.html#sensitivityresult.to_dataframe', - 'fasterai/analyze/sensitivity.py'), - 'fasterai.analyze.sensitivity.SensitivityResult.to_layer_targets': ( 'analyze/sensitivity.html#sensitivityresult.to_layer_targets', - 'fasterai/analyze/sensitivity.py'), - 'fasterai.analyze.sensitivity.SensitivityResult.top': ( 'analyze/sensitivity.html#sensitivityresult.top', - 'fasterai/analyze/sensitivity.py'), - 'fasterai.analyze.sensitivity.analyze_sensitivity': ( 'analyze/sensitivity.html#analyze_sensitivity', - 'fasterai/analyze/sensitivity.py')}, + 'syms': { 'fasterai.analyze.sensitivity': { 'fasterai.analyze.sensitivity.__getattr__': ( 'analyze/sensitivity.html#__getattr__', + 'fasterai/analyze/sensitivity.py')}, 'fasterai.core.all': {}, 'fasterai.core.criteria': { 'fasterai.core.criteria.Criteria': ('core/criteria.html#criteria', 'fasterai/core/criteria.py'), 'fasterai.core.criteria.Criteria.__call__': ( 'core/criteria.html#criteria.__call__', diff --git a/fasterai/analyze/sensitivity.py b/fasterai/analyze/sensitivity.py index 3ba6ba2..cbb5bab 100644 --- a/fasterai/analyze/sensitivity.py +++ b/fasterai/analyze/sensitivity.py @@ -1,810 +1,15 @@ # AUTOGENERATED! DO NOT EDIT! File to edit: ../../nbs/analyze/sensitivity.ipynb. -# %% ../../nbs/analyze/sensitivity.ipynb #imports -from __future__ import annotations -import torch -import torch.nn as nn -import numpy as np -from copy import deepcopy -from dataclasses import dataclass, field, asdict -from typing import Callable, Any, Literal -from collections import OrderedDict -from contextlib import contextmanager -from fastcore.basics import store_attr - -# fasterai imports (relative within fasterai package) -from ..sparse.all import Sparsifier -from ..prune.all import Pruner -from ..core.all import large_final, Criteria, Granularities - # %% auto #0 -__all__ = ['LayerSensitivity', 'SensitivityResult', 'SensitivityAnalyzer', 'analyze_sensitivity'] - -# %% ../../nbs/analyze/sensitivity.ipynb #dataclasses -@dataclass(slots=True) -class LayerSensitivity: - """Sensitivity result for a single layer.""" - name: str # layer name - layer_type: str # e.g., "Conv2d", "Linear" - params: int # number of parameters - baseline_metric: float # metric before compression - compressed_metric: float # metric after compression - delta: float # metric change (positive = degradation) - group_id: int | None = None # pruning: dependency-group id (coupled layers share one); None for sparsity/quant - group_members: list[str] = field(default_factory=list) # names of all layers co-pruned with this one - prunable: bool = True # False if the layer could not be pruned (genuine no-op) — not "robust" - - def as_dict(self) -> dict[str, Any]: - """Convert to dictionary.""" - return asdict(self) - - -@dataclass(slots=True) -class SensitivityResult: - """Structured result from sensitivity analysis.""" - compression_type: str # "sparsity", "pruning", "quantization" - compression_level: float # e.g., 50 for 50% sparsity - baseline_metric: float # overall baseline metric - layers: list[LayerSensitivity] # per-layer results - metric_name: str = "accuracy" # name of the metric - higher_is_better: bool = True # whether higher metric is better - _results: list[LayerSensitivity] = field(default=None, init=False, repr=False) # for top() compatibility - - def __post_init__(self): - self._results = self.layers # for compatibility with top() pattern - - def as_dict(self) -> dict[str, Any]: - """Convert to flat dictionary.""" - return { - "compression_type": self.compression_type, - "compression_level": self.compression_level, - "baseline_metric": self.baseline_metric, - "metric_name": self.metric_name, - "higher_is_better": self.higher_is_better, - "layers": [l.as_dict() for l in self.layers], - } - - def top( - self, - n: int = 5, # number of layers to return - *, - most_sensitive: bool = True, # True=highest delta (fragile), False=lowest (robust) - ) -> list[LayerSensitivity]: - """Return top N most or least sensitive layers. - - Layers that could not be pruned (`prunable=False`) are excluded — a no-op - prune is NOT evidence of robustness, so it must never rank as compressible. - """ - candidates = [l for l in self.layers if l.prunable] - sorted_layers = sorted(candidates, key=lambda x: x.delta, reverse=most_sensitive) - return sorted_layers[:n] - - def summary( - self, - *, - top: int = 5, # number of layers to show per category - ) -> None: - """Print a formatted summary of sensitivity analysis.""" - print(f"{'═' * 60}") - print(f"Sensitivity Analysis: {self.compression_type} @ {self.compression_level}%") - print(f"{'═' * 60}") - print(f" Baseline {self.metric_name}: {self.baseline_metric:.4f}") - print(f" Layers analyzed: {len(self.layers)}") - print() - - def _line(i, layer): - sign = "+" if layer.delta > 0 else "" - grp = f" [group {layer.group_id}]" if layer.group_id is not None else "" - print(f" {i}. {layer.name:30} Δ={sign}{layer.delta:.4f}{grp}") - - # Most sensitive (fragile) layers - print(f" 🔴 Most Sensitive (fragile):") - for i, layer in enumerate(self.top(top, most_sensitive=True), 1): - _line(i, layer) - print() - - # Most robust layers - print(f" 🟢 Most Robust (compressible):") - for i, layer in enumerate(self.top(top, most_sensitive=False), 1): - _line(i, layer) - - # Layers that could not be pruned in isolation — surfaced, NOT ranked as robust - not_prunable = [l for l in self.layers if not l.prunable] - if not_prunable: - print() - print(f" ⚪ Not prunable in isolation ({len(not_prunable)}):") - for i, layer in enumerate(not_prunable[:top], 1): - print(f" {i}. {layer.name:30} (group {layer.group_id})") - - def to_dataframe(self): - """Convert to pandas DataFrame.""" - import pandas as pd - rows = [layer.as_dict() for layer in self.layers] - return pd.DataFrame(rows) - - def to_layer_targets( - self, - model: nn.Module, # model (used for parameter counts) - target_pct: float = 50, # target mean compression percentage - min_pct: float = 0, # minimum compression for any layer - max_pct: float = 90, # maximum compression for any layer - gamma: float = 1.0, # exponent for sensitivity scaling (higher = more differentiation) - ) -> dict[str, float]: - """Convert sensitivity to non-uniform per-layer compression targets. - - High sensitivity layers get lower compression, robust layers get higher. - Uses parameter-weighted optimization to hit target_pct exactly. - - Coupled layers (same `group_id`) are optimized as a SINGLE knob — they - physically share one pruning ratio — then the group's target is expanded - back to every member, so the returned dict stays per-layer. Layers that - are not prunable receive `min_pct`. - """ - if not self.layers: - return {} - - # Convert to fractions - target = target_pct / 100.0 - smin = min_pct / 100.0 - smax = max_pct / 100.0 - - # Collapse to one entry per dependency group (a singleton group per layer when - # group_id is None, e.g. sparsity/quant). This stops a coupled group of N layers - # from being double-counted as N independent knobs in the weighted-mean solve. - prunable = [l for l in self.layers if l.prunable] - not_prunable = [l for l in self.layers if not l.prunable] - groups: OrderedDict[Any, dict] = OrderedDict() - for l in prunable: - key = l.group_id if l.group_id is not None else ("solo", l.name) - g = groups.setdefault(key, {"names": [], "params": 0.0, "delta": 0.0}) - g["names"].append(l.name) - g["params"] += float(l.params) - g["delta"] = max(g["delta"], max(0.0, l.delta)) # delta is shared within a group - - # Not-prunable layers are protected at min_pct and kept out of the optimization - targets: dict[str, float] = {l.name: round(float(min_pct), 2) for l in not_prunable} - if not groups: - return targets - - deltas = np.array([g["delta"] for g in groups.values()], dtype=float) - weights = np.array([g["params"] for g in groups.values()], dtype=float) - group_names = [g["names"] for g in groups.values()] - - if weights.sum() == 0: - for names in group_names: - for n in names: targets[n] = target_pct - return targets - - # Normalize sensitivity and invert (high sensitivity -> low compression) - if np.allclose(deltas, deltas[0]): - s0 = np.full_like(deltas, target) - else: - norm = (deltas - deltas.min()) / (np.ptp(deltas) + 1e-12) - inv = (1.0 - norm) ** gamma - s0 = smin + (smax - smin) * inv - - # Binary search for lambda to hit target weighted mean - W = weights.sum() - tgt = target * W - - def f(lam): - s = np.clip(s0 + lam, smin, smax) - return float(np.dot(weights, s)) - - # Find lambda via bisection - lam_lo, lam_hi = -1.0, 1.0 - while f(lam_lo) > tgt: - lam_lo *= 2 - while f(lam_hi) < tgt: - lam_hi *= 2 - - for _ in range(60): - lam_mid = 0.5 * (lam_lo + lam_hi) - if f(lam_mid) < tgt: - lam_lo = lam_mid - else: - lam_hi = lam_mid - - final_s = np.clip(s0 + 0.5 * (lam_lo + lam_hi), smin, smax) - - # Expand each group's target back to all its member layers (they share the ratio) - for names, s in zip(group_names, final_s): - for n in names: - targets[n] = round(s * 100, 2) - return targets - - def plot( - self, - figsize: tuple = (12, 5), # figure size (width, height) - ) -> None: - """Plot sensitivity as a bar chart.""" - import matplotlib.pyplot as plt - - names = [l.name for l in self.layers] - deltas = np.array([l.delta for l in self.layers], dtype=float) - - # Color by sensitivity - norm = (deltas - deltas.min()) / (np.ptp(deltas) + 1e-9) - colors = plt.cm.RdYlGn_r(norm) # Red=sensitive, Green=robust - - plt.figure(figsize=figsize) - plt.bar(range(len(deltas)), deltas, color=colors) - plt.axhline(0, color='gray', linewidth=1.2, linestyle='--') - plt.xticks(range(len(names)), names, rotation=60, ha='right') - plt.ylabel(f"{self.metric_name} drop (Δ)") - plt.title(f"Layer Sensitivity to {self.compression_type} @ {self.compression_level}%", - pad=12, weight='bold') - plt.grid(axis='y', linestyle=':', alpha=0.6) - plt.tight_layout() - plt.show() - -# %% ../../nbs/analyze/sensitivity.ipynb #analyzer -class SensitivityAnalyzer: - """Analyze per-layer sensitivity to compression methods. - - Uses fasterai's Sparsifier for sparsity analysis and Pruner for structural pruning. - Supports sparsity (weight zeroing), pruning (structural), and quantization. - """ - - VALID_COMPRESSIONS = frozenset({"sparsity", "pruning", "quantization"}) - COMPRESSIBLE_LAYERS = Granularities.available_modules() # Use fasterai's layer registry - - def __init__( - self, - model: nn.Module, # model to analyze - sample: torch.Tensor, # example input (for Pruner dependency analysis) - eval_fn: Callable[[nn.Module], float], # evaluation function returning metric - *, - criteria: Criteria = large_final, # fasterai criteria for importance scoring - higher_is_better: bool = True, # whether higher metric values are better - metric_name: str = "accuracy", # name of the metric for display - device: str | torch.device | None = None, # device for computation - calibration_data: torch.Tensor | None = None, # for observer-based quantization - ): - store_attr() - self.device = device or next(model.parameters()).device - self._results: SensitivityResult | None = None - self._sparsifier: Sparsifier | None = None - self._activation_hooks: list[Any] = [] - self._activation_quantize_config: dict[str, bool] = {} - - @staticmethod - def _out_dim( - module: nn.Module, # layer to inspect - ) -> int | None: - """Output channels (Conv) or features (Linear) of a layer.""" - return getattr(module, 'out_channels', None) or getattr(module, 'out_features', None) - - @staticmethod - @contextmanager - def _quiet(): - """Silence the per-construction notices fasterai's Pruner prints (ignore / per-layer info).""" - import io, contextlib - with contextlib.redirect_stdout(io.StringIO()): - yield - - def _get_compressible_layers( - self, - layer_types: type | tuple[type, ...] | None = None, # a module type or tuple of types to restrict to (None = all compressible) - ) -> list[tuple[str, nn.Module]]: - """Get all compressible layers (Conv2d, Linear, etc.), optionally filtered to `layer_types`.""" - if layer_types is None: - types = self.COMPRESSIBLE_LAYERS - else: - types = (layer_types,) if isinstance(layer_types, type) else tuple(layer_types) - return [ - (name, module) - for name, module in self.model.named_modules() - if isinstance(module, self.COMPRESSIBLE_LAYERS) and isinstance(module, types) - and hasattr(module, 'weight') and module.weight is not None - ] - - def _init_sparsifier( - self, - granularity: str = "weight", # sparsity granularity - ) -> None: - """Initialize fasterai Sparsifier (saves initial weights for all layers).""" - if self._sparsifier is None: - self._sparsifier = Sparsifier( - self.model, - granularity=granularity, - context='local', - criteria=self.criteria, - ) - - def _cleanup_sparsifier(self) -> None: - """Remove sparsifier buffers from model.""" - if self._sparsifier is not None: - self._sparsifier._clean_buffers() - self._sparsifier = None - - def _apply_sparsity( - self, - module: nn.Module, # layer to sparsify - level: float, # sparsity percentage (0-100) - ) -> None: - """Apply sparsity using fasterai Sparsifier.""" - self._sparsifier.sparsify_layer(module, level) - - def _restore_layer( - self, - module: nn.Module, # layer to restore - ) -> None: - """Restore a single layer from saved initial weights.""" - if hasattr(module, '_init_weights'): - module.weight.data.copy_(module._init_weights) - if hasattr(module, '_init_biases') and module._init_biases is not None: - module.bias.data.copy_(module._init_biases) - if hasattr(module, '_mask'): - del module._buffers['_mask'] - - def _clone_model(self) -> nn.Module: - """Create a clean copy of the model (avoids deepcopy issues with non-leaf tensors).""" - import io - buffer = io.BytesIO() - torch.save(self.model, buffer) - buffer.seek(0) - model_copy = torch.load(buffer, weights_only=False) - model_copy.eval() - return model_copy - - def _fresh_copy(self) -> nn.Module: - """A clean model copy (save/load), falling back to deepcopy if that fails.""" - try: - return self._clone_model() - except Exception: - return deepcopy(self.model) - - def _channel_signature( - self, - model: nn.Module, # model to fingerprint - ) -> tuple: - """Tuple of out-channels/out-features for every compressible layer (detects no-op prunes).""" - return tuple( - self._out_dim(m) - for _, m in model.named_modules() - if isinstance(m, self.COMPRESSIBLE_LAYERS) - ) - - def _build_dependency_groups( - self, - model: nn.Module, # model to analyze (groups keyed by layer NAME, stable across clones) - ) -> tuple[dict[str, int], dict[int, list[str]]]: - """Map each compressible layer to its OUTPUT-coupled dependency group. - - Residual/skip connections force several layers to share an output-channel count, so - pruning any one of them prunes all of them — and they yield the SAME accuracy delta. - We detect these coupled sets (torch-pruning handler `prune_out_channels`) so the result - can tag coupled layers with a shared `group_id` and so `to_layer_targets()` treats each - coupled set as a single knob. Layers fasterai's Pruner ignores (output Linear, attention - qkv) get no group — they are not independently prunable. - - Returns (name->group_id, group_id->member_names). - """ - import torch_pruning as tp - import warnings - name_of = {m: n for n, m in model.named_modules()} - with self._quiet(): - p = Pruner(model, pruning_ratio=0.5, context='local', - criteria=self.criteria, example_inputs=self.sample) - DG = p.pruner.DG - ignored = set(p.ignored_layers) - - name_to_coupled: dict[str, frozenset] = {} - for name, mod in model.named_modules(): - if not isinstance(mod, self.COMPRESSIBLE_LAYERS) or mod in ignored: - continue - fn = tp.prune_conv_out_channels if isinstance(mod, nn.Conv2d) else tp.prune_linear_out_channels - coupled = {name} - try: - group = DG.get_pruning_group(mod, fn, idxs=list(range(self._out_dim(mod)))) - for dep, _ in group: - tmod = dep.target.module - if (isinstance(tmod, self.COMPRESSIBLE_LAYERS) and tmod in name_of - and tmod not in ignored - and getattr(dep.handler, '__name__', '') == 'prune_out_channels'): - coupled.add(name_of[tmod]) - except Exception as e: - # Treat as uncoupled (its own singleton group); warn for parity with the prune path - warnings.warn(f"Could not resolve dependency group for {name}: {e}") - name_to_coupled[name] = frozenset(coupled) - - # Assign a group id per unique coupled-set - key_to_gid: dict[frozenset, int] = {} - name_to_gid: dict[str, int] = {} - gid_to_members: dict[int, list[str]] = {} - for name, key in name_to_coupled.items(): - if key not in key_to_gid: - key_to_gid[key] = len(key_to_gid) - gid_to_members[key_to_gid[key]] = sorted(key) - name_to_gid[name] = key_to_gid[key] - return name_to_gid, gid_to_members - - def _apply_structural_pruning( - self, - target_name: str, # layer to prune — applied EXACTLY as a real per-layer dict prune - level: float, # pruning ratio (0-100) - ) -> tuple[nn.Module, bool]: - """Prune the target layer on a fresh model copy exactly as a real per-layer prune would. - - Uses fasterai's per-layer dict target `{target_name: level}`, so the measured - degradation matches what the user gets from `Pruner(model, {target_name: level})` - or `PruneCallback(pruning_ratio={target_name: level})` — including the residual cascade - for coupled layers. This is what makes the reported Δ a faithful predictor of a real - prune. Returns (pruned_model, prunable); prunable is False if the prune changed nothing. - """ - import warnings - model_copy = self._fresh_copy() - if target_name not in dict(model_copy.named_modules()): - return model_copy, False - - before = self._channel_signature(model_copy) - try: - with self._quiet(): - pruner = Pruner( - model_copy, - pruning_ratio={target_name: level}, - context='local', - criteria=self.criteria, - example_inputs=self.sample, - ) - pruner.prune_model() - except Exception as e: - warnings.warn(f"Structural pruning failed for {target_name}: {e}") - # Never evaluate a half-pruned model — return a clean copy marked not-prunable - return self._fresh_copy(), False - - prunable = self._channel_signature(model_copy) != before - return model_copy, prunable - - # ─── Quantization helpers ──────────────────────────────────────────────────── - - def _compute_qparams_symmetric( - self, - tensor: torch.Tensor, # tensor to compute qparams for - bits: int = 8, # quantization bits - per_channel: bool = False, # per-channel or per-tensor - channel_axis: int = 0, # axis for per-channel quantization - ) -> tuple[torch.Tensor, torch.Tensor]: - """Compute scale and zero_point for symmetric quantization.""" - qmin, qmax = -(2 ** (bits - 1)), 2 ** (bits - 1) - 1 - - if per_channel and tensor.dim() > 1: - dims = list(range(tensor.dim())) - dims.remove(channel_axis) - amax = tensor.abs() - for dim in sorted(dims, reverse=True): - amax = amax.max(dim=dim).values - scale = amax / qmax - scale = torch.clamp(scale, min=1e-8) - zero_point = torch.zeros_like(scale, dtype=torch.int32) - else: - amax = tensor.abs().max() - scale = torch.tensor([max(amax.item() / qmax, 1e-8)], device=tensor.device) - zero_point = torch.tensor([0], dtype=torch.int32, device=tensor.device) - - return scale, zero_point - - def _compute_qparams_asymmetric( - self, - tensor: torch.Tensor, # tensor to compute qparams for - bits: int = 8, # quantization bits - per_channel: bool = False, # per-channel or per-tensor - channel_axis: int = 0, # axis for per-channel quantization - ) -> tuple[torch.Tensor, torch.Tensor]: - """Compute scale and zero_point for asymmetric quantization.""" - qmin, qmax = 0, 2 ** bits - 1 - - if per_channel and tensor.dim() > 1: - dims = list(range(tensor.dim())) - dims.remove(channel_axis) - t_min, t_max = tensor.clone(), tensor.clone() - for dim in sorted(dims, reverse=True): - t_min = t_min.min(dim=dim).values - t_max = t_max.max(dim=dim).values - else: - t_min, t_max = tensor.min(), tensor.max() - - scale = (t_max - t_min) / (qmax - qmin) - scale = torch.clamp(scale, min=1e-8) - zero_point = torch.clamp(torch.round(-t_min / scale), qmin, qmax).to(torch.int32) - - if not per_channel or tensor.dim() <= 1: - scale = scale.view(1) if scale.dim() == 0 else scale - zero_point = zero_point.view(1) if zero_point.dim() == 0 else zero_point - - return scale, zero_point - - def _fake_quantize_per_channel( - self, - tensor: torch.Tensor, # tensor to quantize - bits: int = 8, # quantization bits - symmetric: bool = True, # symmetric or asymmetric - channel_axis: int = 0, # axis for per-channel quantization - ) -> torch.Tensor: - """Apply fake quantization (per-channel).""" - qmin = -(2 ** (bits - 1)) if symmetric else 0 - qmax = (2 ** (bits - 1)) - 1 if symmetric else (2 ** bits) - 1 - - if symmetric: - scale, zero_point = self._compute_qparams_symmetric( - tensor, bits, per_channel=True, channel_axis=channel_axis - ) - else: - scale, zero_point = self._compute_qparams_asymmetric( - tensor, bits, per_channel=True, channel_axis=channel_axis - ) - - return torch.fake_quantize_per_channel_affine( - tensor, scale, zero_point, channel_axis, qmin, qmax - ) - - def _fake_quantize_per_tensor( - self, - tensor: torch.Tensor, # tensor to quantize - bits: int = 8, # quantization bits - symmetric: bool = True, # symmetric or asymmetric - ) -> torch.Tensor: - """Apply fake quantization (per-tensor).""" - qmin = -(2 ** (bits - 1)) if symmetric else 0 - qmax = (2 ** (bits - 1)) - 1 if symmetric else (2 ** bits) - 1 - - if symmetric: - scale, zero_point = self._compute_qparams_symmetric(tensor, bits, per_channel=False) - else: - scale, zero_point = self._compute_qparams_asymmetric(tensor, bits, per_channel=False) - - return torch.fake_quantize_per_tensor_affine( - tensor, scale.item(), int(zero_point.item()), qmin, qmax - ) - - def _apply_weight_quantization( - self, - module: nn.Module, # layer to quantize - bits: int = 8, # quantization bits - per_channel: bool = True, # per-channel or per-tensor - ) -> None: - """Apply weight quantization using fake_quantize.""" - weight = module.weight.data - if per_channel and weight.dim() > 1: - quantized = self._fake_quantize_per_channel(weight, bits, symmetric=True, channel_axis=0) - else: - quantized = self._fake_quantize_per_tensor(weight, bits, symmetric=True) - weight.copy_(quantized) - - def _create_activation_quantize_hook( - self, - layer_name: str, # layer name for config lookup - bits: int = 8, # quantization bits - ): - """Create a forward hook that quantizes activations.""" - def hook(module, input, output): - if self._activation_quantize_config.get(layer_name, False): - return self._fake_quantize_per_tensor(output, bits, symmetric=False) - return output - return hook - - def _setup_activation_hooks( - self, - bits: int = 8, # quantization bits - ) -> None: - """Register activation quantization hooks on all layers.""" - self._remove_activation_hooks() - for name, module in self._get_compressible_layers(): - hook = self._create_activation_quantize_hook(name, bits) - handle = module.register_forward_hook(hook) - self._activation_hooks.append(handle) - self._activation_quantize_config[name] = False - - def _remove_activation_hooks(self) -> None: - """Remove all activation quantization hooks.""" - for handle in self._activation_hooks: - handle.remove() - self._activation_hooks = [] - self._activation_quantize_config = {} - - # ─── Main analysis method ──────────────────────────────────────────────────── - - def analyze( - self, - compression: Literal["sparsity", "pruning", "quantization"] = "sparsity", # compression type - level: float = 50, # compression level (% for sparsity/pruning, bits for quant) - *, - granularity: str = "weight", # granularity for sparsity (fasterai granularities) - layers: list[str] | None = None, # specific layer names to analyze (None = all) - layer_types: type | tuple[type, ...] | None = None, # restrict to a module type or tuple of types, e.g. nn.Conv2d or (nn.Conv2d, nn.Linear) (None = all compressible) - quant_per_channel: bool = True, # use per-channel quantization - quant_activations: bool = False, # also quantize activations - verbose: bool = True, # print progress - ) -> SensitivityResult: - """Analyze per-layer sensitivity to compression. - - For **pruning**, each layer is pruned with the per-layer target `{name: level}` — the - exact operation `Pruner`/`PruneCallback` perform — so the reported Δ faithfully predicts - the degradation of really pruning that layer at `level`. Residual/skip-coupled layers - prune together and therefore share a Δ (tagged with a common `group_id`); layers that - cannot be pruned independently (output Linear, attention) are marked `prunable=False`. - - Pass `layer_types` to restrict the analysis to specific module types — a single type or - a tuple, e.g. `layer_types=nn.Conv2d` analyses only convolutions and skips the - classifier `Linear`. - """ - if compression not in self.VALID_COMPRESSIONS: - raise ValueError(f"compression must be one of {self.VALID_COMPRESSIONS}") - - self.model.eval() - - if compression == "sparsity": - self._init_sparsifier(granularity) - - if compression == "quantization" and quant_activations: - bits = int(level) if level > 1 else 8 - self._setup_activation_hooks(bits) - - if verbose: - print(f"Computing baseline {self.metric_name}...", end=" ", flush=True) - baseline = self.eval_fn(self.model) - if verbose: - print(f"{baseline:.4f}") - - all_layers = self._get_compressible_layers(layer_types) - if layers is not None: - all_layers = [(n, m) for n, m in all_layers if n in layers] - - # Pruning: precompute dependency groups once so coupled layers can share a group_id - # and each coupled set is only pruned/evaluated a single time (deduped by group id). - group_gid: dict[str, int] = {} - group_members: dict[int, list[str]] = {} - group_cache: dict[int, tuple[float, bool]] = {} - if compression == "pruning": - group_gid, group_members = self._build_dependency_groups(self.model) - - mode_info = "" - if compression == "quantization": - mode_info = f" (per-{'channel' if quant_per_channel else 'tensor'}" - mode_info += f", {'weights+activations' if quant_activations else 'weights only'})" - elif compression == "sparsity": - mode_info = f" (granularity={granularity}, criteria={self.criteria.f.__name__})" - elif compression == "pruning": - mode_info = f" (structural group-sensitivity, criteria={self.criteria.f.__name__})" - - if verbose: - unit = 'bits' if compression == 'quantization' else '%' - print(f"Analyzing {len(all_layers)} layers for {compression} @ {level}{unit}{mode_info}") - - results: list[LayerSensitivity] = [] - - for i, (name, module) in enumerate(all_layers): - if verbose: - print(f" [{i+1}/{len(all_layers)}] {name}...", end=" ", flush=True) - - prunable = True - gid = group_gid.get(name) # None for sparsity/quant and for ignored (output/attention) layers - if compression == "sparsity": - self._apply_sparsity(module, level) - compressed_metric = self.eval_fn(self.model) - self._restore_layer(module) - param_count = module.weight.numel() - - elif compression == "pruning": - if gid is None: - # ignored layer (output Linear / attention) — not independently prunable - compressed_metric, prunable = baseline, False - elif gid in group_cache: - compressed_metric, prunable = group_cache[gid] # coupled member: reuse the group's prune - else: - pruned_model, prunable = self._apply_structural_pruning(name, level) - compressed_metric = self.eval_fn(pruned_model) if prunable else baseline - del pruned_model - group_cache[gid] = (compressed_metric, prunable) - param_count = module.weight.numel() - - elif compression == "quantization": - saved_weight = module.weight.data.clone() - saved_bias = module.bias.data.clone() if module.bias is not None else None - - bits = int(level) if level > 1 else 8 - self._apply_weight_quantization(module, bits, per_channel=quant_per_channel) - - if quant_activations: - self._activation_quantize_config[name] = True - - compressed_metric = self.eval_fn(self.model) - - if quant_activations: - self._activation_quantize_config[name] = False - - module.weight.data.copy_(saved_weight) - if saved_bias is not None: - module.bias.data.copy_(saved_bias) - param_count = module.weight.numel() - - if self.higher_is_better: - delta = baseline - compressed_metric - else: - delta = compressed_metric - baseline - - if verbose: - sign = "+" if delta > 0 else "" - tag = "" if prunable else " (not prunable)" - print(f"Δ={sign}{delta:.4f}{tag}") - - results.append(LayerSensitivity( - name=name, - layer_type=module.__class__.__name__, - params=param_count, - baseline_metric=baseline, - compressed_metric=compressed_metric, - delta=delta, - group_id=gid, - group_members=group_members.get(gid, []), - prunable=prunable, - )) - - if compression == "sparsity": - self._cleanup_sparsifier() - if compression == "quantization" and quant_activations: - self._remove_activation_hooks() - - compression_desc = compression - if compression == "quantization": - compression_desc = f"quantization-{int(level) if level > 1 else 8}bit" - if quant_activations: - compression_desc += "+act" - - self._results = SensitivityResult( - compression_type=compression_desc, - compression_level=level, - baseline_metric=baseline, - layers=results, - metric_name=self.metric_name, - higher_is_better=self.higher_is_better, - ) - - if verbose: - print(f"✓ Analysis complete") - - return self._results - - def sweep( - self, - compression: Literal["sparsity", "pruning", "quantization"] = "sparsity", # compression type - levels: list[float] | None = None, # compression levels to test (default: [25, 50, 75]) - **kwargs, - ) -> list[SensitivityResult]: - """Run sensitivity analysis at multiple compression levels.""" - if levels is None: - levels = [25, 50, 75] - results = [] - for level in levels: - print(f"\n{'='*60}") - unit = 'bits' if compression == 'quantization' else '%' - print(f"Sweep: {compression} @ {level}{unit}") - print(f"{'='*60}") - result = self.analyze(compression, level, **kwargs) - results.append(result) - return results - -# %% ../../nbs/analyze/sensitivity.ipynb #convenience -def analyze_sensitivity( - model: nn.Module, # model to analyze - sample: torch.Tensor, # example input tensor - eval_fn: Callable[[nn.Module], float], # evaluation function returning metric - compression: Literal["sparsity", "pruning", "quantization"] = "sparsity", # compression type - level: float = 50, # compression level (% for sparsity/pruning, bits for quant) - *, - criteria: Criteria = large_final, # fasterai criteria for importance scoring - higher_is_better: bool = True, # whether higher metric values are better - metric_name: str = "accuracy", # name of the metric for display - granularity: str = "weight", # granularity for sparsity - verbose: bool = True, # print progress - **kwargs, -) -> SensitivityResult: - """One-line sensitivity analysis using fasterai compression methods.""" - analyzer = SensitivityAnalyzer( - model, sample, eval_fn, - criteria=criteria, - higher_is_better=higher_is_better, - metric_name=metric_name, - ) - return analyzer.analyze(compression, level, granularity=granularity, verbose=verbose, **kwargs) +__all__ = [] + +# %% ../../nbs/analyze/sensitivity.ipynb #sens-stub-3 +# Sensitivity analysis moved out of fasterai (now in the higher-level FasterAI workflow package; fasterai holds compression mechanisms only). +_MOVED = {'analyze_sensitivity', 'SensitivityAnalyzer', 'SensitivityResult', 'LayerSensitivity'} +def __getattr__(name): # PEP 562 + if name in _MOVED: + raise ImportError( + f"`{name}` is no longer part of fasterai; sensitivity analysis moved to the " + f"higher-level FasterAI workflow package. See the FasterAI docs for the new import path." + ) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") diff --git a/nbs/_quarto.yml b/nbs/_quarto.yml index 972f71b..195bfb3 100644 --- a/nbs/_quarto.yml +++ b/nbs/_quarto.yml @@ -80,9 +80,6 @@ website: - section: Export contents: - tutorials/export/onnx_export.ipynb - - section: Analyze - contents: - - tutorials/analyze/sensitivity.ipynb - section: Core contents: - core/granularity.ipynb @@ -107,9 +104,6 @@ website: - section: Regularize contents: - regularize/regularize_callback.ipynb - - section: Analyze - contents: - - analyze/sensitivity.ipynb - section: Misc contents: - misc/bn_folding.ipynb diff --git a/nbs/analyze/sensitivity.ipynb b/nbs/analyze/sensitivity.ipynb index a7582e1..aadfe6a 100644 --- a/nbs/analyze/sensitivity.ipynb +++ b/nbs/analyze/sensitivity.ipynb @@ -2,48 +2,33 @@ "cells": [ { "cell_type": "raw", - "id": "93178b57", + "id": "sens-stub-0", "metadata": {}, "source": [ "---\n", - "description: Per-layer sensitivity analysis for compression methods\n", + "description: Sensitivity analysis has moved out of fasterai\n", "output-file: sensitivity.html\n", - "title: Sensitivity Analysis\n", + "title: Sensitivity Analysis (moved)\n", "skip_showdoc: true\n", "---" ] }, { "cell_type": "markdown", - "id": "header", + "id": "sens-stub-1", "metadata": {}, "source": [ - "## Overview\n", + "## Moved out of fasterai\n", "\n", - "Not all layers in a neural network are equally important. Some are fragile — compressing them even slightly degrades performance — while others are robust and can be heavily compressed with minimal impact. **Sensitivity analysis** measures this per-layer fragility, enabling smarter compression strategies.\n", + "Sensitivity analysis (`analyze_sensitivity`, `SensitivityAnalyzer`, `SensitivityResult`, `LayerSensitivity`) has moved out of fasterai into the higher-level FasterAI workflow package. fasterai now holds the compression *mechanisms* only.\n", "\n", - "The `SensitivityAnalyzer` works by compressing one layer at a time, measuring the impact on a user-provided evaluation metric, and ranking layers by their sensitivity (delta from baseline).\n", - "\n", - "**Key Features:**\n", - "\n", - "- Supports three compression types: **sparsity** (weight zeroing), **pruning** (structural filter removal), and **quantization** (precision reduction)\n", - "- Generates **non-uniform per-layer targets** via `to_layer_targets()` — fragile layers get less compression, robust layers get more\n", - "- Uses fasterai's `Sparsifier` and `Pruner` internally for consistent results\n", - "- Visualization with `plot()` and export to pandas with `to_dataframe()`\n", - "\n", - "### When to Use\n", - "\n", - "| Compression Type | `level` parameter | What it tests | Best for |\n", - "|------------------|-------------------|---------------|----------|\n", - "| **`\"sparsity\"`** | % of weights zeroed (e.g., 50) | Unstructured weight removal | Generating per-layer sparsity targets |\n", - "| **`\"pruning\"`** | % of filters removed (e.g., 30) | Structural filter pruning | Identifying layers to protect via `ignored_layers` |\n", - "| **`\"quantization\"`** | Bit width (e.g., 8) | Precision reduction | Finding layers that need higher precision |" + "Importing any of these names from `fasterai.analyze.sensitivity` raises an `ImportError` — see the FasterAI documentation for the new location." ] }, { "cell_type": "code", "execution_count": null, - "id": "default-exp", + "id": "sens-stub-2", "metadata": {}, "outputs": [], "source": [ @@ -53,1824 +38,43 @@ { "cell_type": "code", "execution_count": null, - "id": "imports", - "metadata": {}, - "outputs": [], - "source": [ - "#| export\n", - "from __future__ import annotations\n", - "import torch\n", - "import torch.nn as nn\n", - "import numpy as np\n", - "from copy import deepcopy\n", - "from dataclasses import dataclass, field, asdict\n", - "from typing import Callable, Any, Literal\n", - "from collections import OrderedDict\n", - "from contextlib import contextmanager\n", - "from fastcore.basics import store_attr\n", - "\n", - "# fasterai imports (relative within fasterai package)\n", - "from fasterai.sparse.all import Sparsifier\n", - "from fasterai.prune.all import Pruner\n", - "from fasterai.core.all import large_final, Criteria, Granularities" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "hide-imports", - "metadata": {}, - "outputs": [], - "source": [ - "#| include: false\n", - "from nbdev.showdoc import *" - ] - }, - { - "cell_type": "markdown", - "id": "dataclasses-header", - "metadata": {}, - "source": [ - "## Data Classes\n", - "\n", - "Sensitivity results are returned as structured dataclasses for easy inspection, sorting, and export." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "dataclasses", - "metadata": {}, - "outputs": [], - "source": [ - "#| export\n", - "@dataclass(slots=True)\n", - "class LayerSensitivity:\n", - " \"\"\"Sensitivity result for a single layer.\"\"\"\n", - " name: str # layer name\n", - " layer_type: str # e.g., \"Conv2d\", \"Linear\"\n", - " params: int # number of parameters\n", - " baseline_metric: float # metric before compression\n", - " compressed_metric: float # metric after compression\n", - " delta: float # metric change (positive = degradation)\n", - " group_id: int | None = None # pruning: dependency-group id (coupled layers share one); None for sparsity/quant\n", - " group_members: list[str] = field(default_factory=list) # names of all layers co-pruned with this one\n", - " prunable: bool = True # False if the layer could not be pruned (genuine no-op) — not \"robust\"\n", - "\n", - " def as_dict(self) -> dict[str, Any]:\n", - " \"\"\"Convert to dictionary.\"\"\"\n", - " return asdict(self)\n", - "\n", - "\n", - "@dataclass(slots=True)\n", - "class SensitivityResult:\n", - " \"\"\"Structured result from sensitivity analysis.\"\"\"\n", - " compression_type: str # \"sparsity\", \"pruning\", \"quantization\"\n", - " compression_level: float # e.g., 50 for 50% sparsity\n", - " baseline_metric: float # overall baseline metric\n", - " layers: list[LayerSensitivity] # per-layer results\n", - " metric_name: str = \"accuracy\" # name of the metric\n", - " higher_is_better: bool = True # whether higher metric is better\n", - " _results: list[LayerSensitivity] = field(default=None, init=False, repr=False) # for top() compatibility\n", - " \n", - " def __post_init__(self):\n", - " self._results = self.layers # for compatibility with top() pattern\n", - " \n", - " def as_dict(self) -> dict[str, Any]:\n", - " \"\"\"Convert to flat dictionary.\"\"\"\n", - " return {\n", - " \"compression_type\": self.compression_type,\n", - " \"compression_level\": self.compression_level,\n", - " \"baseline_metric\": self.baseline_metric,\n", - " \"metric_name\": self.metric_name,\n", - " \"higher_is_better\": self.higher_is_better,\n", - " \"layers\": [l.as_dict() for l in self.layers],\n", - " }\n", - " \n", - " def top(\n", - " self,\n", - " n: int = 5, # number of layers to return\n", - " *,\n", - " most_sensitive: bool = True, # True=highest delta (fragile), False=lowest (robust)\n", - " ) -> list[LayerSensitivity]:\n", - " \"\"\"Return top N most or least sensitive layers.\n", - "\n", - " Layers that could not be pruned (`prunable=False`) are excluded — a no-op\n", - " prune is NOT evidence of robustness, so it must never rank as compressible.\n", - " \"\"\"\n", - " candidates = [l for l in self.layers if l.prunable]\n", - " sorted_layers = sorted(candidates, key=lambda x: x.delta, reverse=most_sensitive)\n", - " return sorted_layers[:n]\n", - " \n", - " def summary(\n", - " self,\n", - " *,\n", - " top: int = 5, # number of layers to show per category\n", - " ) -> None:\n", - " \"\"\"Print a formatted summary of sensitivity analysis.\"\"\"\n", - " print(f\"{'═' * 60}\")\n", - " print(f\"Sensitivity Analysis: {self.compression_type} @ {self.compression_level}%\")\n", - " print(f\"{'═' * 60}\")\n", - " print(f\" Baseline {self.metric_name}: {self.baseline_metric:.4f}\")\n", - " print(f\" Layers analyzed: {len(self.layers)}\")\n", - " print()\n", - "\n", - " def _line(i, layer):\n", - " sign = \"+\" if layer.delta > 0 else \"\"\n", - " grp = f\" [group {layer.group_id}]\" if layer.group_id is not None else \"\"\n", - " print(f\" {i}. {layer.name:30} Δ={sign}{layer.delta:.4f}{grp}\")\n", - "\n", - " # Most sensitive (fragile) layers\n", - " print(f\" 🔴 Most Sensitive (fragile):\")\n", - " for i, layer in enumerate(self.top(top, most_sensitive=True), 1):\n", - " _line(i, layer)\n", - " print()\n", - " \n", - " # Most robust layers\n", - " print(f\" 🟢 Most Robust (compressible):\")\n", - " for i, layer in enumerate(self.top(top, most_sensitive=False), 1):\n", - " _line(i, layer)\n", - "\n", - " # Layers that could not be pruned in isolation — surfaced, NOT ranked as robust\n", - " not_prunable = [l for l in self.layers if not l.prunable]\n", - " if not_prunable:\n", - " print()\n", - " print(f\" ⚪ Not prunable in isolation ({len(not_prunable)}):\")\n", - " for i, layer in enumerate(not_prunable[:top], 1):\n", - " print(f\" {i}. {layer.name:30} (group {layer.group_id})\")\n", - " \n", - " def to_dataframe(self):\n", - " \"\"\"Convert to pandas DataFrame.\"\"\"\n", - " import pandas as pd\n", - " rows = [layer.as_dict() for layer in self.layers]\n", - " return pd.DataFrame(rows)\n", - " \n", - " def to_layer_targets(\n", - " self,\n", - " model: nn.Module, # model (used for parameter counts)\n", - " target_pct: float = 50, # target mean compression percentage\n", - " min_pct: float = 0, # minimum compression for any layer\n", - " max_pct: float = 90, # maximum compression for any layer\n", - " gamma: float = 1.0, # exponent for sensitivity scaling (higher = more differentiation)\n", - " ) -> dict[str, float]:\n", - " \"\"\"Convert sensitivity to non-uniform per-layer compression targets.\n", - " \n", - " High sensitivity layers get lower compression, robust layers get higher.\n", - " Uses parameter-weighted optimization to hit target_pct exactly.\n", - "\n", - " Coupled layers (same `group_id`) are optimized as a SINGLE knob — they\n", - " physically share one pruning ratio — then the group's target is expanded\n", - " back to every member, so the returned dict stays per-layer. Layers that\n", - " are not prunable receive `min_pct`.\n", - " \"\"\"\n", - " if not self.layers:\n", - " return {}\n", - " \n", - " # Convert to fractions\n", - " target = target_pct / 100.0\n", - " smin = min_pct / 100.0\n", - " smax = max_pct / 100.0\n", - "\n", - " # Collapse to one entry per dependency group (a singleton group per layer when\n", - " # group_id is None, e.g. sparsity/quant). This stops a coupled group of N layers\n", - " # from being double-counted as N independent knobs in the weighted-mean solve.\n", - " prunable = [l for l in self.layers if l.prunable]\n", - " not_prunable = [l for l in self.layers if not l.prunable]\n", - " groups: OrderedDict[Any, dict] = OrderedDict()\n", - " for l in prunable:\n", - " key = l.group_id if l.group_id is not None else (\"solo\", l.name)\n", - " g = groups.setdefault(key, {\"names\": [], \"params\": 0.0, \"delta\": 0.0})\n", - " g[\"names\"].append(l.name)\n", - " g[\"params\"] += float(l.params)\n", - " g[\"delta\"] = max(g[\"delta\"], max(0.0, l.delta)) # delta is shared within a group\n", - "\n", - " # Not-prunable layers are protected at min_pct and kept out of the optimization\n", - " targets: dict[str, float] = {l.name: round(float(min_pct), 2) for l in not_prunable}\n", - " if not groups:\n", - " return targets\n", - "\n", - " deltas = np.array([g[\"delta\"] for g in groups.values()], dtype=float)\n", - " weights = np.array([g[\"params\"] for g in groups.values()], dtype=float)\n", - " group_names = [g[\"names\"] for g in groups.values()]\n", - "\n", - " if weights.sum() == 0:\n", - " for names in group_names:\n", - " for n in names: targets[n] = target_pct\n", - " return targets\n", - " \n", - " # Normalize sensitivity and invert (high sensitivity -> low compression)\n", - " if np.allclose(deltas, deltas[0]):\n", - " s0 = np.full_like(deltas, target)\n", - " else:\n", - " norm = (deltas - deltas.min()) / (np.ptp(deltas) + 1e-12)\n", - " inv = (1.0 - norm) ** gamma\n", - " s0 = smin + (smax - smin) * inv\n", - " \n", - " # Binary search for lambda to hit target weighted mean\n", - " W = weights.sum()\n", - " tgt = target * W\n", - " \n", - " def f(lam):\n", - " s = np.clip(s0 + lam, smin, smax)\n", - " return float(np.dot(weights, s))\n", - " \n", - " # Find lambda via bisection\n", - " lam_lo, lam_hi = -1.0, 1.0\n", - " while f(lam_lo) > tgt:\n", - " lam_lo *= 2\n", - " while f(lam_hi) < tgt:\n", - " lam_hi *= 2\n", - " \n", - " for _ in range(60):\n", - " lam_mid = 0.5 * (lam_lo + lam_hi)\n", - " if f(lam_mid) < tgt:\n", - " lam_lo = lam_mid\n", - " else:\n", - " lam_hi = lam_mid\n", - " \n", - " final_s = np.clip(s0 + 0.5 * (lam_lo + lam_hi), smin, smax)\n", - "\n", - " # Expand each group's target back to all its member layers (they share the ratio)\n", - " for names, s in zip(group_names, final_s):\n", - " for n in names:\n", - " targets[n] = round(s * 100, 2)\n", - " return targets\n", - " \n", - " def plot(\n", - " self,\n", - " figsize: tuple = (12, 5), # figure size (width, height)\n", - " ) -> None:\n", - " \"\"\"Plot sensitivity as a bar chart.\"\"\"\n", - " import matplotlib.pyplot as plt\n", - " \n", - " names = [l.name for l in self.layers]\n", - " deltas = np.array([l.delta for l in self.layers], dtype=float)\n", - " \n", - " # Color by sensitivity\n", - " norm = (deltas - deltas.min()) / (np.ptp(deltas) + 1e-9)\n", - " colors = plt.cm.RdYlGn_r(norm) # Red=sensitive, Green=robust\n", - " \n", - " plt.figure(figsize=figsize)\n", - " plt.bar(range(len(deltas)), deltas, color=colors)\n", - " plt.axhline(0, color='gray', linewidth=1.2, linestyle='--')\n", - " plt.xticks(range(len(names)), names, rotation=60, ha='right')\n", - " plt.ylabel(f\"{self.metric_name} drop (Δ)\")\n", - " plt.title(f\"Layer Sensitivity to {self.compression_type} @ {self.compression_level}%\", \n", - " pad=12, weight='bold')\n", - " plt.grid(axis='y', linestyle=':', alpha=0.6)\n", - " plt.tight_layout()\n", - " plt.show()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "me8c3ut7k7a", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "Skipping import of cpp extensions due to incompatible torch version 2.9.1+cu128 for torchao version 0.16.0 Please see https://github.com/pytorch/ao/issues/2919 for more info\n" - ] - }, - { - "data": { - "text/markdown": [ - "---\n", - "\n", - "[source](https://github.com/FasterAI-Labs/fasterai/blob/master/fasterai/analyze/sensitivity.py#L25){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### LayerSensitivity\n", - "\n", - "```python\n", - "\n", - "def LayerSensitivity(\n", - " name:str, layer_type:str, params:int, baseline_metric:float, compressed_metric:float, delta:float,\n", - " group_id:int | None=None, group_members:list[str]=, prunable:bool=True\n", - ")->None:\n", - "\n", - "\n", - "```\n", - "\n", - "*Sensitivity result for a single layer.*" - ], - "text/plain": [ - "def LayerSensitivity(\n", - " name:str, layer_type:str, params:int, baseline_metric:float, compressed_metric:float, delta:float,\n", - " group_id:int | None=None, group_members:list[str]=, prunable:bool=True\n", - ")->None:\n", - "\"\"\"Sensitivity result for a single layer.\"\"\"" - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "show_doc(LayerSensitivity)" - ] - }, - { - "cell_type": "markdown", - "id": "3avy0eebxon", - "metadata": {}, - "source": [ - "`LayerSensitivity` holds the result for a single layer: the metric before and after compression, the delta (positive = degradation), and layer metadata. Use `as_dict()` to serialize.\n", - "\n", - "---" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "q7ms987f6b", - "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "---\n", - "\n", - "[source](https://github.com/FasterAI-Labs/fasterai/blob/master/fasterai/analyze/sensitivity.py#L43){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### SensitivityResult\n", - "\n", - "```python\n", - "\n", - "def SensitivityResult(\n", - " compression_type:str, compression_level:float, baseline_metric:float, layers:list[LayerSensitivity],\n", - " metric_name:str='accuracy', higher_is_better:bool=True\n", - ")->None:\n", - "\n", - "\n", - "```\n", - "\n", - "*Structured result from sensitivity analysis.*" - ], - "text/plain": [ - "def SensitivityResult(\n", - " compression_type:str, compression_level:float, baseline_metric:float, layers:list[LayerSensitivity],\n", - " metric_name:str='accuracy', higher_is_better:bool=True\n", - ")->None:\n", - "\"\"\"Structured result from sensitivity analysis.\"\"\"" - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "show_doc(SensitivityResult)" - ] - }, - { - "cell_type": "markdown", - "id": "c5skl3as33", - "metadata": {}, - "source": [ - "`SensitivityResult` aggregates all per-layer results. It provides methods to inspect, rank, visualize, and convert sensitivity data into actionable compression targets.\n", - "\n", - "### Key Methods" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "dsde2w6uaiq", - "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "---\n", - "\n", - "[source](https://github.com/FasterAI-Labs/fasterai/blob/master/fasterai/analyze/sensitivity.py#L67){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### SensitivityResult.top\n", - "\n", - "```python\n", - "\n", - "def top(\n", - " n:int=5, # number of layers to return\n", - " most_sensitive:bool=True, # True=highest delta (fragile), False=lowest (robust)\n", - ")->list[LayerSensitivity]:\n", - "\n", - "\n", - "```\n", - "\n", - "*Return top N most or least sensitive layers.*\n", - "\n", - "Layers that could not be pruned (`prunable=False`) are excluded — a no-op\n", - "prune is NOT evidence of robustness, so it must never rank as compressible." - ], - "text/plain": [ - "def top(\n", - " n:int=5, # number of layers to return\n", - " most_sensitive:bool=True, # True=highest delta (fragile), False=lowest (robust)\n", - ")->list[LayerSensitivity]:\n", - "\"\"\"Return top N most or least sensitive layers.\n", - "\n", - "Layers that could not be pruned (`prunable=False`) are excluded — a no-op\n", - "prune is NOT evidence of robustness, so it must never rank as compressible.\"\"\"" - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "show_doc(SensitivityResult.top)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "fkfqbj9et8", - "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "---\n", - "\n", - "[source](https://github.com/FasterAI-Labs/fasterai/blob/master/fasterai/analyze/sensitivity.py#L125){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### SensitivityResult.to_layer_targets\n", - "\n", - "```python\n", - "\n", - "def to_layer_targets(\n", - " model:nn.Module, # model (used for parameter counts)\n", - " target_pct:float=50, # target mean compression percentage\n", - " min_pct:float=0, # minimum compression for any layer\n", - " max_pct:float=90, # maximum compression for any layer\n", - " gamma:float=1.0, # exponent for sensitivity scaling (higher = more differentiation)\n", - ")->dict[str, float]:\n", - "\n", - "\n", - "```\n", - "\n", - "*Convert sensitivity to non-uniform per-layer compression targets.*\n", - "\n", - "High sensitivity layers get lower compression, robust layers get higher.\n", - "Uses parameter-weighted optimization to hit target_pct exactly.\n", - "\n", - "Coupled layers (same `group_id`) are optimized as a SINGLE knob — they\n", - "physically share one pruning ratio — then the group's target is expanded\n", - "back to every member, so the returned dict stays per-layer. Layers that\n", - "are not prunable receive `min_pct`." - ], - "text/plain": [ - "def to_layer_targets(\n", - " model:nn.Module, # model (used for parameter counts)\n", - " target_pct:float=50, # target mean compression percentage\n", - " min_pct:float=0, # minimum compression for any layer\n", - " max_pct:float=90, # maximum compression for any layer\n", - " gamma:float=1.0, # exponent for sensitivity scaling (higher = more differentiation)\n", - ")->dict[str, float]:\n", - "\"\"\"Convert sensitivity to non-uniform per-layer compression targets.\n", - "\n", - "High sensitivity layers get lower compression, robust layers get higher.\n", - "Uses parameter-weighted optimization to hit target_pct exactly.\n", - "\n", - "Coupled layers (same `group_id`) are optimized as a SINGLE knob — they\n", - "physically share one pruning ratio — then the group's target is expanded\n", - "back to every member, so the returned dict stays per-layer. Layers that\n", - "are not prunable receive `min_pct`.\"\"\"" - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "show_doc(SensitivityResult.to_layer_targets)" - ] - }, - { - "cell_type": "markdown", - "id": "mroawy2h1mb", - "metadata": {}, - "source": [ - "The `to_layer_targets()` method converts sensitivity scores into a `dict[str, float]` mapping layer names to compression percentages. This dict can be passed directly to `Sparsifier.sparsify_model()` or `SparsifyCallback(sparsity=...)` for non-uniform compression.\n", - "\n", - "The `gamma` parameter controls differentiation: higher values protect fragile layers more aggressively.\n", - "\n", - "```python\n", - "targets = result.to_layer_targets(model, target_pct=50, min_pct=10, max_pct=80, gamma=1.5)\n", - "# {'conv1': 62.5, 'layer1.0.conv1': 65.3, 'layer1.1.conv2': 10.0, ...}\n", - "```\n", - "\n", - "---" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "h5tjjn2ajnq", - "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "---\n", - "\n", - "[source](https://github.com/FasterAI-Labs/fasterai/blob/master/fasterai/analyze/sensitivity.py#L82){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### SensitivityResult.summary\n", - "\n", - "```python\n", - "\n", - "def summary(\n", - " top:int=5, # number of layers to show per category\n", - ")->None:\n", - "\n", - "\n", - "```\n", - "\n", - "*Print a formatted summary of sensitivity analysis.*" - ], - "text/plain": [ - "def summary(\n", - " top:int=5, # number of layers to show per category\n", - ")->None:\n", - "\"\"\"Print a formatted summary of sensitivity analysis.\"\"\"" - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "show_doc(SensitivityResult.summary)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "sd_split_0", - "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "---\n", - "\n", - "[source](https://github.com/FasterAI-Labs/fasterai/blob/master/fasterai/analyze/sensitivity.py#L216){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### SensitivityResult.plot\n", - "\n", - "```python\n", - "\n", - "def plot(\n", - " figsize:tuple=(12, 5), # figure size (width, height)\n", - ")->None:\n", - "\n", - "\n", - "```\n", - "\n", - "*Plot sensitivity as a bar chart.*" - ], - "text/plain": [ - "def plot(\n", - " figsize:tuple=(12, 5), # figure size (width, height)\n", - ")->None:\n", - "\"\"\"Plot sensitivity as a bar chart.\"\"\"" - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "show_doc(SensitivityResult.plot)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "sd_split_1", - "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "---\n", - "\n", - "[source](https://github.com/FasterAI-Labs/fasterai/blob/master/fasterai/analyze/sensitivity.py#L119){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### SensitivityResult.to_dataframe\n", - "\n", - "```python\n", - "\n", - "def to_dataframe(\n", - " \n", - "):\n", - "\n", - "\n", - "```\n", - "\n", - "*Convert to pandas DataFrame.*" - ], - "text/plain": [ - "def to_dataframe(\n", - " \n", - "):\n", - "\"\"\"Convert to pandas DataFrame.\"\"\"" - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "show_doc(SensitivityResult.to_dataframe)" - ] - }, - { - "cell_type": "markdown", - "id": "analyzer-header", - "metadata": {}, - "source": [ - "---\n", - "\n", - "## SensitivityAnalyzer\n", - "\n", - "The `SensitivityAnalyzer` class provides full control over the analysis process. For quick one-off analysis, see `analyze_sensitivity()` below." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "analyzer", + "id": "sens-stub-3", "metadata": {}, "outputs": [], "source": [ "#| export\n", - "class SensitivityAnalyzer:\n", - " \"\"\"Analyze per-layer sensitivity to compression methods.\n", - " \n", - " Uses fasterai's Sparsifier for sparsity analysis and Pruner for structural pruning.\n", - " Supports sparsity (weight zeroing), pruning (structural), and quantization.\n", - " \"\"\"\n", - " \n", - " VALID_COMPRESSIONS = frozenset({\"sparsity\", \"pruning\", \"quantization\"})\n", - " COMPRESSIBLE_LAYERS = Granularities.available_modules() # Use fasterai's layer registry\n", - " \n", - " def __init__(\n", - " self,\n", - " model: nn.Module, # model to analyze\n", - " sample: torch.Tensor, # example input (for Pruner dependency analysis)\n", - " eval_fn: Callable[[nn.Module], float], # evaluation function returning metric\n", - " *,\n", - " criteria: Criteria = large_final, # fasterai criteria for importance scoring\n", - " higher_is_better: bool = True, # whether higher metric values are better\n", - " metric_name: str = \"accuracy\", # name of the metric for display\n", - " device: str | torch.device | None = None, # device for computation\n", - " calibration_data: torch.Tensor | None = None, # for observer-based quantization\n", - " ):\n", - " store_attr()\n", - " self.device = device or next(model.parameters()).device\n", - " self._results: SensitivityResult | None = None\n", - " self._sparsifier: Sparsifier | None = None\n", - " self._activation_hooks: list[Any] = []\n", - " self._activation_quantize_config: dict[str, bool] = {}\n", - "\n", - " @staticmethod\n", - " def _out_dim(\n", - " module: nn.Module, # layer to inspect\n", - " ) -> int | None:\n", - " \"\"\"Output channels (Conv) or features (Linear) of a layer.\"\"\"\n", - " return getattr(module, 'out_channels', None) or getattr(module, 'out_features', None)\n", - "\n", - " @staticmethod\n", - " @contextmanager\n", - " def _quiet():\n", - " \"\"\"Silence the per-construction notices fasterai's Pruner prints (ignore / per-layer info).\"\"\"\n", - " import io, contextlib\n", - " with contextlib.redirect_stdout(io.StringIO()):\n", - " yield\n", - " \n", - " def _get_compressible_layers(\n", - " self,\n", - " layer_types: type | tuple[type, ...] | None = None, # a module type or tuple of types to restrict to (None = all compressible)\n", - " ) -> list[tuple[str, nn.Module]]:\n", - " \"\"\"Get all compressible layers (Conv2d, Linear, etc.), optionally filtered to `layer_types`.\"\"\"\n", - " if layer_types is None:\n", - " types = self.COMPRESSIBLE_LAYERS\n", - " else:\n", - " types = (layer_types,) if isinstance(layer_types, type) else tuple(layer_types)\n", - " return [\n", - " (name, module) \n", - " for name, module in self.model.named_modules()\n", - " if isinstance(module, self.COMPRESSIBLE_LAYERS) and isinstance(module, types)\n", - " and hasattr(module, 'weight') and module.weight is not None\n", - " ]\n", - " \n", - " def _init_sparsifier(\n", - " self,\n", - " granularity: str = \"weight\", # sparsity granularity\n", - " ) -> None:\n", - " \"\"\"Initialize fasterai Sparsifier (saves initial weights for all layers).\"\"\"\n", - " if self._sparsifier is None:\n", - " self._sparsifier = Sparsifier(\n", - " self.model,\n", - " granularity=granularity,\n", - " context='local',\n", - " criteria=self.criteria,\n", - " )\n", - " \n", - " def _cleanup_sparsifier(self) -> None:\n", - " \"\"\"Remove sparsifier buffers from model.\"\"\"\n", - " if self._sparsifier is not None:\n", - " self._sparsifier._clean_buffers()\n", - " self._sparsifier = None\n", - " \n", - " def _apply_sparsity(\n", - " self,\n", - " module: nn.Module, # layer to sparsify\n", - " level: float, # sparsity percentage (0-100)\n", - " ) -> None:\n", - " \"\"\"Apply sparsity using fasterai Sparsifier.\"\"\"\n", - " self._sparsifier.sparsify_layer(module, level)\n", - " \n", - " def _restore_layer(\n", - " self,\n", - " module: nn.Module, # layer to restore\n", - " ) -> None:\n", - " \"\"\"Restore a single layer from saved initial weights.\"\"\"\n", - " if hasattr(module, '_init_weights'):\n", - " module.weight.data.copy_(module._init_weights)\n", - " if hasattr(module, '_init_biases') and module._init_biases is not None:\n", - " module.bias.data.copy_(module._init_biases)\n", - " if hasattr(module, '_mask'):\n", - " del module._buffers['_mask']\n", - " \n", - " def _clone_model(self) -> nn.Module:\n", - " \"\"\"Create a clean copy of the model (avoids deepcopy issues with non-leaf tensors).\"\"\"\n", - " import io\n", - " buffer = io.BytesIO()\n", - " torch.save(self.model, buffer)\n", - " buffer.seek(0)\n", - " model_copy = torch.load(buffer, weights_only=False)\n", - " model_copy.eval()\n", - " return model_copy\n", - "\n", - " def _fresh_copy(self) -> nn.Module:\n", - " \"\"\"A clean model copy (save/load), falling back to deepcopy if that fails.\"\"\"\n", - " try:\n", - " return self._clone_model()\n", - " except Exception:\n", - " return deepcopy(self.model)\n", - "\n", - " def _channel_signature(\n", - " self,\n", - " model: nn.Module, # model to fingerprint\n", - " ) -> tuple:\n", - " \"\"\"Tuple of out-channels/out-features for every compressible layer (detects no-op prunes).\"\"\"\n", - " return tuple(\n", - " self._out_dim(m)\n", - " for _, m in model.named_modules()\n", - " if isinstance(m, self.COMPRESSIBLE_LAYERS)\n", + "# Sensitivity analysis moved out of fasterai (now in the higher-level FasterAI workflow package; fasterai holds compression mechanisms only).\n", + "_MOVED = {'analyze_sensitivity', 'SensitivityAnalyzer', 'SensitivityResult', 'LayerSensitivity'}\n", + "def __getattr__(name): # PEP 562\n", + " if name in _MOVED:\n", + " raise ImportError(\n", + " f\"`{name}` is no longer part of fasterai; sensitivity analysis moved to the \"\n", + " f\"higher-level FasterAI workflow package. See the FasterAI docs for the new import path.\"\n", " )\n", - "\n", - " def _build_dependency_groups(\n", - " self,\n", - " model: nn.Module, # model to analyze (groups keyed by layer NAME, stable across clones)\n", - " ) -> tuple[dict[str, int], dict[int, list[str]]]:\n", - " \"\"\"Map each compressible layer to its OUTPUT-coupled dependency group.\n", - "\n", - " Residual/skip connections force several layers to share an output-channel count, so\n", - " pruning any one of them prunes all of them — and they yield the SAME accuracy delta.\n", - " We detect these coupled sets (torch-pruning handler `prune_out_channels`) so the result\n", - " can tag coupled layers with a shared `group_id` and so `to_layer_targets()` treats each\n", - " coupled set as a single knob. Layers fasterai's Pruner ignores (output Linear, attention\n", - " qkv) get no group — they are not independently prunable.\n", - "\n", - " Returns (name->group_id, group_id->member_names).\n", - " \"\"\"\n", - " import torch_pruning as tp\n", - " import warnings\n", - " name_of = {m: n for n, m in model.named_modules()}\n", - " with self._quiet():\n", - " p = Pruner(model, pruning_ratio=0.5, context='local',\n", - " criteria=self.criteria, example_inputs=self.sample)\n", - " DG = p.pruner.DG\n", - " ignored = set(p.ignored_layers)\n", - "\n", - " name_to_coupled: dict[str, frozenset] = {}\n", - " for name, mod in model.named_modules():\n", - " if not isinstance(mod, self.COMPRESSIBLE_LAYERS) or mod in ignored:\n", - " continue\n", - " fn = tp.prune_conv_out_channels if isinstance(mod, nn.Conv2d) else tp.prune_linear_out_channels\n", - " coupled = {name}\n", - " try:\n", - " group = DG.get_pruning_group(mod, fn, idxs=list(range(self._out_dim(mod))))\n", - " for dep, _ in group:\n", - " tmod = dep.target.module\n", - " if (isinstance(tmod, self.COMPRESSIBLE_LAYERS) and tmod in name_of\n", - " and tmod not in ignored\n", - " and getattr(dep.handler, '__name__', '') == 'prune_out_channels'):\n", - " coupled.add(name_of[tmod])\n", - " except Exception as e:\n", - " # Treat as uncoupled (its own singleton group); warn for parity with the prune path\n", - " warnings.warn(f\"Could not resolve dependency group for {name}: {e}\")\n", - " name_to_coupled[name] = frozenset(coupled)\n", - "\n", - " # Assign a group id per unique coupled-set\n", - " key_to_gid: dict[frozenset, int] = {}\n", - " name_to_gid: dict[str, int] = {}\n", - " gid_to_members: dict[int, list[str]] = {}\n", - " for name, key in name_to_coupled.items():\n", - " if key not in key_to_gid:\n", - " key_to_gid[key] = len(key_to_gid)\n", - " gid_to_members[key_to_gid[key]] = sorted(key)\n", - " name_to_gid[name] = key_to_gid[key]\n", - " return name_to_gid, gid_to_members\n", - "\n", - " def _apply_structural_pruning(\n", - " self, \n", - " target_name: str, # layer to prune — applied EXACTLY as a real per-layer dict prune\n", - " level: float, # pruning ratio (0-100)\n", - " ) -> tuple[nn.Module, bool]:\n", - " \"\"\"Prune the target layer on a fresh model copy exactly as a real per-layer prune would.\n", - "\n", - " Uses fasterai's per-layer dict target `{target_name: level}`, so the measured\n", - " degradation matches what the user gets from `Pruner(model, {target_name: level})`\n", - " or `PruneCallback(pruning_ratio={target_name: level})` — including the residual cascade\n", - " for coupled layers. This is what makes the reported Δ a faithful predictor of a real\n", - " prune. Returns (pruned_model, prunable); prunable is False if the prune changed nothing.\n", - " \"\"\"\n", - " import warnings\n", - " model_copy = self._fresh_copy()\n", - " if target_name not in dict(model_copy.named_modules()):\n", - " return model_copy, False\n", - "\n", - " before = self._channel_signature(model_copy)\n", - " try:\n", - " with self._quiet():\n", - " pruner = Pruner(\n", - " model_copy,\n", - " pruning_ratio={target_name: level},\n", - " context='local',\n", - " criteria=self.criteria,\n", - " example_inputs=self.sample,\n", - " )\n", - " pruner.prune_model()\n", - " except Exception as e:\n", - " warnings.warn(f\"Structural pruning failed for {target_name}: {e}\")\n", - " # Never evaluate a half-pruned model — return a clean copy marked not-prunable\n", - " return self._fresh_copy(), False\n", - "\n", - " prunable = self._channel_signature(model_copy) != before\n", - " return model_copy, prunable\n", - " \n", - " # ─── Quantization helpers ────────────────────────────────────────────────────\n", - " \n", - " def _compute_qparams_symmetric(\n", - " self,\n", - " tensor: torch.Tensor, # tensor to compute qparams for\n", - " bits: int = 8, # quantization bits\n", - " per_channel: bool = False, # per-channel or per-tensor\n", - " channel_axis: int = 0, # axis for per-channel quantization\n", - " ) -> tuple[torch.Tensor, torch.Tensor]:\n", - " \"\"\"Compute scale and zero_point for symmetric quantization.\"\"\"\n", - " qmin, qmax = -(2 ** (bits - 1)), 2 ** (bits - 1) - 1\n", - " \n", - " if per_channel and tensor.dim() > 1:\n", - " dims = list(range(tensor.dim()))\n", - " dims.remove(channel_axis)\n", - " amax = tensor.abs()\n", - " for dim in sorted(dims, reverse=True):\n", - " amax = amax.max(dim=dim).values\n", - " scale = amax / qmax\n", - " scale = torch.clamp(scale, min=1e-8)\n", - " zero_point = torch.zeros_like(scale, dtype=torch.int32)\n", - " else:\n", - " amax = tensor.abs().max()\n", - " scale = torch.tensor([max(amax.item() / qmax, 1e-8)], device=tensor.device)\n", - " zero_point = torch.tensor([0], dtype=torch.int32, device=tensor.device)\n", - " \n", - " return scale, zero_point\n", - " \n", - " def _compute_qparams_asymmetric(\n", - " self,\n", - " tensor: torch.Tensor, # tensor to compute qparams for\n", - " bits: int = 8, # quantization bits\n", - " per_channel: bool = False, # per-channel or per-tensor\n", - " channel_axis: int = 0, # axis for per-channel quantization\n", - " ) -> tuple[torch.Tensor, torch.Tensor]:\n", - " \"\"\"Compute scale and zero_point for asymmetric quantization.\"\"\"\n", - " qmin, qmax = 0, 2 ** bits - 1\n", - " \n", - " if per_channel and tensor.dim() > 1:\n", - " dims = list(range(tensor.dim()))\n", - " dims.remove(channel_axis)\n", - " t_min, t_max = tensor.clone(), tensor.clone()\n", - " for dim in sorted(dims, reverse=True):\n", - " t_min = t_min.min(dim=dim).values\n", - " t_max = t_max.max(dim=dim).values\n", - " else:\n", - " t_min, t_max = tensor.min(), tensor.max()\n", - " \n", - " scale = (t_max - t_min) / (qmax - qmin)\n", - " scale = torch.clamp(scale, min=1e-8)\n", - " zero_point = torch.clamp(torch.round(-t_min / scale), qmin, qmax).to(torch.int32)\n", - " \n", - " if not per_channel or tensor.dim() <= 1:\n", - " scale = scale.view(1) if scale.dim() == 0 else scale\n", - " zero_point = zero_point.view(1) if zero_point.dim() == 0 else zero_point\n", - " \n", - " return scale, zero_point\n", - " \n", - " def _fake_quantize_per_channel(\n", - " self,\n", - " tensor: torch.Tensor, # tensor to quantize\n", - " bits: int = 8, # quantization bits\n", - " symmetric: bool = True, # symmetric or asymmetric\n", - " channel_axis: int = 0, # axis for per-channel quantization\n", - " ) -> torch.Tensor:\n", - " \"\"\"Apply fake quantization (per-channel).\"\"\"\n", - " qmin = -(2 ** (bits - 1)) if symmetric else 0\n", - " qmax = (2 ** (bits - 1)) - 1 if symmetric else (2 ** bits) - 1\n", - " \n", - " if symmetric:\n", - " scale, zero_point = self._compute_qparams_symmetric(\n", - " tensor, bits, per_channel=True, channel_axis=channel_axis\n", - " )\n", - " else:\n", - " scale, zero_point = self._compute_qparams_asymmetric(\n", - " tensor, bits, per_channel=True, channel_axis=channel_axis\n", - " )\n", - " \n", - " return torch.fake_quantize_per_channel_affine(\n", - " tensor, scale, zero_point, channel_axis, qmin, qmax\n", - " )\n", - " \n", - " def _fake_quantize_per_tensor(\n", - " self,\n", - " tensor: torch.Tensor, # tensor to quantize\n", - " bits: int = 8, # quantization bits\n", - " symmetric: bool = True, # symmetric or asymmetric\n", - " ) -> torch.Tensor:\n", - " \"\"\"Apply fake quantization (per-tensor).\"\"\"\n", - " qmin = -(2 ** (bits - 1)) if symmetric else 0\n", - " qmax = (2 ** (bits - 1)) - 1 if symmetric else (2 ** bits) - 1\n", - " \n", - " if symmetric:\n", - " scale, zero_point = self._compute_qparams_symmetric(tensor, bits, per_channel=False)\n", - " else:\n", - " scale, zero_point = self._compute_qparams_asymmetric(tensor, bits, per_channel=False)\n", - " \n", - " return torch.fake_quantize_per_tensor_affine(\n", - " tensor, scale.item(), int(zero_point.item()), qmin, qmax\n", - " )\n", - " \n", - " def _apply_weight_quantization(\n", - " self, \n", - " module: nn.Module, # layer to quantize\n", - " bits: int = 8, # quantization bits\n", - " per_channel: bool = True, # per-channel or per-tensor\n", - " ) -> None:\n", - " \"\"\"Apply weight quantization using fake_quantize.\"\"\"\n", - " weight = module.weight.data\n", - " if per_channel and weight.dim() > 1:\n", - " quantized = self._fake_quantize_per_channel(weight, bits, symmetric=True, channel_axis=0)\n", - " else:\n", - " quantized = self._fake_quantize_per_tensor(weight, bits, symmetric=True)\n", - " weight.copy_(quantized)\n", - " \n", - " def _create_activation_quantize_hook(\n", - " self,\n", - " layer_name: str, # layer name for config lookup\n", - " bits: int = 8, # quantization bits\n", - " ):\n", - " \"\"\"Create a forward hook that quantizes activations.\"\"\"\n", - " def hook(module, input, output):\n", - " if self._activation_quantize_config.get(layer_name, False):\n", - " return self._fake_quantize_per_tensor(output, bits, symmetric=False)\n", - " return output\n", - " return hook\n", - " \n", - " def _setup_activation_hooks(\n", - " self,\n", - " bits: int = 8, # quantization bits\n", - " ) -> None:\n", - " \"\"\"Register activation quantization hooks on all layers.\"\"\"\n", - " self._remove_activation_hooks()\n", - " for name, module in self._get_compressible_layers():\n", - " hook = self._create_activation_quantize_hook(name, bits)\n", - " handle = module.register_forward_hook(hook)\n", - " self._activation_hooks.append(handle)\n", - " self._activation_quantize_config[name] = False\n", - " \n", - " def _remove_activation_hooks(self) -> None:\n", - " \"\"\"Remove all activation quantization hooks.\"\"\"\n", - " for handle in self._activation_hooks:\n", - " handle.remove()\n", - " self._activation_hooks = []\n", - " self._activation_quantize_config = {}\n", - " \n", - " # ─── Main analysis method ────────────────────────────────────────────────────\n", - " \n", - " def analyze(\n", - " self,\n", - " compression: Literal[\"sparsity\", \"pruning\", \"quantization\"] = \"sparsity\", # compression type\n", - " level: float = 50, # compression level (% for sparsity/pruning, bits for quant)\n", - " *,\n", - " granularity: str = \"weight\", # granularity for sparsity (fasterai granularities)\n", - " layers: list[str] | None = None, # specific layer names to analyze (None = all)\n", - " layer_types: type | tuple[type, ...] | None = None, # restrict to a module type or tuple of types, e.g. nn.Conv2d or (nn.Conv2d, nn.Linear) (None = all compressible)\n", - " quant_per_channel: bool = True, # use per-channel quantization\n", - " quant_activations: bool = False, # also quantize activations\n", - " verbose: bool = True, # print progress\n", - " ) -> SensitivityResult:\n", - " \"\"\"Analyze per-layer sensitivity to compression.\n", - "\n", - " For **pruning**, each layer is pruned with the per-layer target `{name: level}` — the\n", - " exact operation `Pruner`/`PruneCallback` perform — so the reported Δ faithfully predicts\n", - " the degradation of really pruning that layer at `level`. Residual/skip-coupled layers\n", - " prune together and therefore share a Δ (tagged with a common `group_id`); layers that\n", - " cannot be pruned independently (output Linear, attention) are marked `prunable=False`.\n", - "\n", - " Pass `layer_types` to restrict the analysis to specific module types — a single type or\n", - " a tuple, e.g. `layer_types=nn.Conv2d` analyses only convolutions and skips the\n", - " classifier `Linear`.\n", - " \"\"\"\n", - " if compression not in self.VALID_COMPRESSIONS:\n", - " raise ValueError(f\"compression must be one of {self.VALID_COMPRESSIONS}\")\n", - " \n", - " self.model.eval()\n", - " \n", - " if compression == \"sparsity\":\n", - " self._init_sparsifier(granularity)\n", - " \n", - " if compression == \"quantization\" and quant_activations:\n", - " bits = int(level) if level > 1 else 8\n", - " self._setup_activation_hooks(bits)\n", - " \n", - " if verbose:\n", - " print(f\"Computing baseline {self.metric_name}...\", end=\" \", flush=True)\n", - " baseline = self.eval_fn(self.model)\n", - " if verbose:\n", - " print(f\"{baseline:.4f}\")\n", - " \n", - " all_layers = self._get_compressible_layers(layer_types)\n", - " if layers is not None:\n", - " all_layers = [(n, m) for n, m in all_layers if n in layers]\n", - "\n", - " # Pruning: precompute dependency groups once so coupled layers can share a group_id\n", - " # and each coupled set is only pruned/evaluated a single time (deduped by group id).\n", - " group_gid: dict[str, int] = {}\n", - " group_members: dict[int, list[str]] = {}\n", - " group_cache: dict[int, tuple[float, bool]] = {}\n", - " if compression == \"pruning\":\n", - " group_gid, group_members = self._build_dependency_groups(self.model)\n", - " \n", - " mode_info = \"\"\n", - " if compression == \"quantization\":\n", - " mode_info = f\" (per-{'channel' if quant_per_channel else 'tensor'}\"\n", - " mode_info += f\", {'weights+activations' if quant_activations else 'weights only'})\"\n", - " elif compression == \"sparsity\":\n", - " mode_info = f\" (granularity={granularity}, criteria={self.criteria.f.__name__})\"\n", - " elif compression == \"pruning\":\n", - " mode_info = f\" (structural group-sensitivity, criteria={self.criteria.f.__name__})\"\n", - " \n", - " if verbose:\n", - " unit = 'bits' if compression == 'quantization' else '%'\n", - " print(f\"Analyzing {len(all_layers)} layers for {compression} @ {level}{unit}{mode_info}\")\n", - " \n", - " results: list[LayerSensitivity] = []\n", - " \n", - " for i, (name, module) in enumerate(all_layers):\n", - " if verbose:\n", - " print(f\" [{i+1}/{len(all_layers)}] {name}...\", end=\" \", flush=True)\n", - "\n", - " prunable = True\n", - " gid = group_gid.get(name) # None for sparsity/quant and for ignored (output/attention) layers\n", - " if compression == \"sparsity\":\n", - " self._apply_sparsity(module, level)\n", - " compressed_metric = self.eval_fn(self.model)\n", - " self._restore_layer(module)\n", - " param_count = module.weight.numel()\n", - " \n", - " elif compression == \"pruning\":\n", - " if gid is None:\n", - " # ignored layer (output Linear / attention) — not independently prunable\n", - " compressed_metric, prunable = baseline, False\n", - " elif gid in group_cache:\n", - " compressed_metric, prunable = group_cache[gid] # coupled member: reuse the group's prune\n", - " else:\n", - " pruned_model, prunable = self._apply_structural_pruning(name, level)\n", - " compressed_metric = self.eval_fn(pruned_model) if prunable else baseline\n", - " del pruned_model\n", - " group_cache[gid] = (compressed_metric, prunable)\n", - " param_count = module.weight.numel()\n", - " \n", - " elif compression == \"quantization\":\n", - " saved_weight = module.weight.data.clone()\n", - " saved_bias = module.bias.data.clone() if module.bias is not None else None\n", - " \n", - " bits = int(level) if level > 1 else 8\n", - " self._apply_weight_quantization(module, bits, per_channel=quant_per_channel)\n", - " \n", - " if quant_activations:\n", - " self._activation_quantize_config[name] = True\n", - " \n", - " compressed_metric = self.eval_fn(self.model)\n", - " \n", - " if quant_activations:\n", - " self._activation_quantize_config[name] = False\n", - " \n", - " module.weight.data.copy_(saved_weight)\n", - " if saved_bias is not None:\n", - " module.bias.data.copy_(saved_bias)\n", - " param_count = module.weight.numel()\n", - " \n", - " if self.higher_is_better:\n", - " delta = baseline - compressed_metric\n", - " else:\n", - " delta = compressed_metric - baseline\n", - " \n", - " if verbose:\n", - " sign = \"+\" if delta > 0 else \"\"\n", - " tag = \"\" if prunable else \" (not prunable)\"\n", - " print(f\"Δ={sign}{delta:.4f}{tag}\")\n", - " \n", - " results.append(LayerSensitivity(\n", - " name=name,\n", - " layer_type=module.__class__.__name__,\n", - " params=param_count,\n", - " baseline_metric=baseline,\n", - " compressed_metric=compressed_metric,\n", - " delta=delta,\n", - " group_id=gid,\n", - " group_members=group_members.get(gid, []),\n", - " prunable=prunable,\n", - " ))\n", - " \n", - " if compression == \"sparsity\":\n", - " self._cleanup_sparsifier()\n", - " if compression == \"quantization\" and quant_activations:\n", - " self._remove_activation_hooks()\n", - " \n", - " compression_desc = compression\n", - " if compression == \"quantization\":\n", - " compression_desc = f\"quantization-{int(level) if level > 1 else 8}bit\"\n", - " if quant_activations:\n", - " compression_desc += \"+act\"\n", - " \n", - " self._results = SensitivityResult(\n", - " compression_type=compression_desc,\n", - " compression_level=level,\n", - " baseline_metric=baseline,\n", - " layers=results,\n", - " metric_name=self.metric_name,\n", - " higher_is_better=self.higher_is_better,\n", - " )\n", - " \n", - " if verbose:\n", - " print(f\"✓ Analysis complete\")\n", - " \n", - " return self._results\n", - " \n", - " def sweep(\n", - " self,\n", - " compression: Literal[\"sparsity\", \"pruning\", \"quantization\"] = \"sparsity\", # compression type\n", - " levels: list[float] | None = None, # compression levels to test (default: [25, 50, 75])\n", - " **kwargs,\n", - " ) -> list[SensitivityResult]:\n", - " \"\"\"Run sensitivity analysis at multiple compression levels.\"\"\"\n", - " if levels is None:\n", - " levels = [25, 50, 75]\n", - " results = []\n", - " for level in levels:\n", - " print(f\"\\n{'='*60}\")\n", - " unit = 'bits' if compression == 'quantization' else '%'\n", - " print(f\"Sweep: {compression} @ {level}{unit}\")\n", - " print(f\"{'='*60}\")\n", - " result = self.analyze(compression, level, **kwargs)\n", - " results.append(result)\n", - " return results" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4q5hi4zx5zh", - "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "---\n", - "\n", - "[source](https://github.com/FasterAI-Labs/fasterai/blob/master/fasterai/analyze/sensitivity.py#L242){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### SensitivityAnalyzer\n", - "\n", - "```python\n", - "\n", - "def SensitivityAnalyzer(\n", - " model:nn.Module, # model to analyze\n", - " sample:torch.Tensor, # example input (for Pruner dependency analysis)\n", - " eval_fn:Callable[[nn.Module], float], # evaluation function returning metric\n", - " criteria:Criteria=, # fasterai criteria for importance scoring\n", - " higher_is_better:bool=True, # whether higher metric values are better\n", - " metric_name:str='accuracy', # name of the metric for display\n", - " device:str | torch.device | None=None, # device for computation\n", - " calibration_data:torch.Tensor | None=None, # for observer-based quantization\n", - "):\n", - "\n", - "\n", - "```\n", - "\n", - "*Analyze per-layer sensitivity to compression methods.*\n", - "\n", - "Uses fasterai's Sparsifier for sparsity analysis and Pruner for structural pruning.\n", - "Supports sparsity (weight zeroing), pruning (structural), and quantization." - ], - "text/plain": [ - "def SensitivityAnalyzer(\n", - " model:nn.Module, # model to analyze\n", - " sample:torch.Tensor, # example input (for Pruner dependency analysis)\n", - " eval_fn:Callable[[nn.Module], float], # evaluation function returning metric\n", - " criteria:Criteria=, # fasterai criteria for importance scoring\n", - " higher_is_better:bool=True, # whether higher metric values are better\n", - " metric_name:str='accuracy', # name of the metric for display\n", - " device:str | torch.device | None=None, # device for computation\n", - " calibration_data:torch.Tensor | None=None, # for observer-based quantization\n", - "):\n", - "\"\"\"Analyze per-layer sensitivity to compression methods.\n", - "\n", - "Uses fasterai's Sparsifier for sparsity analysis and Pruner for structural pruning.\n", - "Supports sparsity (weight zeroing), pruning (structural), and quantization.\"\"\"" - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "show_doc(SensitivityAnalyzer)" - ] - }, - { - "cell_type": "markdown", - "id": "dmm4r4vwvhm", - "metadata": {}, - "source": [ - "The `eval_fn` should take a `nn.Module` and return a scalar metric (e.g., accuracy, loss). The analyzer will call it once per layer, so it should be reasonably fast — a forward pass on a small validation batch is typical.\n", - "\n", - "### Key Methods" + " raise AttributeError(f\"module {__name__!r} has no attribute {name!r}\")" ] }, { "cell_type": "code", "execution_count": null, - "id": "kkscpjtnhoc", - "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "---\n", - "\n", - "[source](https://github.com/FasterAI-Labs/fasterai/blob/master/fasterai/analyze/sensitivity.py#L608){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### SensitivityAnalyzer.analyze\n", - "\n", - "```python\n", - "\n", - "def analyze(\n", - " compression:Literal['sparsity', 'pruning', 'quantization']='sparsity', # compression type\n", - " level:float=50, # compression level (% for sparsity/pruning, bits for quant)\n", - " granularity:str='weight', # granularity for sparsity (fasterai granularities)\n", - " layers:list[str] | None=None, # specific layer names to analyze (None = all)\n", - " layer_types:type | tuple[type, ...] | None=None, # restrict to a module type or tuple of types, e.g. nn.Conv2d or (nn.Conv2d, nn.Linear) (None = all compressible)\n", - " quant_per_channel:bool=True, # use per-channel quantization\n", - " quant_activations:bool=False, # also quantize activations\n", - " verbose:bool=True, # print progress\n", - ")->SensitivityResult:\n", - "\n", - "\n", - "```\n", - "\n", - "*Analyze per-layer sensitivity to compression.*\n", - "\n", - "For **pruning**, each layer is pruned with the per-layer target `{name: level}` — the\n", - "exact operation `Pruner`/`PruneCallback` perform — so the reported Δ faithfully predicts\n", - "the degradation of really pruning that layer at `level`. Residual/skip-coupled layers\n", - "prune together and therefore share a Δ (tagged with a common `group_id`); layers that\n", - "cannot be pruned independently (output Linear, attention) are marked `prunable=False`.\n", - "\n", - "Pass `layer_types` to restrict the analysis to specific module types — a single type or\n", - "a tuple, e.g. `layer_types=nn.Conv2d` analyses only convolutions and skips the\n", - "classifier `Linear`." - ], - "text/plain": [ - "def analyze(\n", - " compression:Literal['sparsity', 'pruning', 'quantization']='sparsity', # compression type\n", - " level:float=50, # compression level (% for sparsity/pruning, bits for quant)\n", - " granularity:str='weight', # granularity for sparsity (fasterai granularities)\n", - " layers:list[str] | None=None, # specific layer names to analyze (None = all)\n", - " layer_types:type | tuple[type, ...] | None=None, # restrict to a module type or tuple of types, e.g. nn.Conv2d or (nn.Conv2d, nn.Linear) (None = all compressible)\n", - " quant_per_channel:bool=True, # use per-channel quantization\n", - " quant_activations:bool=False, # also quantize activations\n", - " verbose:bool=True, # print progress\n", - ")->SensitivityResult:\n", - "\"\"\"Analyze per-layer sensitivity to compression.\n", - "\n", - "For **pruning**, each layer is pruned with the per-layer target `{name: level}` — the\n", - "exact operation `Pruner`/`PruneCallback` perform — so the reported Δ faithfully predicts\n", - "the degradation of really pruning that layer at `level`. Residual/skip-coupled layers\n", - "prune together and therefore share a Δ (tagged with a common `group_id`); layers that\n", - "cannot be pruned independently (output Linear, attention) are marked `prunable=False`.\n", - "\n", - "Pass `layer_types` to restrict the analysis to specific module types — a single type or\n", - "a tuple, e.g. `layer_types=nn.Conv2d` analyses only convolutions and skips the\n", - "classifier `Linear`.\"\"\"" - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "show_doc(SensitivityAnalyzer.analyze)" - ] - }, - { - "cell_type": "markdown", - "id": "ad92a424", - "metadata": {}, - "source": [ - "> **Pruning is group-aware and faithful.** In `pruning` mode each layer is pruned with the exact per-layer target `{name: level}` that `Pruner`/`PruneCallback` use, so the reported `delta` *is* the degradation you get from really pruning that layer at `level` — not an approximation. Residual/skip-connected layers (e.g. a ResNet stem conv, block `conv2`s, `downsample`s) share output channels and can only be pruned together, so they are pruned as a group and report the **same** `delta`, tagged with a shared `group_id`. Layers that cannot be pruned independently (the output `Linear`, attention `qkv`) are marked `prunable=False` and surfaced separately rather than ranked as \"robust\". `to_layer_targets()` collapses each `group_id` to a single knob so coupled layers aren't double-counted.\n", - ">\n", - "> ⚠️ The `delta` is **level-specific** — an analysis at `level=50` predicts a 50% prune. To plan a 10% prune, analyze at `level=10`." - ] - }, - { - "cell_type": "markdown", - "id": "e0c5hyu284p", - "metadata": {}, - "source": [ - "For **quantization** analysis, additional parameters control the behavior:\n", - "\n", - "- `quant_per_channel=True` — per-channel quantization (more accurate, standard for weights)\n", - "- `quant_activations=False` — set to `True` to also quantize activations (slower but more realistic)\n", - "- `level` is interpreted as **bit width** (e.g., 8 for INT8) instead of percentage\n", - "\n", - "---" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "w2e32ob21ma", - "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "---\n", - "\n", - "[source](https://github.com/FasterAI-Labs/fasterai/blob/master/fasterai/analyze/sensitivity.py#L769){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### SensitivityAnalyzer.sweep\n", - "\n", - "```python\n", - "\n", - "def sweep(\n", - " compression:Literal['sparsity', 'pruning', 'quantization']='sparsity', # compression type\n", - " levels:list[float] | None=None, # compression levels to test (default: [25, 50, 75])\n", - " kwargs:VAR_KEYWORD\n", - ")->list[SensitivityResult]:\n", - "\n", - "\n", - "```\n", - "\n", - "*Run sensitivity analysis at multiple compression levels.*" - ], - "text/plain": [ - "def sweep(\n", - " compression:Literal['sparsity', 'pruning', 'quantization']='sparsity', # compression type\n", - " levels:list[float] | None=None, # compression levels to test (default: [25, 50, 75])\n", - " kwargs:VAR_KEYWORD\n", - ")->list[SensitivityResult]:\n", - "\"\"\"Run sensitivity analysis at multiple compression levels.\"\"\"" - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "show_doc(SensitivityAnalyzer.sweep)" - ] - }, - { - "cell_type": "markdown", - "id": "convenience-header", - "metadata": {}, - "source": [ - "---\n", - "\n", - "## Convenience Function\n", - "\n", - "For quick one-off analysis without creating an analyzer instance:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "convenience", - "metadata": {}, - "outputs": [], - "source": [ - "#| export\n", - "def analyze_sensitivity(\n", - " model: nn.Module, # model to analyze\n", - " sample: torch.Tensor, # example input tensor\n", - " eval_fn: Callable[[nn.Module], float], # evaluation function returning metric\n", - " compression: Literal[\"sparsity\", \"pruning\", \"quantization\"] = \"sparsity\", # compression type\n", - " level: float = 50, # compression level (% for sparsity/pruning, bits for quant)\n", - " *,\n", - " criteria: Criteria = large_final, # fasterai criteria for importance scoring\n", - " higher_is_better: bool = True, # whether higher metric values are better\n", - " metric_name: str = \"accuracy\", # name of the metric for display\n", - " granularity: str = \"weight\", # granularity for sparsity\n", - " verbose: bool = True, # print progress\n", - " **kwargs,\n", - ") -> SensitivityResult:\n", - " \"\"\"One-line sensitivity analysis using fasterai compression methods.\"\"\"\n", - " analyzer = SensitivityAnalyzer(\n", - " model, sample, eval_fn, \n", - " criteria=criteria, \n", - " higher_is_better=higher_is_better,\n", - " metric_name=metric_name,\n", - " )\n", - " return analyzer.analyze(compression, level, granularity=granularity, verbose=verbose, **kwargs)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "j2k207hhds", - "metadata": {}, - "outputs": [ - { - "data": { - "text/markdown": [ - "---\n", - "\n", - "[source](https://github.com/FasterAI-Labs/fasterai/blob/master/fasterai/analyze/sensitivity.py#L789){target=\"_blank\" style=\"float:right; font-size:smaller\"}\n", - "\n", - "### analyze_sensitivity\n", - "\n", - "```python\n", - "\n", - "def analyze_sensitivity(\n", - " model:nn.Module, # model to analyze\n", - " sample:torch.Tensor, # example input tensor\n", - " eval_fn:Callable[[nn.Module], float], # evaluation function returning metric\n", - " compression:Literal['sparsity', 'pruning', 'quantization']='sparsity', # compression type\n", - " level:float=50, # compression level (% for sparsity/pruning, bits for quant)\n", - " criteria:Criteria=, # fasterai criteria for importance scoring\n", - " higher_is_better:bool=True, # whether higher metric values are better\n", - " metric_name:str='accuracy', # name of the metric for display\n", - " granularity:str='weight', # granularity for sparsity\n", - " verbose:bool=True, # print progress\n", - " kwargs:VAR_KEYWORD\n", - ")->SensitivityResult:\n", - "\n", - "\n", - "```\n", - "\n", - "*One-line sensitivity analysis using fasterai compression methods.*" - ], - "text/plain": [ - "def analyze_sensitivity(\n", - " model:nn.Module, # model to analyze\n", - " sample:torch.Tensor, # example input tensor\n", - " eval_fn:Callable[[nn.Module], float], # evaluation function returning metric\n", - " compression:Literal['sparsity', 'pruning', 'quantization']='sparsity', # compression type\n", - " level:float=50, # compression level (% for sparsity/pruning, bits for quant)\n", - " criteria:Criteria=, # fasterai criteria for importance scoring\n", - " higher_is_better:bool=True, # whether higher metric values are better\n", - " metric_name:str='accuracy', # name of the metric for display\n", - " granularity:str='weight', # granularity for sparsity\n", - " verbose:bool=True, # print progress\n", - " kwargs:VAR_KEYWORD\n", - ")->SensitivityResult:\n", - "\"\"\"One-line sensitivity analysis using fasterai compression methods.\"\"\"" - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "show_doc(analyze_sensitivity)" - ] - }, - { - "cell_type": "markdown", - "id": "09g23bbjnc35", - "metadata": {}, - "source": [ - "### Usage Example\n", - "\n", - "```python\n", - "from fasterai.analyze.sensitivity import analyze_sensitivity\n", - "\n", - "result = analyze_sensitivity(model, sample, eval_fn, compression=\"sparsity\", level=50)\n", - "\n", - "# Inspect results\n", - "result.summary() # formatted console output\n", - "fragile = result.top(5, most_sensitive=True) # most sensitive layers\n", - "\n", - "# Generate per-layer targets for non-uniform compression\n", - "targets = result.to_layer_targets(model, target_pct=50, min_pct=10, max_pct=80)\n", - "# Pass directly to Sparsifier: sparsifier.sparsify_model(sparsity=targets)\n", - "```" - ] - }, - { - "cell_type": "markdown", - "id": "kptxpwxi3b", - "metadata": {}, - "source": [ - "---\n", - "\n", - "## See Also\n", - "\n", - "- [Sensitivity Tutorial](tutorials/analyze/sensitivity.html) - Step-by-step guide with real examples on ResNet18\n", - "- [Sparsifier](../sparse/sparsifier.html) - Apply non-uniform sparsity using `to_layer_targets()` output\n", - "- [Pruner](../prune/pruner.html) - Structural pruning (use `top()` to find layers to protect)\n", - "- [Criteria](../core/criteria.html) - Importance scoring methods used during analysis\n", - "- [Schedules](../core/schedules.html) - Control compression progression during training" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "khcwer3hi0j", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Ignoring output layer: 8\n", - "Total ignored layers: 1\n", - "Using per-layer pruning with 1 layer-specific ratios\n" - ] - } - ], - "source": [ - "#| hide\n", - "from fastcore.test import *\n", - "import torch.nn as nn\n", - "\n", - "def _test_model():\n", - " return nn.Sequential(\n", - " nn.Conv2d(3, 16, 3, padding=1),\n", - " nn.BatchNorm2d(16),\n", - " nn.ReLU(),\n", - " nn.Conv2d(16, 32, 3, padding=1),\n", - " nn.BatchNorm2d(32),\n", - " nn.ReLU(),\n", - " nn.AdaptiveAvgPool2d(1),\n", - " nn.Flatten(),\n", - " nn.Linear(32, 10)\n", - " )\n", - "\n", - "model = _test_model()\n", - "sample = torch.randn(2, 3, 8, 8)\n", - "def _eval(m):\n", - " m.eval()\n", - " with torch.no_grad(): return m(sample).abs().mean().item()\n", - "\n", - "analyzer = SensitivityAnalyzer(model, sample, _eval)\n", - "\n", - "# Invalid compression raises ValueError\n", - "with ExceptionExpected(ValueError):\n", - " analyzer.analyze('invalid', 50, verbose=False)\n", - "\n", - "# Result structure for sparsity analysis\n", - "result = analyzer.analyze('sparsity', 30, verbose=False)\n", - "assert isinstance(result, SensitivityResult)\n", - "test_eq(result.compression_level, 30)\n", - "assert len(result.layers) > 0\n", - "assert isinstance(result.layers[0], LayerSensitivity)\n", - "\n", - "# Backward-compat: sparsity mode leaves the new pruning fields at defaults\n", - "test_eq(result.layers[0].group_id, None)\n", - "test_eq(result.layers[0].prunable, True)\n", - "test_eq(result.layers[0].group_members, [])\n", - "\n", - "# top() sorted correctly (most sensitive first)\n", - "top3 = result.top(3, most_sensitive=True)\n", - "for i in range(len(top3)-1):\n", - " assert top3[i].delta >= top3[i+1].delta\n", - "\n", - "# to_layer_targets returns dict with layer names\n", - "sched = result.to_layer_targets(model, target_pct=50)\n", - "assert isinstance(sched, dict)\n", - "assert len(sched) == len(result.layers)\n", - "# All values should be percentages in valid range\n", - "for v in sched.values():\n", - " assert 0 <= v <= 90\n", - "\n", - "# LayerSensitivity.as_dict returns proper dict (now includes group fields)\n", - "d = result.layers[0].as_dict()\n", - "assert 'name' in d\n", - "assert 'delta' in d\n", - "assert 'params' in d\n", - "assert 'group_id' in d and 'prunable' in d\n", - "\n", - "# SensitivityResult.as_dict works\n", - "full_d = result.as_dict()\n", - "assert 'compression_type' in full_d\n", - "assert 'layers' in full_d\n", - "\n", - "# analyze_sensitivity convenience function works\n", - "result2 = analyze_sensitivity(\n", - " _test_model(), sample, _eval,\n", - " compression='sparsity', level=20, verbose=False\n", - ")\n", - "assert isinstance(result2, SensitivityResult)\n", - "\n", - "# ── Pruning mode: every prunable layer gets a group_id; the result is FAITHFUL ──\n", - "from fasterai.prune.pruner import Pruner\n", - "from fasterai.core.criteria import large_final\n", - "\n", - "_pm = _test_model()\n", - "_psample = torch.randn(2, 3, 8, 8)\n", - "_pbaseline = _eval(_pm)\n", - "_pres = SensitivityAnalyzer(_pm, _psample, _eval, criteria=large_final).analyze('pruning', 50, verbose=False)\n", - "_pby = {l.name: l for l in _pres.layers}\n", - "\n", - "# Internal conv ('0') and Linear analyzed; conv layers are prunable with a group_id\n", - "assert _pby['0'].group_id is not None\n", - "assert _pby['0'].prunable is True\n", - "# Output Linear ('8') is not independently prunable -> excluded from robust ranking\n", - "assert _pby['8'].prunable is False\n", - "assert _pby['8'] not in _pres.top(5, most_sensitive=False)\n", - "\n", - "# Faithfulness: analysis Δ for a layer == really pruning {layer: level} on a fresh copy\n", - "import io\n", - "def _clone(mdl):\n", - " b = io.BytesIO(); torch.save(mdl, b); b.seek(0); return torch.load(b, weights_only=False)\n", - "_c = _clone(_pm)\n", - "Pruner(_c, pruning_ratio={'0': 50}, context='local', criteria=large_final,\n", - " example_inputs=_psample).prune_model()\n", - "_real_delta = _pbaseline - _eval(_c)\n", - "test_close(_pby['0'].delta, _real_delta, eps=1e-5)\n", - "\n", - "# layer_types accepts a single type, a tuple, or a list (all equivalent) and filters out Linear\n", - "for _lt in (nn.Conv2d, (nn.Conv2d,), [nn.Conv2d]):\n", - " _r = SensitivityAnalyzer(_test_model(), sample, _eval).analyze('pruning', 50, layer_types=_lt, verbose=False)\n", - " test_eq({l.layer_type for l in _r.layers}, {'Conv2d'})" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "4zkpwcyede8", + "id": "sens-stub-4", "metadata": {}, "outputs": [], "source": [ "#| hide\n", - "#| slow\n", - "# Full sensitivity analysis on a larger model\n", - "from torchvision.models import resnet18\n", - "\n", - "_model_lg = resnet18(weights=None)\n", - "_sample_lg = torch.randn(2, 3, 32, 32)\n", - "def _eval_lg(m):\n", - " m.eval()\n", - " with torch.no_grad(): return m(_sample_lg).abs().mean().item()\n", - "\n", - "_result_lg = analyze_sensitivity(_model_lg, _sample_lg, _eval_lg,\n", - " compression='sparsity', level=30, verbose=False)\n", - "assert isinstance(_result_lg, SensitivityResult)\n", - "assert len(_result_lg.layers) > 3 # resnet18 has many conv layers\n", - "\n", - "_sched_lg = _result_lg.to_layer_targets(_model_lg, target_pct=30)\n", - "assert len(_sched_lg) > 3\n", - "for v in _sched_lg.values():\n", - " assert 0 <= v <= 90" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "60a3a921", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Ignoring output layer: fc\n", - "Total ignored layers: 1\n", - "Using per-layer pruning with 1 layer-specific ratios\n", - "════════════════════════════════════════════════════════════\n", - "Sensitivity Analysis: pruning @ 50%\n", - "════════════════════════════════════════════════════════════\n", - " Baseline accuracy: 0.3108\n", - " Layers analyzed: 21\n", - "\n", - " 🔴 Most Sensitive (fragile):\n", - " 1. layer2.0.conv2 Δ=+0.1348 [group 4]\n", - " 2. layer2.0.downsample.0 Δ=+0.1348 [group 4]\n", - " 3. layer2.1.conv2 Δ=+0.1348 [group 4]\n", - " 4. layer3.0.conv2 Δ=+0.1316 [group 7]\n", - " 5. layer3.0.downsample.0 Δ=+0.1316 [group 7]\n", - "\n", - " 🟢 Most Robust (compressible):\n", - " 1. layer1.0.conv1 Δ=-0.0081 [group 1]\n", - " 2. layer4.0.conv1 Δ=+0.0003 [group 9]\n", - " 3. layer4.1.conv1 Δ=+0.0025 [group 11]\n", - " 4. layer3.1.conv1 Δ=+0.0195 [group 8]\n", - " 5. layer1.1.conv1 Δ=+0.0224 [group 2]\n", - "\n", - " ⚪ Not prunable in isolation (1):\n", - " 1. fc (group None)\n" - ] - } - ], - "source": [ - "#| hide\n", - "#| slow\n", - "# Pruning group-sensitivity on ResNet-18 — regression test for the false-robust bug\n", - "# (residual-coupled layers used to report Δ=0 \"robust\" because they could not be pruned in isolation).\n", - "import io\n", - "from torchvision.models import resnet18\n", - "from fasterai.prune.pruner import Pruner\n", - "from fasterai.core.criteria import large_final\n", - "\n", - "torch.manual_seed(0)\n", - "_rn = resnet18(weights=None).eval()\n", - "_s = torch.randn(2, 3, 32, 32)\n", - "def _ev(m):\n", - " m.eval()\n", - " with torch.no_grad(): return m(_s).abs().mean().item()\n", - "_base = _ev(_rn)\n", - "\n", - "_res = SensitivityAnalyzer(_rn, _s, _ev, criteria=large_final).analyze('pruning', 50, verbose=False)\n", - "_by = {l.name: l for l in _res.layers}\n", - "\n", - "# Residual-coupled layers (stem conv1, block conv2, downsample) are NO LONGER falsely \"robust\":\n", - "# the bug made their delta EXACTLY 0 (a no-op prune). Now each is prunable and actually moves the metric.\n", - "_coupled = ['conv1', 'layer1.0.conv2', 'layer2.0.downsample.0']\n", - "_robust_names = {l.name for l in _res.top(5, most_sensitive=False)}\n", - "for _name in _coupled:\n", - " assert _name in _by, f\"{_name} missing from result\"\n", - " assert _by[_name].prunable, f\"{_name} should be prunable via its group\"\n", - " assert abs(_by[_name].delta) > 1e-6, f\"{_name} falsely reports Δ≈0 (the no-op bug)\"\n", - " assert _name not in _robust_names, f\"{_name} wrongly ranked most-robust\"\n", - "\n", - "# Coupled layers share a group_id and therefore an identical delta\n", - "assert _by['conv1'].group_id == _by['layer1.0.conv2'].group_id\n", - "test_close(_by['conv1'].delta, _by['layer1.0.conv2'].delta, eps=1e-6)\n", - "\n", - "# An internal conv (output feeds only the next conv) is in a DIFFERENT group, prunable, and actually moved\n", - "assert _by['layer1.0.conv1'].group_id != _by['conv1'].group_id\n", - "assert _by['layer1.0.conv1'].prunable and abs(_by['layer1.0.conv1'].delta) > 1e-6\n", - "\n", - "# Faithfulness: the reported Δ equals really pruning {layer: level} on a fresh copy\n", - "def _clone(mdl):\n", - " b = io.BytesIO(); torch.save(mdl, b); b.seek(0); return torch.load(b, weights_only=False)\n", - "_c = _clone(_rn)\n", - "Pruner(_c, pruning_ratio={'conv1': 50}, context='local', criteria=large_final,\n", - " example_inputs=_s).prune_model()\n", - "_real = _base - _ev(_c)\n", - "test_close(_by['conv1'].delta, _real, eps=1e-5)\n", - "\n", - "# summary() runs without error\n", - "_res.summary()" + "# The fasterai stub raises a helpful ImportError for the moved names\n", + "# (they moved to the higher-level FasterAI package; fasterai holds mechanisms only).\n", + "from fastcore.test import ExceptionExpected\n", + "import fasterai.analyze.sensitivity as s\n", + "for _name in ('analyze_sensitivity', 'SensitivityAnalyzer', 'SensitivityResult', 'LayerSensitivity'):\n", + " with ExceptionExpected(ImportError):\n", + " getattr(s, _name)" ] }, { "cell_type": "code", "execution_count": null, - "id": "export", + "id": "sens-stub-5", "metadata": {}, "outputs": [], "source": [ diff --git a/nbs/tutorials/analyze/sensitivity.ipynb b/nbs/tutorials/analyze/sensitivity.ipynb deleted file mode 100644 index 8ece04b..0000000 --- a/nbs/tutorials/analyze/sensitivity.ipynb +++ /dev/null @@ -1,1015 +0,0 @@ -{ - "cells": [ - { - "cell_type": "raw", - "id": "frontmatter", - "metadata": {}, - "source": [ - "---\n", - "title: Sensitivity Analysis\n", - "description: Find which layers are most sensitive to compression\n", - "output-file: tutorial.sensitivity.html\n", - "skip_showdoc: true\n", - "skip_exec: true\n", - "---" - ] - }, - { - "cell_type": "markdown", - "id": "overview", - "metadata": {}, - "source": [ - "## Overview\n", - "\n", - "Not all layers in a neural network respond equally to compression. Some layers are robust and can be heavily compressed with minimal impact, while others are fragile and degrade quickly. **Sensitivity analysis** helps you identify which layers fall into each category.\n", - "\n", - "This tutorial shows you how to:\n", - "1. Analyze layer sensitivity to different compression methods\n", - "2. Identify fragile vs robust layers\n", - "3. Generate non-uniform per-layer compression targets based on sensitivity" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "imports", - "metadata": {}, - "outputs": [], - "source": "#| include: false\nimport torch\nimport torch.nn as nn\nfrom torchvision.models import resnet18\nfrom fasterai.analyze.sensitivity import SensitivityAnalyzer, analyze_sensitivity\nfrom fasterai.core.all import large_final" - }, - { - "cell_type": "markdown", - "id": "setup-header", - "metadata": {}, - "source": [ - "## 1. Setup\n", - "\n", - "First, we need a model and an evaluation function. The evaluation function takes a model and returns a metric (e.g., accuracy, loss)." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "setup", - "metadata": {}, - "outputs": [ - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/nathan/miniconda3/envs/dev/lib/python3.12/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n", - " warnings.warn(\n", - "/home/nathan/miniconda3/envs/dev/lib/python3.12/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=ResNet18_Weights.IMAGENET1K_V1`. You can also use `weights=ResNet18_Weights.DEFAULT` to get the most up-to-date weights.\n", - " warnings.warn(msg)\n" - ] - } - ], - "source": [ - "# Load a pretrained model\n", - "model = resnet18(pretrained=True)\n", - "model.eval()\n", - "\n", - "# Create sample input\n", - "sample = torch.randn(1, 3, 224, 224)\n", - "\n", - "# Define an evaluation function\n", - "# In practice, this would evaluate on your validation set\n", - "def eval_fn(m):\n", - " \"\"\"Simple proxy: measure output magnitude (replace with real accuracy)\"\"\"\n", - " with torch.no_grad():\n", - " out = m(sample)\n", - " return out.abs().mean().item()" - ] - }, - { - "cell_type": "markdown", - "id": "basic-header", - "metadata": {}, - "source": [ - "## 2. Basic Sensitivity Analysis\n", - "\n", - "The simplest way to run sensitivity analysis is with the `analyze_sensitivity()` function:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "basic-analysis", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Computing baseline accuracy... 1.1848\n", - "Analyzing 21 layers for sparsity @ 50% (granularity=weight, criteria=abs)\n", - " [1/21] conv1... Δ=+0.0064\n", - " [2/21] layer1.0.conv1... Δ=-0.0987\n", - " [3/21] layer1.0.conv2... Δ=-0.0649\n", - " [4/21] layer1.1.conv1... Δ=-0.0779\n", - " [5/21] layer1.1.conv2... Δ=+0.0648\n", - " [6/21] layer2.0.conv1... Δ=-0.1699\n", - " [7/21] layer2.0.conv2... Δ=-0.1183\n", - " [8/21] layer2.0.downsample.0... Δ=-0.2051\n", - " [9/21] layer2.1.conv1... Δ=-0.1259\n", - " [10/21] layer2.1.conv2... Δ=+0.0256\n", - " [11/21] layer3.0.conv1... Δ=-0.0785\n", - " [12/21] layer3.0.conv2... Δ=+0.0398\n", - " [13/21] layer3.0.downsample.0... Δ=-0.0064\n", - " [14/21] layer3.1.conv1... Δ=-0.0353\n", - " [15/21] layer3.1.conv2... Δ=-0.0569\n", - " [16/21] layer4.0.conv1... Δ=-0.0521\n", - " [17/21] layer4.0.conv2... Δ=+0.0452\n", - " [18/21] layer4.0.downsample.0... Δ=+0.0108\n", - " [19/21] layer4.1.conv1... Δ=-0.0230\n", - " [20/21] layer4.1.conv2... Δ=-0.0772\n", - " [21/21] fc... Δ=+0.0573\n", - "✓ Analysis complete\n" - ] - } - ], - "source": [ - "result = analyze_sensitivity(\n", - " model, \n", - " sample, \n", - " eval_fn,\n", - " compression=\"sparsity\", # \"sparsity\", \"pruning\", or \"quantization\"\n", - " level=50, # 50% sparsity\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "results-header", - "metadata": {}, - "source": [ - "## 3. Understanding the Results\n", - "\n", - "The `SensitivityResult` object provides several ways to inspect the results:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "results-summary", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "════════════════════════════════════════════════════════════\n", - "Sensitivity Analysis: sparsity @ 50%\n", - "════════════════════════════════════════════════════════════\n", - " Baseline accuracy: 1.1848\n", - " Layers analyzed: 21\n", - "\n", - " 🔴 Most Sensitive (fragile):\n", - " 1. layer1.1.conv2 Δ=+0.0648\n", - " 2. fc Δ=+0.0573\n", - " 3. layer4.0.conv2 Δ=+0.0452\n", - " 4. layer3.0.conv2 Δ=+0.0398\n", - " 5. layer2.1.conv2 Δ=+0.0256\n", - "\n", - " 🟢 Most Robust (compressible):\n", - " 1. layer2.0.downsample.0 Δ=-0.2051\n", - " 2. layer2.0.conv1 Δ=-0.1699\n", - " 3. layer2.1.conv1 Δ=-0.1259\n", - " 4. layer2.0.conv2 Δ=-0.1183\n", - " 5. layer1.0.conv1 Δ=-0.0987\n" - ] - } - ], - "source": [ - "# Print a formatted summary\n", - "result.summary()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "results-top", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "layer1.1.conv2: Δ=0.0648\n", - "fc: Δ=0.0573\n", - "layer4.0.conv2: Δ=0.0452\n" - ] - } - ], - "source": [ - "# Get the top 3 most sensitive layers\n", - "fragile = result.top(3, most_sensitive=True)\n", - "for layer in fragile:\n", - " print(f\"{layer.name}: Δ={layer.delta:.4f}\")" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "results-robust", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "layer2.0.downsample.0: Δ=-0.2051\n", - "layer2.0.conv1: Δ=-0.1699\n", - "layer2.1.conv1: Δ=-0.1259\n" - ] - } - ], - "source": [ - "# Get the top 3 most robust layers (safe to compress heavily)\n", - "robust = result.top(3, most_sensitive=False)\n", - "for layer in robust:\n", - " print(f\"{layer.name}: Δ={layer.delta:.4f}\")" - ] - }, - { - "cell_type": "markdown", - "id": "visualize-header", - "metadata": {}, - "source": [ - "## 4. Visualizing Sensitivity\n", - "\n", - "A bar chart helps visualize which layers are most sensitive:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "visualize", - "metadata": {}, - "outputs": [ - { - "data": { - "image/png": "iVBORw0KGgoAAAANSUhEUgAABW0AAAHpCAYAAAD5+R5uAAAAOnRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjEwLjMsIGh0dHBzOi8vbWF0cGxvdGxpYi5vcmcvZiW1igAAAAlwSFlzAAAPYQAAD2EBqD+naQAAxXxJREFUeJzs3Xd8FHX+x/H3zG4KLQkQegtNiqAgXUFUUBBPD8+KeIoFFSvinWcX5RT1Zy8cep6enLST4zhED+VUbCAqikYCiPReUwlJdne+vz8mGRNSyEIgyeb1fDzyyCezs7Ofz35nv7v7yeysZYwxAgAAAAAAAABUCXZlJwAAAAAAAAAA+BVNWwAAAAAAAACoQmjaAgAAAAAAAEAVQtMWAAAAAAAAAKoQmrYAAAAAAAAAUIXQtAUAAAAAAACAKoSmLQAAAAAAAABUITRtAQAAAAAAAKAKoWkLAAAAAAAAAFUITVsAAAAcUxs3bpRlWbIsS2eccUa5rzdx4kTven//+9+P6LYLrp+UlHRE18fxURFjDQAAEElo2gIAgIhRuPEzZsyYyk7nmHr77bc1cOBAxcXFKSYmRs2aNVPv3r11ww036Kuvvqrs9MolLS1NEydO1MSJE497o67gdp9//vljsv0VK1Z4t7F48eJjchs1weLFi737ccWKFcf89jZu3Kg//OEP6tq1q+rUqaPExET17dtXTz31lPbv3x/WtgrPRyX9lLRfzJ49W6eddprq1q2runXr6rTTTtM///nPYuu99tpr6tKli2rXrq0uXbro9ddfL7bOjBkzZFmWJk+eHFbeAACgavBXdgIAAAAIzyOPPKKJEycWWbZz507t3LlTy5cvV+vWrdW/f//KSa4EzZo10+effy5Jio+P95anpaXpkUcekSQNHjy4WKP92muv1dChQyVJJ5xwwhHddsHtxsbGFllecLtt2rTR+PHjj2jbZVmxYoV3G5LCOsK4JiptrBcvXuzdj0lJSerRo8cxy+G5557Tfffdp5ycHG9Zdna29u3bp2+++UbPPfec3nrrLZ1zzjnH5PYnTpxYZJ+RpCVLlmjJkiX6+eef9cADD0iS5s6dqxtvvFFnnHGG/v73v+uee+7R2LFjlZiYqJEjR0qSDhw4oLvvvlvt2rXThAkTjkm+AADg2KJpCwAAUMUcOHBAderUKfGyrKws78i5WrVqadKkSerRo4f279+vtWvX6t1335VlWccz3cOKiYnRwIEDw75e69at1bp166O67SO5XRy9vLw82bYtv798bzcqYqyPRuF/hPTp00e///3v1aVLF+Xl5SklJUUzZszQ999/rwsuuECLFi3SoEGDwtp+wT8PCuvevbsXr1ixQpMmTZIk1atXTy+88IIk6Y477lBmZqYmTpyoCy64QCeddJLeeecdSdLtt9+ufv366bbbbtPixYv1zjvveE3byZMna9u2bZo3b55iYmLCvTsAAEBVYAAAACLEww8/bCQZSebqq68uc93JkyebwYMHmxYtWpjY2FhTq1Yt06VLF3P//febAwcOGGOMef31173tPfTQQ0WuP2/ePO+yW2+91VuemZlpHn74YXPiiSea2NhYU69ePTN48GDz/vvvF7n+hg0bvOsPHjzYfPrpp6Z///4mNja2zNy/+uor73q/+93vSlynIP9D8x0yZIhJSEgw0dHR5oQTTjATJ0402dnZRdYbPHiwt/0ffvjB3HrrraZRo0YmNjbWDB8+3GzcuLHI+p988okZMmSIqV+/vvH7/SYxMdH06dPH3H777SYtLa3EWo0x5uqrr/aWHfpTsE7h8XzzzTeNMcZ0797dSDI+n8/s2bOnSC5nn322t/7KlSuNMcb7u02bNsW2eehPmzZtjmjMD9WmTZtSb+Phhx/21lu7dq0ZM2aMadmypYmKijINGjQw5557rvnf//5X6rYLy87ONn/4wx9Mhw4dTHR0tKldu7ZJSkoyF154oZk7d663XuH7+sMPPzQPPPCAad68uYmNjTWDBg0yy5cvL7Ldf//73+b88883SUlJpm7duiYqKsq0bt3ajBkzxmzYsKHIuoW3/f7775sJEyaYpk2bGsuyzIYNG0woFDJ//vOfvcdDTEyMadWqlRkxYoR5/fXXve2UNNal3YeSzN/+9jfTunVrI8nUrl3bZGZmFsmrR48e3n6ya9euMu/Hzz77zNi2beLj482CBQtKXMdxHDNlyhTj8/lMixYtTFZW1uGGp0hNhzNu3Dhv3cmTJ3vLJ0+eXGyfu/DCC40kL9cFCxYUmQ/Wr19vYmNjzdlnn33Y2wUAAFUXTVsAABAxwmnadurUqdSG0JlnnmmMcRuwdevWNZJMhw4dilz/2muv9dZfsmSJMcaYtLQ0r6lY0s8rr7ziXb9wI7OggVae3FeuXOmtFxcXZ6ZOnWq2bNlSZq0PPvhgqTkNGjTI5ObmeusWbtq2a9eu2PqnnXaat+7q1atNrVq1St322rVri9V6tE3bJ554wlv22muvebns27fP+P1+I8n07NnTWx5u0zbcMS9JeZq2y5YtM/Xq1StxHcuyzJQpU8oc00PzOfRn9OjR3nqF7+uS9vu4uDizZs0ab/0bb7yx1O02adKkSBO08LYP3V82bNhgHn300VK3VXhfCrdp++abbxa5zj/+8Q9vW5s3b/aWn3POOYe9H08//XTj8/nMl19+aYwx5uOPPzZdu3Y1derUMRMmTDC33HKLV8/LL79sJJlXX331sNstnF+rVq1MVFSUadasmRk9erT5+eefi6xbeN749NNPveWffvqpt/zkk082xhgvhyuuuMJkZWWZ0aNHG0nePnPhhRcav9/v/eMCAABUT3wRGQAAqJFuuukm/eMf/9D777+vxYsXa/78+RoxYoQk6ZNPPtGSJUtUt25dXXrppZKkX375RcuWLZMkOY6j9957T5J7ns0BAwZIku6//34lJydLkkaMGKH33ntP06ZNU9OmTSVJd955p7Zs2VIsl+3bt6tly5Z6++239f7773sfcS5Jx44dvY+RZ2Rk6KabblKrVq3UqlUrXXPNNVq6dGmR9b/55hvvY9fNmjXT3/72Ny1cuFDnnXeeJPdj288991yJt7Vnzx5NnTpVb7/9thISEiRJX375pVauXClJWrRokQ4ePCjJ/Rj3Rx99pDlz5ujPf/6zevfuXeZpGu6//37vY96S1KNHD33++ef6/PPP9dJLL5V6vSuuuMLb7pw5c7zl8+bNUzAYlCRdeeWVpV7/2muvLfJR9aZNm3q3O2fOnLDHvCRz5szRfffd5/19zTXXeLdx7bXXyhija665RpmZmZKkiy++WO+9954efPBB2bYtY4zGjx9f4r5S2H/+8x9J7nl558yZow8//FB/+9vfdNVVV6l+/folXmfLli164YUXNG/ePPXu3VuSux/de++93jrnnHOOXn31Vb377rtavHixFi5cqLvuukuStGvXrhK/9EqS1q9fr9tvv10LFy7Uq6++qnr16nk5JiQk6O2339b//vc/TZs2TTfddJOaNWtWZn2ff/65rrnmGu/v++67z7sfR4wYoWuuucbbF6ZPn+6tN3/+fC8eNWpUmbexefNmffbZZxo7dqxOPfVU7d69W+eff75SUlJ04MABPfvss/rggw+89ceNG6fmzZvr/fffL3O7h9qyZYsCgYB27Nih6dOnq3fv3t5cIblfgFagSZMmXty4cWMv3rBhgyR37rr55ps1e/Zs1a1bV7NmzdKtt96qG2+8UR999JH+/e9/65ZbblHXrl0luWNW8NgAAADVSGV3jQEAACpKOEfa/vTTT+byyy/3PpquQ47ke+GFF4wxxnz55Zfesttuu80YY8zSpUu9Zffcc48xxphQKGTq169vJJno6Gjzv//9z3z++efm888/NzfffLO3/tNPP22MKXr0qW3bZvXq1eWu89NPPzWNGzcu9SjEgtyNMeaOO+7wlt93331eTu+++663vFu3bt76hY+0fe6557zlN910k7d83rx5xhhjpk6d6i17/vnnzY4dO0rMt6QjbctaXqCkoy8L5xgVFWX2799vjDHm3HPP9e7L7du3e+sWXL/gSNvDLTem/GNeljfffLPY0bUFvvvuO++ypk2bmry8PO+yiy66qMT7vyRNmzb1jsD8/vvvTU5OTonrFT4a9v777/eW//zzz97y2NhYL499+/aZCRMmmE6dOpV4JPWFF15Y4ravuOKKYrfdv39/I8m0aNHCLF26tMRTdxhT+liXtrxAwSkx/H6/dwTwsGHDjCQTExPjnaKjNG+//baRZJYuXWqMMWbixIlGkunTp4957733zIQJE4ocOWyMMSNGjDBdu3Ytc7vGGPPkk0+aUaNGmTfffNN8+OGH5pVXXvHGTJIZMmSIt65t297y9evXe8vXrVvnLff5fEW2f/DgQbNu3Tpz8OBBY4wxgUDAnHjiiSYxMdGkpqaamTNnmsTERO++uPvuu00oFDps3gAAoGrgSFsAAFDjbNq0SaeeeqpmzZqlrVu3KhAIFFsnLS1NknTqqaeqc+fOkqTZs2crFAqVeCTf3r17lZqaKsn9EqahQ4dq0KBBGjRokKZMmeKtv2rVqmK31bFjR3Xq1Knc+Z9++ulas2aNXn/9df32t79Vw4YNi1x+zz33ePn//PPP3vLHH3/cy+n888/3lq9evbrE2xk8eLAXF76Ngm0Xvu3x48erWbNmatCggc4999wiR9FWtIIjaQOBgObNm6e0tDR99NFHkqSzzjrrsEdwHk55x/xIFR6TU045RVFRUd7fffv2LXG9klx33XWSpB9++EE9e/ZUnTp11LVrV02YMEE7duwo8Tr9+vXz4o4dO3pH5Obk5Gj79u0KhUIaOnSonn32Wa1Zs8Y7krqwgvE/VOF96tAct23bpgEDBqhu3brq0KGDbrzxxsPWVx4F2w8Gg5o9e7aysrK0ePFiSe7R7vHx8WVef/v27ZKkLl26SJJ39Ovzzz+vESNG6JlnnlGHDh2KXMfv98txnMPmdvfdd2vGjBkaM2aMzj77bN18882aMWOGd/nixYu9+7fwFw/m5uZ6cV5enhcf+uWEsbGxateunWJjYyVJf/nLX7Ry5Uo99thj2r59u37/+9/LcRz95S9/Uffu3fXUU0+VepQ0AACoemjaAgCAGuett95SRkaGJGnAgAGaN2+ePv/8c919993eOoWbMgWNod27d+vDDz/0GngnnniiTjrppLBu+8CBA8WWFf44dHklJCTouuuu07x587R792699957qlWrliTp4MGDpTZiSxIMBos0igoU/oi93+/3YmOMJPfUAsuXL9ef/vQnDRw4UA0bNlRqaqoWLlyoSy+9VLNmzQq7rvK4+OKLFRMTI8k9FcH8+fO95tbo0aMr5DYqcszDUdYpJQ41adIkzZw5U5dccok6deoky7K0atUqPffcczrnnHPK9ZH4Q2/vyy+/1Pfffy/JPZ3GW2+9pc8++0wzZ8701imtYVnSfnz99dfrv//9r37/+9+rW7duio6O1rp16/Taa69p8ODBpTaAy2vkyJFq0KCBJOntt9/WwoULvX35iiuuKPd2Cvbvgv2obt263mX16tXz4gMHDmjJkiVq06bNEeVbuCkfCoW8f/QkJSV5y3ft2uXFO3fu9OK2bduWut19+/bp4YcfVo8ePXT99ddrzpw5CgaDGjdunG666SY9/vjjktx/QgAAgOqBpi0AAKhxtm3b5sX33Xeffvvb32rgwIFKT08vcf2rrrrKOxryscce887pWviIy8TERK/JWbduXWVmZsq4X/rq/YRCIb355pvFth9Oo27//v366quviiyzbVsjRozwjg6V3IaQJJ1wwgnesjfffLNYTsYYHThwwGuChsMYozZt2uiJJ57Q559/rr179+qbb77xLp87d26Z17ftX1+KlufIxQIJCQneOXn/97//6W9/+5skqVatWrrooovKtY2C+7y02y3PmJelrNoKj8n3339fpLlacA7dQ9crzeWXX65//vOfWr16tTIzM3XxxRdLkn766acSj2T9+uuvvfiXX37R/v37JblHbTZv3rzIY+OKK67QVVddpUGDBh02D6nk/dgYo+HDh2vatGlKTk5WVlaWxo8fL8ltSC5ZsqTMbR5uH4mJifGOvP7666/1/PPPS3IbrQX7SFkKjspOSUmRJLVr106S9MwzzygjI0Pz58/XihUrJLnnnb388su1d+9eXXDBBYfd9rfffltsWeHx9fv9XsN54MCB3vLC90nhc1SXNQ4PPPCAUlNT9eKLL8q2ba/ZW9BcLmgKF24CAwCAqs1/+FUAAACqn+XLl+uee+4ptvxPf/pTkaPkXnzxRUVHR2vZsmVe8+9QjRs31m9+8xv9+9//1pdffuktv/zyy73Ytm2NGjVKU6ZMUVZWls455xzdfvvtSkxM1NatW/XTTz9p7ty5euONN3TGGWcccV379+/XgAED1L9/f1144YXq3r27oqKi9PHHH3vNpZiYGO9o0CuuuEIvvPCCJPeL0Pbv36+TTjpJaWlpWrdunT788EO1adNGb7zxRti5zJw5U1OnTtXIkSPVtm1bxcfH6+OPP/YuL+no3cIKH8mbnJysefPmKTExUa1bt/a+bK00V155pebOnau8vDx99tlnkqQLLrigyFGRh7vt/fv3a/v27Zo+fbratGmjJk2aqGPHjpLKN+blrW3hwoU6/fTTFRsbq+7du6tHjx7q0qWLVq1apR07dmj06NEaM2aMli1bpn//+9+SpOjo6MM2oE877TT17NlTffv2VYsWLZSZmek1H6WS7//nnntOTZo0UevWrfXYY495y88991xFRUUVeWz861//0sCBA5WamlriY6k8Lr74YtWrV0+DBg1Sy5YtFQwGizQzw9lH/vWvf6lt27aKiopSnz59vH80XHfddXrxxRclyRurkSNHekeel6WgWfr222+rT58+GjNmjF544QVNmzZN06ZN83JITU3VmWeeKck9pUXBkdhl6du3r84991xddNFFatOmjdasWeN9KaAkDRs2zDu1wdixY/Xqq6/KcRw9/vjjatKkiSzL8o6Q9fl8Gjt2bIm388MPP+ivf/2rLr/8cq+xW9Ck3bNnT5HfR3qEMAAAqASVcypdAACAilf4S4tK+9mwYYPZtGmTqV27drHLTjvttFK/PGrBggVF1u3bt2+x209NTTXdu3cv8/Y/+eQTY8zhv4SrNGvXrj1sjRMnTixynQcffLDM9Qt/aVvhLyIr+OKlQ+/bgi+E+sc//lHmdmfOnHnYWnv16lXsegX3fVlfQpWTk2MSEhKKXG/+/PnF7q+Cyw79wrHCX/hV0v1gTPnGvDR79uwxMTExpY7/smXLTL169Uq83yzLMlOmTDnsbbRv377U+75r164mGAwaY4p+WdhJJ51UbN26deuaVatWGWOMCQaDJa5T+LFReAwLb7ugtsKGDBlSao5NmjTxviistLH+8ccfjWVZJT6OC+vdu3eRy99///1yj1X//v1NTEyMWb58uTHGmMcff9z4fD4jyfTr189cd911RnK/rO2666477JebFSjrsdGkSRPzyy+/FFm/rPlr0qRJpd7O4MGDTe3atc3mzZu9ZVu3bjV16tQx7du3N1988YUZOXKkkX79EkEAAFD1cXoEAABQ47Ru3Voffvih+vbtq1q1aql9+/aaMmWKrr/++lKvM3z4cDVv3tz7u6SPySckJGjp0qWaNGmSTj75ZNWqVUu1a9dWx44ddfHFF2vmzJnq37//UeXepk0bzZ07VzfeeKN69Oihxo0by+/3KyEhQWeccYamT5+uhx9+uMh1Hn30US1YsEDDhw9Xw4YNFRUVpRYtWmjgwIF64okn9MgjjxxRLgMGDNAdd9yhU045RYmJifL5fIqPj9egQYM0e/bsch2VOnPmTA0fPrzIEZXlERMTo0suucT7u2HDhho+fHi5r//yyy/r0ksvVaNGjUpdpzxjXprExETNmzdPPXv2LPGIz759+2r58uW6+uqr1aJFC/n9ftWvX1/Dhw/Xhx9+qHHjxh32Nu6991799re/VZs2bVS7dm1FRUUpKSlJN910kz7++GP5fL5i13nmmWc0ceJEtWjRQjExMRo4cKA++eQT79QaPp9P7733nn77298qPj5ejRo10h133HHEX2B1880367LLLlP79u1Vt25d+f1+tWjRQqNHj9YXX3xx2C8K6969u6ZNm6YuXbqUeQqPwke+JiYm6uyzzy53jpMmTVJeXp5GjBihzz77TPfee6+2b9+u7777Tl988YXuu+8+/fDDD0pOTtbrr79+2JwLzJ07V1deeaVOOOEE1atXTzExMerYsaPGjx+vH374Qe3bty+y/sSJEzVr1iwNGDBAderUUZ06dTRgwADNnj1bDzzwQIm38c9//lOffvqp/vSnP6lVq1be8hYtWuiDDz5Qo0aNNGzYMK1atUqvvfaafvvb35b7fgEAAJXLMib/myQAAABQpmuvvVZvvvmmbNvW1q1bvfNhInJFwpiPGTNGb731liTpk08+OarTc1RVmzdv9j76P27cOE2ZMiWs6993332aPHmyJOk3v/mNfvvb36pt27YKBAJKSUnRvHnztGTJEn3++ecaMGBAhecPAABwKM5pCwAAUAaT/0Vd69at03vvvSdJOvvss6tl8w7lw5hXH7m5ucrKyvLOaSu5XyIXrscff1x169bVxIkTtWDBAi1YsKDYOrfddpv69et3VPkCAACUF01bAACAMmzatElt27b1/rYsq9SPKiMyMObVx4033ugdRSy5zfUjPQXJfffdp4suukgvvfSSPvzwQ23dulV16tTRaaedpvHjx0fkEcoAAKDqomkLAABQDj6fTx06dNDEiRO9b5xHZGPMq4/4+HgNHz5cL7300lFtp1OnTnr55ZcrKCsAAIAjxzltAQAAAAAAAKAKsSs7AQAAAAAAAADAr2jaAgAAAAAAAEAVQtMWAAAAAAAAAKoQmrYAAAAAAAAAUIXQtAUAAAAAAACAKoSmLQAAAAAAAABUITRtAQAAAAAAAKAKoWkLAAAAAAAAAFUITVsAAAAAAAAAqEJo2gIAAAAAAABAFeKv7AQigeM42r59u+rVqyfLsio7HQAAAAAAAABVkDFGmZmZat68uWy79ONpadpWgO3bt6tVq1aVnQYAAAAAAACAamDLli1q2bJlqZfTtK0A9erVk+Te2XFxcZWcDQAAAAAAAICqKCMjQ61atfL6iaWhaVsBCk6JEBcXR9MWAAAAAAAAQJkOd4pVvogMAAAAAAAAAKoQmrYAAAAAAAAAUIXQtAUAAAAAAACAKoSmLQAAAAAAAABUITRtAQAAAAAAAKAKoWkLAAAAAAAAAFUITVsAAAAAAAAAqEJo2gIAAAAAAABAFULTFgAAAAAAAACqEH9lJwAAODIzrE6VnULYrjBrKjsFAAAAAACqPI60BQAAAAAAAIAqhKYtAAAAAAAAAFQhNG0BAAAAAAAAoAqhaQsAAAAAAAAAVQhNWwAAAAAAAACoQmjaAgAAAAAAAEAVQtMWAAAAAAAAAKoQf2UnAAAAAAAAAKDiLIjrVNkphO03GWsqO4UqhSNtAQAAAAAAAKAKoWkLAAAAAAAAAFVItWvavvLKK0pKSlJsbKz69eunr7/+usz133nnHXXu3FmxsbHq3r273n///SKXjxkzRpZlFfkZPnz4sSwBAAAAAAAAAEpVrZq2s2fP1oQJE/Twww/ru+++08knn6xhw4Zp9+7dJa6/ZMkSjRo1Stddd52+//57jRw5UiNHjtRPP/1UZL3hw4drx44d3s/MmTOPRzkAAAAAAAAAUEy1ato+++yzGjt2rK655hp17dpVU6dOVe3atfXGG2+UuP4LL7yg4cOH649//KO6dOmiSZMm6ZRTTtHLL79cZL2YmBg1bdrU+6lfv/7xKAcAAAAAAAAAiqk2Tdu8vDwtX75cQ4cO9ZbZtq2hQ4dq6dKlJV5n6dKlRdaXpGHDhhVbf/HixWrcuLE6deqkcePGad++fWXmkpubq4yMjCI/kuQ4jvf7cHEoFCoSG2PCjo0xxWJJYceO4xSJw6mDmqiJmiqvJkX5JcsqOS5QUmxZpcd+nxvbhWO79NiX/zTiOyS282O/r0hcE8eJmqiJmqiJmqiJmqiJmqiJmqjpuNckuT9R/iKxJBnLKj3Of79XJLbtorGvcOy+3zO+Q2K7IPb9GvsPifPfwxaOa9I4HU61adru3btXoVBITZo0KbK8SZMm2rlzZ4nX2blz52HXHz58uKZNm6aPPvpITz75pD799FOde+653sCVZPLkyYqPj/d+WrVqJUnatm2bJHmnWZCkrVu3eqdv2Lx5s/bu3StJ2rhxo1JTUyVJ69evV3p6uiRp7dq1yszMlCStXr1a2dnZkqSUlBTl5ORIkpKTkxUIBOQ4jpKTk+U4jgKBgJKTkyVJOTk5SklJkSRlZ2dr9erVkqTMzEytXbtWkpSenq7169dLklJTU7Vx40bvft68ebMkaffu3dq6dSs1URM1VdGaokadLaux+8mA6KtGyEqo58bXXyDVqSVF+d04yi/VqeXGkqyEeoq+aoQbN66vqFFnu3HLxoq6+CxJkt22uaLOH+TGnVrLP7y/G3drJ/9ZvSVJvl6d5B/Uw437nShfvxMlSf5BPeTr1cmNz+otu1s7Nx7ev0aOEzVREzVREzVREzVREzVREzVR0/GuSbXd94Tmyvz3hLVrubEkxdeTucx9T6jE+jK/c98TqnljmfPd94Rq01xmmPueUB1ay5zlvidUl3Yyp7vvCXVyJ5n+Pdy414nuj+QuO9l9T2hO7y11cd8TmrP6Sx1au/GwQVKb5m58/llS88Y1bpwOxzLlbe9Wsu3bt6tFixZasmSJBgwY4C2/++679emnn2rZsmXFrhMdHa233npLo0aN8pZNmTJFjzzyiHbt2lXi7axfv17t27fX//73Pw0ZMqTEdXJzc5Wbm+v9nZGRoVatWik1NVUJCQleh9227VLjUCgky7K82LZtWZYVViy53f3Csc/nkzEmrNhxHBljvPhwuVMTNVFT1ahpRvSJUjAkGeM+CReOA0F3giopLjiitqTYZ7vbsS33CNlgyP1tWyXHlqSQ8+tRtgWxkeQ47rYd48WX5/5U48aJmqiJmqiJmqiJmqiJmqiJmqjpeNf0fv2uklTsPaEVCLpHtfp9Jcc+W1YwVDS23feBXmxZskIFsWSFHO8oWy82kuU47lG5xrhx/vtDLw45sozx4vPTV9eIccrKylJCQoLS09MVFxen0lSbpm1eXp5q166tOXPmaOTIkd7yq6++WmlpafrPf/5T7DqtW7fWhAkTNH78eG/Zww8/rHnz5umHH34o9bYaNWqkP//5z7rxxhvLlVtGRobi4+MPe2cDQEWaYXWq7BTCdoVZU9kpAAAAAEDEWxBX/d4v/iajZrxfLG8fsdqcHiE6Olq9evXSRx995C1zHEcfffRRkSNvCxswYECR9SVp0aJFpa4vuYc779u3T82aNauYxAEAAAAAAAAgDP7Dr1J1TJgwQVdffbV69+6tvn376vnnn9eBAwd0zTXXSJKuuuoqtWjRQpMnT5Yk3XHHHRo8eLCeeeYZnXfeeZo1a5a+/fZbvfbaa5KkrKwsPfLII7rooovUtGlTrVu3Tnfffbc6dOigYcOGVVqdAAAAAABUR8vad67sFMLWb93qyk4BAIqpVk3byy67THv27NFDDz2knTt3qkePHlq4cKH3ZWObN2/2zmkhSaeeeqpmzJihBx54QPfdd586duyoefPmqVu3bpIkn8+nH3/8UW+99ZbS0tLUvHlznXPOOZo0aZJiYmIqpUYAAAAAAAAANVu1OadtVcY5bQFUBs5pCwAAgKqGI22BqoFz2lZdEXdOWwAAAAAAAACoCWjaAgAAAAAAAEAVUq3OaQsAAAAgfMndqt/Hlbv/xMeVAQBAzcWRtgAAAAAAAABQhdC0BQAAAAAAAIAqhKYtAAAAAAAAAFQhNG0BAAAAAAAAoAqhaQsAAAAAAAAAVQhNWwAAAAAAAACoQmjaAgAAAAAAAEAVQtMWAAAAAAAAAKoQmrYAAAAAAAAAUIXQtAUAAAAAAACAKoSmLQAAAAAAAABUITRtAQAAAAAAAKAKoWkLAAAAAAAAAFWIv7ITAAAAQNW17fw+lZ3CEWnx7jeVnQIAAABwxDjSFgAAAAAAAACqEJq2AAAAAAAAAFCF0LQFAAAAAAAAgCqEpi0AAAAAAAAAVCE0bQEAAAAAAACgCqFpCwAAAAAAAABVCE1bAAAAAAAAAKhCaNoCAAAAAAAAQBVC0xYAAAAAAAAAqhCatgAAAAAAAABQhdC0BQAAAAAAAIAqhKYtAAAAAAAAAFQhNG0BAAAAAAAAoArxV3YCAAAAAAAA1cGB24dWdgpHpM6L/6vsFACEiSNtAQAAAAAAAKAKoWkLAAAAAAAAAFUITVsAAAAAAAAAqEI4py3Cln3PuZWdQthqP/Hfyk4BAAAAAAAAKBeOtAUAAAAAAACAKoSmLQAAAAAAAABUIdWuafvKK68oKSlJsbGx6tevn77++usy13/nnXfUuXNnxcbGqnv37nr//feLXG6M0UMPPaRmzZqpVq1aGjp0qNauXXssSwAAAAAAAACAUlWrpu3s2bM1YcIEPfzww/ruu+908skna9iwYdq9e3eJ6y9ZskSjRo3Sddddp++//14jR47UyJEj9dNPP3nrPPXUU3rxxRc1depULVu2THXq1NGwYcOUk5NzvMoCAAAAAAAAAI9ljDGVnUR59evXT3369NHLL78sSXIcR61atdJtt92me+65p9j6l112mQ4cOKAFCxZ4y/r3768ePXpo6tSpMsaoefPmuuuuu/SHP/xBkpSenq4mTZro73//uy6//PIS88jNzVVubq73d0ZGhlq1aqX/+7//U+3atVVwl1qWpUsvvVQNGjRQWlqaZs6c6S0vvM6NN94o27a1bt06ffDBB7IsS5J7FLBlWapfv74uvfRS2batb775Rt98802xddq1a6ezzz5bPp9PCxcu1Lp164qt07t3b/Xq1Us+n08zZ87U/v37i+Vy9tlnq3379rJtW6+88kqJ+Z6/9X+q7+Qo3YrWv6M7lXgfjcn7SZYx2mjH6ZOoNoUusSQZJZgc/TawTrYJ6XtfE63wN5VkiqyT5KRrcHCrfCakj6PaaKOdUGydHsFd6uHskW0czY3upDQrplguZwQ2q+ufp5dZUySOEzVFfk3pKb9IRopKzVTzOUukYEhpvTsove8JUsHUblmSMaq9bqcaffyjFAhqz/BTlN2+Wf46Vv7DySj+21+U8N06KRjS9lGnK9CgXrF1EhetUJ0Nu6RgSJtuPs9brvycZIya/fMLRe/PUl5Cbe0YNTj/YeuuE9elfY0bJ2qipupeU2DzeklSl90b1WXfVtlOSB926KOMmNo6VL+tKWqRuU+242jOiYOLXS5JQ9ctV3zuAWVE19KiDn1KXOd3q76QZRxtq5eor1p1LXZ5XO4BDd3wvaxQSKsatVFK46Ri63Q6Y2iNGqfD1ZSz7hf3jjFS9+QVarJ7l6xQSIuGjfCWK38ql5H6L/1CdQ9k6UCt2lo68PQS1xn60QeS42h34yb6sccph75MU50Dmeq/bKmsUEjr23XQ+g4di63TeNcOdV+ZLDsY1A89TtHuJk29dWLat69x40RN1BQJNR38ZZ0sSae9584Re5s10erepxR+6EuSamdm6pTPl0qhkDaf0EGbO3XUoRK371DnFcmygkGt6nOK9jZrWmQblqRWa9aq9S/rZYVCWn7m6cquV7fYOp2Wr1DiTnfe+/yCEcVz6dih3ON08M7h2hgVr8W123tbMJYlyxglODm64MAa2U5IK2Ka6ftazWWZouu0CaRpcM4m+ZyQPq7dThuj6xdb5+ScHeqRt8t9n1uvq9LtWjp0Ah18cIOSAumyjaM3408pVNGvE/UFWatV3zmodCtG/+l4bsTve9RUtKas1fnP/Zb7njBu0061/PJHWYGgtpxxijKSir8nbPTDL2r00zpZwZB+GXm6chPqFXu/1/KzFYrb7D6eVo45z1teeJ12736h2LQs5cbV1rqRg0tcp+v0D6SQo4zWTbX1TPd1RL0uHWrEOGVlZSkhIUHp6emKi4tTafylXlLF5OXlafny5br33nu9ZbZta+jQoVq6dGmJ11m6dKkmTJhQZNmwYcM0b948SdKGDRu0c+dODR061Ls8Pj5e/fr109KlS0tt2k6ePFmPPPJIseWhUEiSFAwGJUlRUVHauXOngsGgoqOjFQgEZNu2/H6/AoGAfD6ffD6f1q9fr0aNGnl1+v1++Xw+5eXlKSoqSpKUkpKiDh06SHKbxjExMUVix3GUnJysHj16KBgMKjc3V7GxsXIcR4FAQDExMcrJydHatWvVuXNnBQIBBQIBRUdHy3EchUIhRUdHKz09XZs3b1ZSUpJCoZCMMYqKiipSU/oVjyoqMVG1oqMVnDbNqykvL8+racfZ16tRo0aK2btXgXff9WrKzc1VVFSUfA0bamO369WhQwdFJycr74svitVkt22rdc2bq0ePHtKCBcr7+ediNTndztPWxo3VuXNnOdOmKbh/v6KjoxUKhbyacvpectiaCo/Twbw0WZaR7XMUCvhk2Y5sn9HSn95Q7YQDSsvM1sFc93LLNgrm+eTzO8oM7NMX301T/Zb7tTM7R9k5lnxR7m2EAn75ooLal52qz77dqSYdd2nvgZCycyR/dFDGsRQK2vJHh7Q19RPl/pishm32Kv1gUAdzJV9USE7IknFs+aJC+mXnfO3Nq6v4ZmnKCQRkjFQ7pkH59z07Vz6fpfVr/6tGiZYUylBeXpb8fslnW8rLNYqKkmQCSvnpfXVo55NCu5Wbk6X8YVJurhQTIznBjUr+4X316O5XMLBJublZio2x5DhGgYAUE2MpJ3ul1q7epc4n+BXI26VAXq6ioy05IaNQSIqOtpSe+o02b1ijpNY+hYJZMkaKirIUDLoTYJTfCuvxlBNaqtz8mmxbysuV/FFSyOTqh58WKKmdlBfao5ycLEXn15SXK0XHSLnB9fr+hwXq0l3KDWxTTq5bt+NIwYC7Tmb2j1q5eofanyDl5O10H6/RUigkOSEpKlram/qVnA0patlaCgSzJOPmkD9M8vulzds+VlZOjPxRuWHPEbHtWykqKkoNGzZUt3tvV4cOHZScnKwvSng8tT3vbDV/wn08LViwQD+X8HjqdtmFapz/eJo2bZr2l/B46nnNaCUmJiopKUkvvvii93gKBALevtf7k+uUmJio6OhoTTtkjnAcp9zz3gNzn9GXeSlKTIvVSZnNleW4n4Coa8cqy8nRN6t/1peZP2lB7jc6ZW8LtcxJUJZzUD7LVqxidMAc1FffrlXDjR9qcV6yBu9sp3qBGGWbXEVZfvnl00GTq2WfTlXUd7FaHvhFwzM6yZalHJOnGMvNI9cEdM/7zygzKlfbcnbrjIz2CslRngmqlhWjoEIKmKDu+NckrQ/u1It9/1DuuTxgp0syUjBa8udpfdp2LViyQVbLtdqaVkdZB2Mkf55kLCkUJfnz9OPOBdrx9VJZTTdpZ2ac8nJ8ki8gObZk3PirjTOUkl5bVsMdysqNd6/vC0qOz9357JA+XvVXxcQ5ylOqsnLiJMtIdsi9HSsk2Y4WfPuiVDdVmak5yjpYT/KFJMtRtKlf7uenVxbP0fxfvtJfVi7UbxNOUrfazZWamyW/7VPdqFil5R7Qf+Z9qwafN9CsX77QdYmnqlFUXWXkZSvGF6Von1+ZeQf172lPyK4VpQ+3rNBdTc6Sbdk6EMhRbb97e9nBXF015QHtDWZp3f7tur7RqQoaRznBPNWLrqW8UFC5oYAueeYu/bhvo5QT1KWJpyg7kKeAE1RCTB1lBXK0Z8tP6tatW1jPuRnZ+2Xyx8lyomR8efpm2zxtzPtCuYnrtDOjoYIH/TK+gKz8cTK+gBb/8g8t31dLgYStyshpJMmSsYOyHPflmbGDev/HvygqLqSDZr8yDiZKliNjh2Tlj5OxHc356lkF6+xTVmquMg42kOygjOXICkXL2AEdyNqqf32xR3kNNyptl62s7ATVq12/3K8jnCYtFBMTo9gRF+hA/hzhmzZNpoQ5wn/x5QolJqpVUpKsUuYIc+UY+RMT1TQ6WqaU1xE5192oRo0aqeHevXJKeB0R27Ch0q69WR06dFB8crJCJcx74bw2atV+TYXN5XmBLNm25PNJgTzJ9rnx6rUL1CBRygllVNjzUziv96KS2nrjVP+KKxSXP5d/Uso4xV17rTeXLytlnGqNG6dGjRqp9t69Si5hnOIaNpR9q/v8lJOcrPUljFODoUNlmjdX9x49tGnBAu0u9PyUl5cX1mvYDf94UK2zf1EwrZOMZSnKyVPQducqvxPQtn8/oxxfrqIO7FYgs71s48hngsqzY+QzIflMUGtnTFJi7k6FsvKUl91GfhOQbULKtWspyuTKpK/XT7O2qF1WikJZscrJa66YkPv8lOuLVUwoR8EDP+uH/T+pe/o3CmS1UG4wQTGhg3IsWwErRjHOQWUvWavVP3+oE698oPyvy/PSFWWHFMyfy6PskHZ+P02BulK0k67AwSjZcuS3HQUcv2zLkd9ytO7LV9UoNkMm7aDysqPktx35LEd5oShF2UHJt1srP31DHeN3SdsCys2OVowdlCyj3FCUYnwBhXbvUfLirTo5cYuCu33KzfYp1heQYywFHL9ifAHlbPpQP2d/p871dyiQ5ldetqUYX1BOVMPyv9ews7Vz40IF06MV7T+oQF6mbMvI75MCQUs+28hnS+tT5qpR/ZB0IEt5uZny+9zleXmWoqKMFMxWyvf/VofWQSknVbk5mYqJMpIl5eZZiok2cnLWKHl5lnp0Cih4cI9yc7IVG23kGCkQcNfJSV+utcmb1bltQIGcbQrkBhUdZeQ4UsixFO03St/1mTZHRZX7vUYgN122Lflto0DQlm0b+W2jdT/OUqOEgEx6lvJyMuX3G/kso7yArSi/I+VlauXyf6pji4NSVoZyDx5UTLTj7nt5tmKiHYUOrFTyN2k6uX2WggfSlZuTp9gox60paCsmylHO/q/084pf1Ll1tgLZ+5WXG1KM31HIseQ4lqL9jtK3f6RNOcuU1DRHobx0GbtO2TUd8ho2pl1b+Xw+NXzPnSP27t2rtSXMEfUbNlSdO9w5wk5O1tYS5ojGZw9V7L3ua9g9CxZofwmvYZteOFIJ+c9Pa6ZNU14Jz09NR1/hvYZdVsK8J6n87zUefUexZbzP3dRtrPc+N1Coprz8mnxtz9D6/Pe51oIFChSqKZhfk+n2G23Lr8mU8j43t++l2pdfk1PKXJ5x+lWKTkxU7ehoBQrN5Yd7r2Ef3K1cE61oKygT3KqVC/eqQ9RGmV22crPiFW3lyZJRrolRjJWr0OYt+nHhOp0cs1rBzfWUm1VbMVaujGzlGb9irDzlrP6Pft77pTpFr1dgT33lHfQr2gooJJ8c2YpSQOnfva2Nv9RSUtQ2BQ8kuvO3ggrmt678CmrH568qr1ZQ0YH9ChxoKFuOfAopEN0wrB7LDz8tUFJ7S3nOXuXkZik62u0nFrzPzQuu1/c/LlDX7rZyA1tKfE+YlZ2slDU71P4EW7mHvo5wpKgoaW/aMpkNq9SyjaVg/vvcQ19HbNn+iQ7klvA6IiAvXv3Le2rR5PQyazr0dXndzkVfz7Y592w1f7Ts94QdLjn8e8JOV//6nnB9Kftelxt/fU+4tZTXEZ2X3ezNEbvzH08F9VXFXlhF9vcaN26s8qg2R9pu375dLVq00JIlSzRgwABv+d13361PP/1Uy5YtK3ad6OhovfXWWxo1apS3bMqUKXrkkUe0a9cuLVmyRKeddpq2b9+uZs2aeetceumlsixLs2fPLjGX0o60TU1NVUJCghzHffK0bbvUOBQKybIsL7ZtW5ZlhRVL7tHGhWOfzydjTFix4zgyxnjx4XKP9JpW7n3S/UeTJTmOJcsyRxVLkjFFY9s2MuboYhnJsuXFJyb+qfzjFPxAtm0pFDKybeWPQfljd2xUJPb5rPwxKH/sOG4tBbGbYxmxb1i5973s4Hw5Ifc+siz3idMuIy65Jvf+LS02jvtmvHDsOO54eLHcbZYnruU7P+IeT0czR/R5e5RCcmQk+WUrKHdbJcWWJF8psS1bITmyZMmWpZAc2bJklRJLkiNTauyTLSNTYuzI6LsrZ5V7nBZs+qO7AxjLbZrmx5b3OK+gWO4cZIxbx6+xO18Zx7398sbntf6/cu979s0DFGX7FHBCsiT5S4l9lk9BE5ItS7Zlu7FlyVbh2FLQOLKt/DEzjnyWe3sFsZGRY4z8ljsebuyTI+fX2Dhy5MYhE5KRFGX7FHTcOPjyl2E9nt5edbtkO+7BLcYuFFuSbQ6JLffvglhyxz7c2Mk/IsKL3WZE0diWLKfE+MouL1b7OaIi570cZ0GFzeVF4pAk69e44Dmpop6favl+U6PG6XA1BV4dLVtO/uwv2TJlxLYkI1tG7szza2zLyPJi91qHiyXJOST25T+HlRXHjptZ7nEyPz4i25LyXxqVELvzum1JIWO5NeXHdv58XyR2LNlW6bHkbrNw7Mt/Xik1liWfdUjc/eHy73uZs0t9PIXyp7CC2M5/vV4kPgav9xwj+eyisZM/zRfEir+s/I+hXa+780J+7lahuMSaDhO7Y1M0PjTf8sSOcf8uiGXyx8BIVuK1ETFHROK8d6xqMh/drpDJn+sslRiXOO9ZTv7jv7S4+BzhGMt9POXH7pwWfmzOfD6sccrMna+KeJ/rvnYoGjuOyX8dUc73uaXFIZP/OsKN60SfH/H7Xk2pKeKOtK1KYmJivO5+YQU7TMHvsmKfz3fMYsuyworLk29NqskqdKZn2zYVEhc0bAvHlnV0ccGnXgrH5R4nx8qPrULLw41VLHbHoPyxbf+6vfLF4e17dgk5hhtbVumxVUJsF9l/wo0j7/F0NDUVNGbLE5sy4pAXG4XyP1bmvl0uLVaZcajQ7ZcWl7dWq2D3LjRHqMjjvIJiFcSlzEt2eHG4Yxlw3E+imDLioHFjt9GaHxsjR4Vj48UFNYVMoTEoFAeLxKHDxgW5hFufZVmSnX9bltzGqBebEmLjzdkljXu5Y7s8sVNqXPBRreo8R1TovOdU5FxeKPYdPj6656caNk6HqamgcWAXmrNLjwvN2ccwtsoTl3OcCv5vU+il0SFxoXm6PLFdjriE61pWGbGKx+Hvbyox9pUnPgav9wpe/haOi+UYxvvAsGsqT2wVj0vLvdSaCr+nOCS2eJ9b42pym6iF5rTS4pLmPausuIQ5otA8czRx+GNQMe9zrRLi8N/nlhL7isZlvX4LN66q+15NqalgLA/HPvwqVUNiYqJ8Pp927dpVZPmuXbvUtGnTEq/TtGnTMtcv+B3ONgEAAAAAAADgWKo2Tdvo6Gj16tVLH330kbfMcRx99NFHRU6XUNiAAQOKrC9JixYt8tZv27atmjZtWmSdjIwMLVu2rNRtAgAAAAAAAMCxVK1OjzBhwgRdffXV6t27t/r27avnn39eBw4c0DXXXCNJuuqqq9SiRQtNnjxZknTHHXdo8ODBeuaZZ3Teeedp1qxZ+vbbb/Xaa69Jcg+LHj9+vP785z+rY8eOatu2rR588EE1b95cI0eOrKwyAQAAAAAAANRg1appe9lll2nPnj166KGHtHPnTvXo0UMLFy5UkyZNJEmbN28ucl6JU089VTNmzNADDzyg++67Tx07dtS8efPUrVs3b527775bBw4c0A033KC0tDQNHDhQCxcuVGxs7HGvDwAAAMCRiRpX8pcIAwAAVEfVqmkrSbfeeqtuvfXWEi9bvHhxsWWXXHKJLrnkklK3Z1mWHn30UT366KMVlSIAAAAAAAAAHLGwm7YbNmzQ559/rk2bNik7O1uNGjVSz549NWDAAI5OBQAAAAAAAICjVO6m7fTp0/XCCy/o22+/VZMmTdS8eXPVqlVL+/fv17p16xQbG6vRo0frT3/6k9q0aXMscwYAAAAAAACAiFWupm3Pnj0VHR2tMWPG6F//+pdatWpV5PLc3FwtXbpUs2bNUu/evTVlypQyT0kAAAAAAAAAAChZuZq2TzzxhIYNG1bq5TExMTrjjDN0xhln6LHHHtPGjRsrKj8AAAAAAAAAqFHK1bQtq2Fb2P79+9WwYUM1bNjwqJICAAAAAAAAgJrKroiNfPjhh7rsssvUsmXLitgcAAAAAAAAANRYR9y03bRpkx5++GElJSXp/PPP16ZNm5Sbm1uRuQEAAAAAAABAjRNW0zYvL0+zZs3S0KFD1aFDB/3vf//T3XffrW3btun1118/VjkCAAAAAAAAQI1RrnPaStJtt92mGTNmqEmTJho9erT++te/qm3btt7lu3btOiYJAgAAAAAAAEBNUu6m7SuvvKKrrrpKTz75pJo0aXIscwIAAAAAAACAGqvcp0eYPn26tm3bptatW2vYsGH6xz/+oaysrGOZGwAAAAAAAADUOOVu2o4aNUqLFi3S6tWr1a9fPz3wwANq0qSJLr/8cr377rvKy8s7lnkCAAAAAAAAQI0Q1heRSVLbtm316KOPauPGjZo7d66MMbrkkks0cODAY5EfAAAAAAAAANQoYTdtC1iWpWHDhmn27Nnavn27HnvsMXXr1q0icwMAAAAAAACAGqfcX0RWlgYNGmj8+PEaP358RWwOAAAAAAAAlcAe+lJlpwBA5TzS9oknntDBgwfLtcFly5bpvffeO6qkAAAAAAAAAKCmKlfTNiUlRa1bt9bNN9+s//73v9qzZ493WTAY1I8//qgpU6bo1FNP1WWXXaZ69eods4QBAAAAAAAAIJKV6/QI06ZN0w8//KCXX35ZV1xxhTIyMuTz+RQTE6Ps7GxJUs+ePXX99ddrzJgxio2NPaZJAwAAAAAAAECkKvc5bU8++WT99a9/1auvvqoff/xRmzZt0sGDB5WYmKgePXooMTHxWOYJAAAAAAAAADVC2F9EZtu2evTooR49ehyDdAAAAAAAAACgZivXOW0BAAAAAAAAAMcHTVsAAAAAAAAAqEJo2gIAAAAAAABAFULTFgAAAAAAAACqkKNq2m7ZskVbtmypqFwAAAAAAAAAoMYLu2kbDAb14IMPKj4+XklJSUpKSlJ8fLweeOABBQKBY5EjAAAAAAAAANQY/nCvcNttt2nu3Ll66qmnNGDAAEnS0qVLNXHiRO3bt09/+ctfKjxJAAAAAKjJrJMfqewUAADAcRR203bGjBmaNWuWzj33XG/ZSSedpFatWmnUqFE0bQEAAAAAAADgKIR9eoSYmBglJSUVW962bVtFR0dXRE4AAAAAAAAAUGOF3bS99dZbNWnSJOXm5nrLcnNz9dhjj+nWW2+t0OQAAAAAAAAAoKYJ+/QI33//vT766CO1bNlSJ598siTphx9+UF5enoYMGaLf/e533rpz586tuEwBAAAAAAAAoAYIu2mbkJCgiy66qMiyVq1aVVhCAAAAAAAAAFCThd20ffPNN49FHgAAAAAAAAAAHUHTtsCePXu0Zs0aSVKnTp3UqFGjCksKAAAAAAAAAGqqsL+I7MCBA7r22mvVrFkznX766Tr99NPVvHlzXXfddcrOzj4WOQIAAAAAAABAjRF203bChAn69NNP9e677yotLU1paWn6z3/+o08//VR33XXXscgRAAAAAAAAAGqMsJu2//rXv/S3v/1N5557ruLi4hQXF6cRI0bor3/9q+bMmXMscpQk7d+/X6NHj1ZcXJwSEhJ03XXXKSsrq8zr5OTk6JZbblHDhg1Vt25dXXTRRdq1a1eRdSzLKvYza9asY1YHAAAAAAAAAJQl7KZtdna2mjRpUmx548aNj+npEUaPHq2VK1dq0aJFWrBggT777DPdcMMNZV7nzjvv1Lvvvqt33nlHn376qbZv367f/e53xdZ78803tWPHDu9n5MiRx6gKAAAAAAAAAChb2F9ENmDAAD388MOaNm2aYmNjJUkHDx7UI488ogEDBlR4gpK0atUqLVy4UN9884169+4tSXrppZc0YsQIPf3002revHmx66Snp+tvf/ubZsyYobPOOkuS25zt0qWLvvrqK/Xv399bNyEhQU2bNj0muQMAAAAAAABAOMI+0vb555/Xl19+qZYtW2rIkCEaMmSIWrVqpSVLluiFF144Fjlq6dKlSkhI8Bq2kjR06FDZtq1ly5aVeJ3ly5crEAho6NCh3rLOnTurdevWWrp0aZF1b7nlFiUmJqpv37564403ZIwpM5/c3FxlZGQU+ZEkx3G834eLQ6FQkbjgNsOJjTHFYklhx47jFInDqSMSazKOVLALOI511LExxWM336OLjZtukbj841Rwv5tCY1D+2L2vi8a/jkH5Y8cpGhfkVXoc3r7nhH4dy9BhYmOKxwX3b2mxU0LsOIfETjhx5D2ejqYmv2y5e7wbq4zYKiP2ebHlxXYZccGtlhb7ZJcaW/lx+cep4DFsFYndMajA2BTE1iGxez8aJ7w4nH1PkqJsnzcepcV+y+fd115sHRrbXuzLj32WXSS2Lbc+f5HYVzTWr3HBPhZl/xqH+3iSk79PGh0SWyXEVtHYHGHsHBqrhNguNY6EOaLi572KmssLxaGisfcaocKen2riOFFT5NdU8uMpdEjsPVacino8lRI7xWPHKRqHN06SUyh3p6Q6wogL8iocl5V7qTUdEjtF4pqy71FTzaqpYt7nOiXEjnNIfNj3uaXEoaJxzRynyK7pcMJu2nbv3l1r167V5MmT1aNHD/Xo0UNPPPGE1q5dqxNPPDHczZXLzp071bhx4yLL/H6/GjRooJ07d5Z6nejoaCUkJBRZ3qRJkyLXefTRR/XPf/5TixYt0kUXXaSbb75ZL730Upn5TJ48WfHx8d5Pq1atJEnbtm2TJO80C5K0detW7d69W5K0efNm7d27V5K0ceNGpaamSpLWr1+v9PR0SdLatWuVmZkpSVq9erV3yomUlBTl5ORIkpKTkxUIBOQ4jpKTk+U4jgKBgJKTkyW55/JNSUmR5J7OYvXq1ZKkzMxMrV27VpJ7JPL69eslSampqdq4caMkae/evdq8ebMkaffu3dq6dWuNqyljV4IOptV2t7kjQTkZtSRJadvqKzcrxt3+lobKy46WJO3flKhgTpQkad+GRgrluQew71nXWE7QljGW9qxrLGMsOUFbe9a5+3Ioz699GxpJkoI5Udq/KVGSlJcdrdQtDSVJuVkxSttW370PMmopfUeCJOlgWm1l7HLjA/vrKnNPXHjjlOZOEOs3OkrPcOO160LKzHLj1WtDyj4od5zWhJST68bJKSEFgu6LuOSUkBxHCgTdWJJyct31JSn7oLsdScrMMlq7zo3TM4zWb3QntNQ0o42b3XjvfqPNW9149x6jrdvdeMcuox27zOFrKmHf27BOOpB/6uv1a6WD+TX9skbKy6/p5xQpmF/Tzynu72DQjSV3vV/WuPHBg+52JHe7G9a5cWaGtHmjG2ekSVvd3U2p+6Xt7u6mfXukndvdeM8u90dyl+3bo3LXVN0eT0dT09CYnqpruY+/4TG9FKso+WVreEwv+WUrVlEaHtNLklTXqqWhMT0lSQlWHZ0RfZIkKdGO16Bo97mpqZ2g/tGdJEktfYnqHdVRkpTka6weUe0kSR18zdXdnyRJ6uxvqc7+lpKk7v4kdfC5n+roEdVOST73cdw7qqNa+tzHbv/oTmpqJ4Q1Tgr53cbb9g7u75DfjSUpGC3tcPNSXqy0y81LubWl3a3d+GBdaa+bo7LjpP35nzzJSpBS8z9BktlQSst/Dk1v6P5I7rLM/Di1qXsdyd1GtjunaG9L9zYk9zZzax+2pkP3vTr+WF3f5Wx3bGLq6qoTzpQkNa6VoFEdBrnjUTdRF7d3P63TNq6Jzk/qI0nqFN9Cw1u749qtQRud1dId116J7TWoWVdJUr/GJ6hf4xMkSYOadVWvxPaSpLNanqRuDdpIkoa37qlO8S0kSecn9VHbOPc0Txe3H6CWdd3xG9VhkBrXSihXTYc+nmL2uPuSFailmH3u7dt5dRSz3x0/O7eeolPdXHwH4xWd7r5u8GXXV1S6m5f/QKKiMtwx82c1lj/LHbOojKbyH3BzjEpvIV+2+5wQnd5KvoPxbpzaRnZuPUlSzP52svPquPG+9rIC7mMoZk9HWSH3OSx2d6eImCMqet6rqLl8+1b3OpK7jYw0N9680b0NqeKen2riOFFTDahpt0+797v/sNu806+9ae5b1o3b/UrNcOP1W/1Kz3L/cbV2U5Qys9149cYoZee4ccr6KOXkuXHyL1G/vob9JerX17C/uK/hc/Ispax34+wcS6s3unFmtqW1m9w4PcvS+q3u6/zUDFsbt7vx3jQ7vHHaHat96e42N+2spdRMdzvrt9dS+gE3Xru1tjIPuvfBms11lJ3r1p2yqY5yA27804a6CoQsOcaNHSMFQpZ+2uA+b+cGbKVscp8PsnNtrdnsxpkHfVq7Nf+9zgG/1m93nydSM/3atNON96VHafNu91O1u1Oja86+R001qqa1q41y859z16QY7zl3TYrxnnPXpLjvQ3Nz3fWl/PeEP7ux+57QjTMzpE1uGUpPO+R1xBY3LtfriC1FX0ekp7nxpo2qkeMU6TUdjmXK296VFAgE1LlzZy1YsEBdunQp79VKdc899+jJJ58sc51Vq1Zp7ty5euutt7RmzZoilzVu3FiPPPKIxo0bV+x6M2bM0DXXXKPcgkdhvr59++rMM88s9XYfeughvfnmm9qyZUupOeXm5hbZbkZGhlq1aqXU1FQlJCR4HXbbtkuNQ6GQLMvyYtu2ZVlWWLHkdvcLxz6fz/0PTxix4zgyxnjx4XKP9JpW7n1SsiTLkhzHkmWZo4ol94i2wrFtm/z/hh95LCNZtrz4xMQ/lX+cgh/Iti2FQka2rfwxKH/sjo2KxD6flT8G5Y8dx62lIHZzLCP2DSv3vpcdnC8n5N5HluUeMWGXEZdcU/6RV6XExpHsQ2LHccfDi+VuszxxLd/5Efd4Opo5os/boxSSIyP3yNmg3G2VFBccUVtSbMtWSI6s/KNlQ3Lyj4ktOZYkR6bU2CdbRqbE2JHRd1fOKvc4Ldj0R3cHMJaUP0fIWLK8x3kFxXLnoIKjbH+N3fnKOO7tlzc+r/X/lXvfs28eoCjbp4ATco+oLSX2WT4FTcg9qtmy3diyZKtwbCloHNlW/pgZxzvKtiA2MnKMkd9yx8ONfXLk/BobR47cOGRCMnKPtA06bhx8+cuwHk9vr7pdsh33SFZjF4otyTaHxPlHwhbEkjv24caO5R6i7MXG/btIbEuWU2J8ZZcXq/0cUZHzXo6zoMLm8iJxSJL1a1zwnFRRz0+1fL+pUeNETTWgpszZpT6eQvlTWEFs579eLxIfg9d7jpF8dtHYyZ/mC2LFX1azxomaqCkCasrMnZ8/Rxzd+1z3tUPR2HFM/uuIcr7PLS0OmfzXEW5cJ/r8GjdOkVpTVlaWEhISlJ6erri4OJUmrHPaRkVFlbsbXB533XWXxowZU+Y67dq1U9OmTb1udoFgMKj9+/eXei7apk2bKi8vT2lpaUWOtt21a1eZ56/t16+fJk2apNzcXMXExJS4TkxMTImXFewwBb/Lin0+3zGLLcsKKy5PvjWpJqvQ8ee2bSokLmjYFo4t6+jigs/xFo7LPU75H831+axCy8ONVSx2x6D8sW3/ur3yxeHte3YJOYYbW1bpsVVCbBfZf8KNI+/xdDQ1FTRmyxObMuKQFxuF8j8f7raCS4tVZhwqdPulxeWt1SrYvQvNESryOK+gWAVxKfOSHV4c7lgG8j9nbsqIgyb/o0kycgpiY+SocGy8uKCmkCk0BoXiYJE4dNi4IJdw67MsS7Lzb8uS2xj1YlNCbLw5u6RxL3dslyd2So0tq+B5oPrOERU67zkVOZcXin2Hj4/u+amGjRM11ZCaVGLsK098DF7vFbz8LRwXyzGM94GRM07URE3VvaaKeZ9rlRCH/z63lNhXNC7r9Vu4cfUZp8isqWAsDyfsLyK75ZZb9OSTT+r111+X3x/21Yto1KiRGjVqdNj1BgwYoLS0NC1fvly9evWSJH388cdyHEf9+vUr8Tq9evVSVFSUPvroI1100UWSpDVr1mjz5s1lfmHaihUrVL9+/VIbtgAAAAAAAABwLIXddf3mm2/00Ucf6cMPP1T37t1Vp06dIpfPnTu3wpIr0KVLFw0fPlxjx47V1KlTFQgEdOutt+ryyy9X8+bu+fu2bdumIUOGaNq0aerbt6/i4+N13XXXacKECWrQoIHi4uJ02223acCAAerfv78k6d1339WuXbvUv39/xcbGatGiRXr88cf1hz/8ocJrAAAAAAAAAIDyCLtpm5CQ4B25ejxNnz5dt956q4YMGSLbtnXRRRfpxRdf9C4PBAJas2aNd2JfSXruuee8dXNzczVs2DBNmTLFuzwqKkqvvPKK7rzzThlj1KFDBz377LMaO3bsca0NAAAAAAAAAAqE3bR98803j0Ueh9WgQQPNmDGj1MuTkpJ06HeqxcbG6pVXXtErr7xS4nWGDx+u4cOHV2ieAAAAAIAjED+qsjMAAKDKsA+/CgAAAAAAAADgeCnXkbY9e/Ys9zebfffdd0eVEAAAAAAAAADUZOVq2o4cOdKLc3JyNGXKFHXt2lUDBgyQJH311VdauXKlbr755mOSJAAAAAAAAADUFOVq2j788MNefP311+v222/XpEmTiq2zZcuWis0OAAAAAAAAAGqYsM9p+8477+iqq64qtvzKK6/Uv/71rwpJCgAAAAAAAABqqrCbtrVq1dKXX35ZbPmXX36p2NjYCkkKAAAAAAAAAGqqcp0eobDx48dr3Lhx+u6779S3b19J0rJly/TGG2/owQcfrPAEAQAAAAAAAKAmCbtpe88996hdu3Z64YUX9Pbbb0uSunTpojfffFOXXnpphScI4BiIOreyMwAAAAAAAEApwm7aStKll15KgxYAAAAAAAAAjoGwz2kLAAAAAAAAADh2aNoCAAAAAAAAQBVC0xYAAAAAAAAAqhCatgAAAAAAAABQhYTdtP3kk0+ORR4AAAAAAAAAAB1B03b48OFq3769/vznP2vLli3HIicAAAAAAAAAqLHCbtpu27ZNt956q+bMmaN27dpp2LBh+uc//6m8vLxjkR8AAAAAAAAA1ChhN20TExN15513asWKFVq2bJlOOOEE3XzzzWrevLluv/12/fDDD8ciTwAAAAAAAACoEY7qi8hOOeUU3Xvvvbr11luVlZWlN954Q7169dKgQYO0cuXKisoRAAAAAAAAAGoM/5FcKRAI6D//+Y/eeOMNLVq0SL1799bLL7+sUaNGac+ePXrggQd0ySWXKCUlpaLzBQAAAAAAAI5Ybf/5lZ0CcFhhN21vu+02zZw5U8YY/f73v9dTTz2lbt26eZfXqVNHTz/9tJo3b16hiQJAOOpE/bayUwAAAAAAADgiYTdtU1JS9NJLL+l3v/udYmJiSlwnMTFRn3zyyVEnBwAAABxr/KMPAAAAVU3YTduPPvro8Bv1+zV48OAjSggAAAAAAAAAarKwv4hs8uTJeuONN4otf+ONN/Tkk09WSFIAAAAAAAAAUFOF3bR99dVX1blz52LLTzzxRE2dOrVCkgIAAAAAAACAmirspu3OnTvVrFmzYssbNWqkHTt2VEhSAAAAAAAAAFBThd20bdWqlb788stiy7/88ks1b968QpICAAAAAAAAgJoq7C8iGzt2rMaPH69AIKCzzjpLkvvlZHfffbfuuuuuCk8QAAAAAAAAAGqSsJu2f/zjH7Vv3z7dfPPNysvLkyTFxsbqT3/6k+69994KTxAAAAAAAAAAapKwm7aWZenJJ5/Ugw8+qFWrVqlWrVrq2LGjYmJijkV+AAAAAAAAAFCjhN20LVC3bl316dOnInMBAAAAAAAAgBrviJq23377rf75z39q8+bN3ikSCsydO7dCEgMAAAAAAACAmsgO9wqzZs3SqaeeqlWrVunf//63AoGAVq5cqY8//ljx8fHHIkcAAAAAAAAAqDHCbto+/vjjeu655/Tuu+8qOjpaL7zwglavXq1LL71UrVu3PhY5AgAAAAAAAECNEXbTdt26dTrvvPMkSdHR0Tpw4IAsy9Kdd96p1157rcITBAAAAAAAAICaJOymbf369ZWZmSlJatGihX766SdJUlpamrKzsys2OwAAAAAAAACoYcL+IrLTTz9dixYtUvfu3XXJJZfojjvu0Mcff6xFixZpyJAhxyJHAAAAAAAAAKgxwm7avvzyy8rJyZEk3X///YqKitKSJUt00UUX6YEHHqjwBAEAAAAAAACgJgnr9AjBYFALFiyQz+dzr2zbuueeezR//nw988wzql+//jFJUpL279+v0aNHKy4uTgkJCbruuuuUlZVV5nVee+01nXHGGYqLi5NlWUpLS6uQ7QIAAAAAAADAsRJW09bv9+umm27yjrQ9nkaPHq2VK1dq0aJFWrBggT777DPdcMMNZV4nOztbw4cP13333Veh2wUAAAAAAACAYyXs0yP07dtXK1asUJs2bY5FPiVatWqVFi5cqG+++Ua9e/eWJL300ksaMWKEnn76aTVv3rzE640fP16StHjx4grdbm5urnJzc72/MzIyJEmO4xT5bdt2qXEoFJJlWV5s27YsyworLritwrHP55MxJqzYcRwZY7z4cLlHek2dE/5YLWti36OmSKvJL1shOTKS/LIVlLutkmJLkq+U2M7fjiVLtiyF5MiWJauUWJIcmVJjn2wZmRJjN1uVe5yMyX/wGkuyjBdbtnEvq6hYlizLyBi3jl9jI8uSjOPefnnjcPe9KNungBOSJclfSuyzfAqakGxZsi3bjS1LtgrHloLGkW3lj5lx5LPc2yiIjYwcY+S33PFwY58cOb/GxpEjNw6ZkEx+jkHHjcN9PMmxJduRjCRTOLYk2xwSW+7fBbHkjn24sWNJVuHYuH8XiW3JckqMTf7OV53niEic96iJmqiJmqiJmqiJmqgp8msqeC1+OGEdaStJN998syZMmKCXX35ZS5cu1Y8//ljk51hYunSpEhISvMaqJA0dOlS2bWvZsmXHfbuTJ09WfHy899OqVStJ0rZt2yRJO3bs0I4dOyRJW7du1e7duyVJmzdv1t69eyVJGzduVGpqqiRp/fr1Sk9PlyStXbtWmZmZkqTVq1crOztbkpSSkuId4ZycnKxAICDHcZScnCzHcRQIBJScnCxJysnJUUpKiiT3aOPVq1dLkjIzM7V27VpJUnp6utavXy9JSk1N1caNGyVJe/fu1ebNmyVJu3fv1tatW6mJmqiJmiqlpqExPVXXqiVJGh7TS7GKkl+2hsf0kl+2YhWl4TG9JEl1rVoaGtNTkpRg1dEZ0SdJkhLteA2KPlGS1NROUP/oTpKklr5E9Y7qKElK8jVWj6h2kqQOvubq7k+SJHX2t1Rnf0tJUnd/kjr43H/k9YhqpyRfY0lS76iOaulLlCT1j+6kpnZCWOOkkN9txG3v4P4O+d1YkoLR0g43L+XFSrvcvJRbW9rd2o0P1pX2ujkqO07an//PxqwEKbWpG2c2lNLcfJXe0P2R3GWZ+XFqU/c6kruN7Dg33tvSvQ3Jvc3c2oet6dB9r44/Vtd3Odsdm5i6uuqEMyVJjWslaFSHQe541E3Uxe0HSJLaxjXR+Ul9JEmd4ltoeGt3XLs1aKOzWrrj2iuxvQY16ypJ6tf4BPVrfIIkaVCzruqV2F6SdFbLk9StgfsP5uGte6pTfAtJ0vlJfdQ2rokk6eL2A9Syrjt+ozoMUuNaCeWq6dDHU8wed1+yArUUs8+9fTuvjmL2u+Nn59ZTdKqbi+9gvKLT3dcNvuz6ikp38/IfSFRUhjtm/qzG8me5YxaV0VT+A26OUekt5Mt2T0UVnd5KvoPxbpzaRnZuPUlSzP52svPquPG+9rIC7mMoZk9HWaEYSVLs7k4RMUdE4rxHTdRETdRETdRETdRETTWnpsOxTHnbu/kKOtlFNmJZMsZ4HeOK9vjjj+utt97SmjVriixv3LixHnnkEY0bN67M6y9evFhnnnmmUlNTlZCQcNTbLelI21atWnnb578L1ERN1ERNR19Tn7dHVcsjbb+7cla5x2nBJvfI/up2pO15rf+v3PueffOAanekbfDlL8N6PL296vZqd6TtlV1erPZzRCTOe9RETdRETdRETdRETdQU+TVlZWUpISFB6enpiovLP2CmBGGfHmHDhg3hXqVU99xzj5588sky11m1alWF3V5FiYmJUUxMTLHlBTtMwe+yYp/Pd8xiy7LCisuTLzVREzVR0/GuqaAxW57YlBGHvNgolH/6ArcVXFqsMuNQodsvLS5vrVZ+D85r2BaKLasCYxXEv95OkdgOLw53LAOO+w9dU0YcNG7sNlrzY2PkqHBsvLigppApNAaF4mCROHTYuCCXcOuzLMtt0kr5TdTCsSkhzm+oSiWOe7ljuzyxU2ps5e981XmOiMR5j5qoiZqoiZqoiZqoiZoiv6aC1+KHE3bTtiLPZXvXXXdpzJgxZa7Trl07NW3a1DsEuUAwGNT+/fvVtGnTI779Y7VdAAAAAAAAADhSYTdtp02bVublV111Vbm31ahRIzVq1Oiw6w0YMEBpaWlavny5evXqJUn6+OOP5TiO+vXrV+7bO17bBQAAAAAAAIAjFXbT9o477ijydyAQUHZ2tqKjo1W7du2wmrbl1aVLFw0fPlxjx47V1KlTFQgEdOutt+ryyy9X8+bul65s27ZNQ4YM0bRp09S3b19J0s6dO7Vz50798ssvktyTFterV0+tW7dWgwYNyrVdAAAAAAAAADie7MOvUlRqamqRn6ysLK1Zs0YDBw7UzJkzj0WOkqTp06erc+fOGjJkiEaMGKGBAwfqtdde8y4PBAJas2aN921skjR16lT17NlTY8eOlSSdfvrp6tmzp+bPn1/u7QIAAAAAAADA8RT2kbYl6dixo5544gldeeWVWr16dUVsspgGDRpoxowZpV6elJQkY0yRZRMnTtTEiROParsAAAAAAAAAcDyFfaRtafx+v7Zv315RmwMAAAAAAACAGinsI20Ln1pAkowx2rFjh15++WWddtppFZYYAAAAAAAAANREYTdtR44cWeRvy7LUqFEjnXXWWXrmmWcqKi8AAAAAAAAAqJHCbto6jnMs8gAAAAAAAAAAqALPaQsAAAAAAAAAOHphN20vuugiPfnkk8WWP/XUU7rkkksqJCkAAAAAAAAAqKnCbtp+9tlnGjFiRLHl5557rj777LMKSQoAAAAAAAAAaqqwm7ZZWVmKjo4utjwqKkoZGRkVkhQAAAAAAAAA1FRhN227d++u2bNnF1s+a9Ysde3atUKSAgAAAAAAAICayh/uFR588EH97ne/07p163TWWWdJkj766CPNnDlT77zzToUnCABApLqg7bOVnQIAAAAAoAoKu2l7/vnna968eXr88cc1Z84c1apVSyeddJL+97//afDgwcciRwAAAAAAAACoMcJu2krSeeedp/POO6+icwEAAAAAAACAGi/sc9p+8803WrZsWbHly5Yt07ffflshSQEAAAAAAABATRV20/aWW27Rli1bii3ftm2bbrnllgpJCgAAAAAAAABqqrCbtikpKTrllFOKLe/Zs6dSUlIqJCkAAAAAAAAAqKnCbtrGxMRo165dxZbv2LFDfv8RnSIXAAAAAAAAAJAv7KbtOeeco3vvvVfp6enesrS0NN133306++yzKzQ5AAAAAAAAAKhpwj409umnn9bpp5+uNm3aqGfPnpKkFStWqEmTJvrHP/5R4QkCAABUZb/v/EplpwAAAAAgwoTdtG3RooV+/PFHTZ8+XT/88INq1aqla665RqNGjVJUVNSxyBEAAAAAAAAAaowjOgltnTp1dMMNN1R0LgAAAAAAAABQ4x3xN4elpKRo8+bNysvLK7L8ggsuOOqkAAAAAAAAAKCmCrtpu379el144YVKTk6WZVkyxkiSLMuSJIVCoYrNEAAAAAAAAABqEDvcK9xxxx1q27atdu/erdq1a2vlypX67LPP1Lt3by1evPgYpAgAAAAAAAAANUfYR9ouXbpUH3/8sRITE2Xbtmzb1sCBAzV58mTdfvvt+v77749FngAAAAAAAABQI4R9pG0oFFK9evUkSYmJidq+fbskqU2bNlqzZk3FZgcAAAAAAAAANUzYR9p269ZNP/zwg9q2bat+/frpqaeeUnR0tF577TW1a9fuWOQIAAAAAAAAADVG2E3bBx54QAcOHJAkPfroo/rNb36jQYMGqWHDhpo9e3aFJwgAAAAAAAAANUnYTdthw4Z5cYcOHbR69Wrt379f9evXl2VZFZocAAAAAAAAANQ0YTdtS9KgQYOK2AwAAAAAAAAA1HhhfxEZAAAAAAAAAODYoWkLAAAAAAAAAFUITVsAAAAAAAAAqEJo2gIAAAAAAABAFULTFgAAAAAAAACqEJq2AAAAAAAAAFCF0LQFAAAAAAAAgCqk2jRt9+/fr9GjRysuLk4JCQm67rrrlJWVVeZ1XnvtNZ1xxhmKi4uTZVlKS0srtk5SUpIsyyry88QTTxyjKgAAAAAAAACgbNWmaTt69GitXLlSixYt0oIFC/TZZ5/phhtuKPM62dnZGj58uO67774y13v00Ue1Y8cO7+e2226ryNQBAAAAAAAAoNyqRdN21apVWrhwoV5//XX169dPAwcO1EsvvaRZs2Zp+/btpV5v/Pjxuueee9S/f/8yt1+vXj01bdrU+6lTp06Z6+fm5iojI6PIjyQ5juP9PlwcCoWKxMaYsGNjTLFYUtix4zhF4nDqoCZqoiZqOlY1+WXLkrxYZcRWGbHPiy0vtsuIC261tNgnu9TYyo9r0jgdriZJirJ93niUFvstn3dfe7F1aGx7sS8/9ll2kdi23DHwF4l9RWP9GhfsY1H2r3FNHCdqoiZqoiZqoiZqoiZqoiZqOr41HU61aNouXbpUCQkJ6t27t7ds6NChsm1by5YtO+rtP/HEE2rYsKF69uyp//u//1MwGCxz/cmTJys+Pt77adWqlSRp27ZtkuQdsStJW7du1e7duyVJmzdv1t69eyVJGzduVGpqqiRp/fr1Sk9PlyStXbtWmZmZkqTVq1crOztbkpSSkqKcnBxJUnJysgKBgBzHUXJyshzHUSAQUHJysiQpJydHKSkpktyjjVevXi1JyszM1Nq1ayVJ6enpWr9+vSQpNTVVGzdulCTt3btXmzdvliTt3r1bW7dupSZqoiZqqpSahsb0VF2rliRpeEwvxSpKftkaHtNLftmKVZSGx/SSJNW1amloTE9JUoJVR2dEnyRJSrTjNSj6RElSUztB/aM7SZJa+hLVO6qjJCnJ11g9otpJkjr4mqu7P0mS1NnfUp39LSVJ3f1J6uBrLknqEdVOSb7GkqTeUR3V0pcoSeof3UlN7YQaN06Hq6mOP1bXdznbHZuYurrqhDMlSY1rJWhUh0HueNRN1MXtB0iS2sY10flJfSRJneJbaHhrd1y7NWijs1q649orsb0GNesqSerX+AT1a3yCJGlQs67qldheknRWy5PUrUEbSdLw1j3VKb6FJOn8pD5qG9dEknRx+wFqWdcdv1EdBqlxrYQaO07URE3URE3URE3URE3URE3UdHxrOhzLlLe9W4kef/xxvfXWW1qzZk2R5Y0bN9YjjzyicePGlXn9xYsX68wzz1RqaqoSEhKKXPbss8/qlFNOUYMGDbRkyRLde++9uuaaa/Tss8+Wur3c3Fzl5uZ6f2dkZKhVq1be9gs67LZtlxqHQiFZluXFtm3LsqywYsnt7heOfT6fjDFhxY7jyBjjxYfLnZqoiZqo6XjU1OftUQrJkZF75GxQ7rZKiguOqC0ptmUrJEdW/tGyITn5x8SWHEuSI1Nq7JMtI1Ni7Mjouytn1ahxOlxN9s0DFGX7FHBC7hG1pcQ+y6egCblHNVu2G1uWbBWOLQWNI9vKHzPjeEfZFsRGRo4x8lvueLixT46cX2PjyJEbh0xIRu6RtkHHjYMvf1njxomaqImaqImaqImaqImaqImajk9NWVlZSkhIUHp6uuLi4lSaSm3a3nPPPXryySfLXGfVqlWaO3fuMWvaHuqNN97QjTfeqKysLMXExJSrjoyMDMXHxx/2zgYAlF/Pty+r7BSOyPdXzq7sFKoUa1zZpyiqisxfvqrsFAAAAABEqPL2Ef3HMadi7rrrLo0ZM6bMddq1a6emTZt6hyAXCAaD2r9/v5o2bVqhOfXr10/BYFAbN25Up06dKnTbAAAAAAAAAHA4ldq0bdSokRo1anTY9QYMGKC0tDQtX75cvXq55y/8+OOP5TiO+vXrV6E5rVixQrZtq3HjxhW6XQAAAAAAAAAoj0pt2pZXly5dNHz4cI0dO1ZTp05VIBDQrbfeqssvv1zNm7tfDLNt2zYNGTJE06ZNU9++fSVJO3fu1M6dO/XLL79Ick9aXK9ePbVu3VoNGjTQ0qVLtWzZMp155pmqV6+eli5dqjvvvFNXXnml6tevX2n1AgAAAAAAAKi57MpOoLymT5+uzp07a8iQIRoxYoQGDhyo1157zbs8EAhozZo13rexSdLUqVPVs2dPjR07VpJ0+umnq2fPnpo/f74kKSYmRrNmzdLgwYN14okn6rHHHtOdd95ZZLsAAAAAAAAAcDxV6heRRQq+iAwAKh5fRBYZ+CIyAAAAAPhVefuI1eZIWwAAAAAAAACoCWjaAgAAAAAAAEAVQtMWAAAAAAAAAKoQmrYAAAAAAAAAUIXQtAUAAAAAAACAKoSmLQAAAAAAAABUITRtAQAAAAAAAKAKoWkLAAAAAAAAAFUITVsAAAAAAAAAqEJo2gIAAAAAAABAFULTFgAAAAAAAACqEH9lJwAAQEm+v3J2ZacAAAAAAECl4EhbAAAAAAAAAKhCaNoCAAAAAAAAQBVC0xYAAAAAAAAAqhCatgAAAAAAAABQhdC0BQAAAAAAAIAqhKYtAAAAAAAAAFQhNG0BAAAAAAAAoAqhaQsAAAAAAAAAVQhNWwAAAAAAAACoQmjaAgAAAAAAAEAVQtMWAAAAAAAAAKoQmrYAAAAAAAAAUIXQtAUAAAAAAACAKoSmLQAAAAAAAABUITRtAQAAAAAAAKAKoWkLAAAAAAAAAFUITVsAAAAAAAAAqEJo2gIAAAAAAABAFULTFgAAAAAAAACqEJq2AAAAAAAAAFCF0LQFAAAAAAAAgCqEpi0AAAAAAAAAVCE0bQEAAAAAAACgCqFpCwAAAAAAAABVSLVp2u7fv1+jR49WXFycEhISdN111ykrK6vM9W+77TZ16tRJtWrVUuvWrXX77bcrPT29yHqbN2/Weeedp9q1a6tx48b64x//qGAweKzLAQAAAAAAAIAS+Ss7gfIaPXq0duzYoUWLFikQCOiaa67RDTfcoBkzZpS4/vbt27V9+3Y9/fTT6tq1qzZt2qSbbrpJ27dv15w5cyRJoVBI5513npo2baolS5Zox44duuqqqxQVFaXHH3/8eJYHAAAAAAAAAJIkyxhjKjuJw1m1apW6du2qb775Rr1795YkLVy4UCNGjNDWrVvVvHnzcm3nnXfe0ZVXXqkDBw7I7/frv//9r37zm99o+/btatKkiSRp6tSp+tOf/qQ9e/YoOjq6XNvNyMhQfHy80tPTFRcXd2RFAgAQgaxx/Ss7hbCZv3xV2SkAAAAAiFDl7SNWi9MjLF26VAkJCV7DVpKGDh0q27a1bNmycm+n4M7w+/3edrt37+41bCVp2LBhysjI0MqVK0vdTm5urjIyMor8SJLjON7vw8WhUKhIXNA7Dyc2xhSLJYUdO45TJA6nDmqiJmqiJmqiprJqkqQo2ydJssqI/ZYb27J+ja1DY9uLffmxz7KLxLZlSZL8RWJf0Vi/xm7k5lIQ18RxoiZqoiZqoiZqoiZqoiZqoqbjW9PhVIum7c6dO9W4ceMiy/x+vxo0aKCdO3eWaxt79+7VpEmTdMMNNxTZbuGGrSTv77K2O3nyZMXHx3s/rVq1kiRt27ZNkrRjxw7t2LFDkrR161bt3r1bknv+3L1790qSNm7cqNTUVEnS+vXrvXPtrl27VpmZmZKk1atXKzs7W5KUkpKinJwcSVJycrICgYAcx1FycrIcx1EgEFBycrIkKScnRykpKZKk7OxsrV69WpKUmZmptWvXSnIb2OvXr5ckpaamauPGjd79tHnzZknS7t27tXXrVmqiJmqiJmqipiOuqY4/Vtd3OVuSlBBTV1edcKYkqXGtBI3qMEiS1LJuoi5uP0CS1Dauic5P6iNJ6hTfQsNb95QkdWvQRme1PEmS1CuxvQY16ypJ6tf4BPVrfIIkaVCzruqV2F6SdFbLk9StQRtJ0vDWPdUpvoUk6fykPmob5z7XX9x+gFrWTZQkjeowSI1rJdTYcaImaqImaqImaqImaqImaqKm41vT4VTq6RHuuecePfnkk2Wus2rVKs2dO1dvvfWW1qxZU+Syxo0b65FHHtG4cePK3EZGRobOPvtsNWjQQPPnz1dUVJQk6YYbbtCmTZv0wQcfeOtmZ2erTp06ev/993XuueeWuL3c3Fzl5uYW2X6rVq2UmpqqhIQEr8Nu23apcSgUkmVZXmzbtizLCiuW3O5+4djn88kYE1bsOI6MMV58uNypiZqoiZqoiZrKW5N98wBF2T4FnJB7RG0psc/yKWhCsmXJtmw3tizZKhxbChpHtmXJkqWQcbyjbAtiIyPHGPktW44X++TI+TU2jhy5cciEZOQeaRt03Dj48pc1bpyoiZqoiZqoiZqoiZqoiZqo6fjUlJWVpYSEhMOeHqFSm7Z79uzRvn37ylynXbt2evvtt3XXXXd5nWtJCgaDio2N1TvvvKMLL7yw1OtnZmZq2LBhql27thYsWKDY2Fjvsoceekjz58/XihUrvGUbNmxQu3bt9N1336lnz57lqoNz2gIAUDLOaQsAAAAAvypvH9F/HHMqplGjRmrUqNFh1xswYIDS0tK0fPly9erVS5L08ccfy3Ec9evXr9TrZWRkaNiwYYqJidH8+fOLNGwLtvvYY49p9+7d3ukXFi1apLi4OHXt2vUoKgMAAAAAAACAI1MtzmnbpUsXDR8+XGPHjtXXX3+tL7/8Urfeeqsuv/xyNW/eXJJ7PtnOnTvr66+/luQ2bM855xwdOHBAf/vb35SRkaGdO3dq586dCoXcEw2fc8456tq1q37/+9/rhx9+0AcffKAHHnhAt9xyi2JiYiqtXgAAAAAAAAA1V6UeaRuO6dOn69Zbb9WQIUNk27Yuuugivfjii97lgUBAa9as8U7s+91332nZsmWSpA4dOhTZ1oYNG5SUlCSfz6cFCxZo3LhxGjBggOrUqaOrr75ajz766PErDACACMapBgAAAAAgfJV6TttIwTltAQAAAAAAABxOefuI1eL0CAAAAAAAAABQU9C0BQAAAAAAAIAqhKYtAAAAAAAAAFQhNG0BAAAAAAAAoAqhaQsAAAAAAAAAVQhNWwAAAAAAAACoQmjaAgAAAAAAAEAVQtMWAAAAAAAAAKoQmrYAAAAAAAAAUIXQtAUAAAAAAACAKsRf2QlEAmOMJCkjI6OSMwEAAAAAAABQVRX0Dwv6iaWhaVsBMjMzJUmtWrWq5EwAAAAAAAAAVHWZmZmKj48v9XLLHK6ti8NyHEfbt29XvXr1ZFlWZadTLWVkZKhVq1basmWL4uLiKjudY4IaI0Ok1xjp9UnUGCkivcZIr0+ixkgR6TVGen0SNUaKSK8x0uuTqDFSRHqNkV7f8WKMUWZmppo3by7bLv3MtRxpWwFs21bLli0rO42IEBcXF/EPfGqMDJFeY6TXJ1FjpIj0GiO9PokaI0Wk1xjp9UnUGCkivcZIr0+ixkgR6TVGen3HQ1lH2Bbgi8gAAAAAAAAAoAqhaQsAAAAAAAAAVQhNW1QJMTExevjhhxUTE1PZqRwz1BgZIr3GSK9PosZIEek1Rnp9EjVGikivMdLrk6gxUkR6jZFen0SNkSLSa4z0+qoavogMAAAAAAAAAKoQjrQFAAAAAAAAgCqEpi0AAAAAAAAAVCE0bQEAAAAAAACgCqFpCwAAAAAAAABVCE1bAAAAAAAAAKhCaNoCAAAAAAAAQBVC0xbHlDGmslMAaryCxyGPR1QHjuNUdgoAagDmGgDHC/MNqjLeI1ZtNG1xTGRmZkqSLMuq5EyOrZo0wUXqi42aMIYFj0PLshjHCBGp9TqOI9su+tIkkmqNpFrKIxLnm5o2hpFab6TPNVLk1VOWSJxrpJo1hlLk1hvp800k1VIekTjfFIzh/Pnz9eGHH9a4Ma3q/JWdACLPvn37dO+99+r888/XoEGDlJCQIMmdDCKtiWtZlkKhkHw+X2WnUuE2bdqkH3/8Uenp6Ro2bJgaNWpU2SkdE5E8hpK0cuVKffXVV9q1a5fGjBmj5s2bV3ZKx0RBQ/rQF8WRYu/evVq7dq3y8vLUo0cPxcfHV3ZKFaLgsbdq1SotXrxYH3/8sZo1a6YLL7xQHTt2VMuWLSPqeSPS55ua8LzBXFM91bS5Rors+aYmzDUS8011VdPmm0iea6TIn2+CwaD8fr+++eYb3XnnnfrTn/6k7Oxs1alTp7JTQz7L0EZHBRs9erRmzpypbt26acSIERo5cqROOeUURUdHS6rezduCJ6QPP/xQa9as0W233VbssuqsoIZ58+bp2Wef1ffff68uXbooMTFRr7zyitq2bVvZKR61wjUuX75ckyZNKnZZdVdQx5w5c/TUU09px44datasmbKysvTGG2+of//+1fpxKP1a45dffqmtW7fqsssu8y6LlDc4BTW+9957evnll/XBBx+od+/e6tWrl5566inVq1evslOsMCeeeKJq1aql9u3ba9WqVVq3bp1GjBihq6++WkOGDFGtWrWq5T4b6c8ZUuQ/bzDXMNdUF5E+30T6XCMx3zDfVA+RPtdINWO+OVSvXr00ePBgPfvss5Kqd88m4higAq1du9acdNJJZsaMGeaJJ54wbdu2Nf369TPPP/+8WbVqlbfezp07zT//+c9KzPToNG/e3CQkJJizzz7bLFiwwFvuOI4JhUKVmFnFaNKkiXn22WdNSkqK+eCDD0z37t3N/fffX9lpVaimTZsay7JMhw4dzD/+8Q9veSgUiogxNMaYxo0bm1deecVs3rzZpKSkmKFDh5obbrih2HqO41RCdhWjZcuWpl27dmbs2LHmiy++8JY7jlOkrupcY7NmzcykSZPMV199ZV5//XXTokUL89JLL1V2WketYEz+/Oc/m549e5rc3FzvskWLFpl+/fp5+3B1F+nPGcZE/vMGc031VZPmGmMif76J9LnGGOab6qwmzTeRPtcYE9nzzccff2wCgYAxxpjvv//edOrUqUi/psC6devM3LlzzcGDB493ishH0xYVauXKleb22283S5YsMcYYs337dnP99deb5s2bm9/85jdm5syZZseOHeamm24yAwcOrORsw1PwJPzaa6+Zk08+2TzzzDPmsssuM506dTKjR482ycnJRdbfunVrtXrCKqjviSeeMKeeemqRF4JvvfWW6dWrl8nMzPSWZWdnH/ccK0pBPW+//ba5+eabTcOGDc1pp51mli1bVmS9n3/+2WzevLmSsjwyBeP29NNPm1NPPdUEg0Hvsg8++MC0b9/ebN++3VuWlZV13HM8WgU1vvzyy6ZLly7mD3/4gxk6dKg57bTTzH333WfWrVtXZP39+/dXRppHpfCL/gEDBhS57IknnjDnnHNOkfklJyfnuOZXURzHMePGjTOXXXaZcRzHBIPBInU9/vjjxrIs8/zzz3vrVxeR/pxhTOQ/bzDXMNdUF5E+30T6XGMM8w3zTfUQ6XONMTVjvpk8ebJp2rSp9/eBAwdMq1atzLRp04wxpsj7x5SUFHP22Web1atXH/c84ar+n7FAlXLCCSfoD3/4g/r06SNJatasmf7617/qX//6l3Jzc3Xvvffqhhtu0KuvvqqnnnqqkrMNT8H5ejZv3qwePXpo7Nixmjx5ssaNG6edO3fq0ksv1f3336/09HRJUv/+/TV9+vRKzrr8LMtSXl6e1q1bpwEDBigUCnknWj/99NO1ZcsWJScne+ufc845ev/99ysr3SPmOI7S09PVp08fjRgxQpMnT9Ybb7yhhIQEDR06VGPGjNG+ffskSUOGDNHChQsrOePwWJalQCCgrVu3qmvXrgoGg5Lcuvv27atQKKRly5Z565988smaPXt2ZaV7RCzLUjAY1M8//6x+/fpp0qRJeuKJJ9S7d299+umnuummmzRlyhQdOHBAktStWze98847lZx1eCzLUk5OjlauXKnhw4fLcRyFQiFJ7n757bffasOGDd76V111lT7//PPKSveIWZalXr166euvv1Z6erp8Pp8sy1Jubq4k6d5779Utt9yi9957z1u/uoj05wwp8p83mGuYa6qLSJ9vIn2ukZhvmG+qh0ifa6TIn2+MMZo/f74eeughSdJTTz2lxYsXq1OnTvrXv/7l7bMF/vKXvygjI0OdOnWqrJRR2V1jRI6C/0IV/s9MKBQq9t8py7LMtddee9zzqyihUMgsXbrU+9txHPPVV1+Z+++/3/Tu3dv079/fnH/++aZBgwaVmOWRCQaDZsaMGd5/2QobNGiQefjhh40xxkyZMsXUr1//OGdXcRzHMd9//32RZZs2bTKvvfaaOeWUU0zTpk3NGWecUS3HsMDHH39spk+fXmz57373OzNu3DhjjDF/+ctfTEJCwvFOrcKkp6ebTz/91Ps7FAqZefPmmSuvvNL069fPXH755eaKK66otuN48OBB8+yzz5q///3vxS7r2rWrmTJlijHGmFdeeaVaPx63b99uunfvbjp06GAWL17sLS94Lvnvf/9runTpYjZs2FBJGR6dSH7OMKZmPG8w1zDXVBeRPN/UhLnGGOYb5pvqIZLnGmMid75xHMfk5eWZSy65xHTr1s28//77xrIs8/PPP5uvv/7aNGrUyLRv39785S9/MTNmzDDjx483DRo0MN98801lp16j0bRFhShozP7000/miSeeKLa84Anq3//+t4mKijKpqanHPcejdehHVxzH8c4DY4z7MfP333/fjBo1yliWZebMmWOMMUXWqcoK6iv8EY/CH2d58cUXzbBhw0wgEDCNGjUyb775pjGm+tRnzOHHMBAImOTkZHP77bcby7LM3LlzveXVRUGNhXMuPI4zZswwvXr1Mjk5OaZRo0bei+bqWGNhhfPfv3+/mTp1qhk6dKixLMs7f3Z1qrFAenq6Fxeu+/777zdXXXWVycvLM4mJidXy8WjMrzWtX7/eXHzxxSYpKclcd9113puYvXv3mj/+8Y+mZ8+elZjlkYn05wxjIv95g7mGuaa6iPT5JtLnGmOYb4xhvqkOIn2uMaZmzDcrVqwwl1xyialdu7bp0KGDOXDggDHGPYjplltuMXXr1jUnnHCCGT58eImNaxxfNG1RoZ5//nljWZaZPXt2scvy8vLMqFGjzIQJEyohs6NXMFlPnz7drF27tshlhZ/Afve735l+/fod19wqQkF9s2fPNj///HOxy7/44gvTq1cvc8EFF5ju3bsf7/QqRME4vfrqq+bbb78tdb1LLrnE9OrV63ilVaEKavzrX/9arMZQKGRSUlLMSSedZAYPHmxOOumkykjxqBXsq++++67ZunWrt/zQLz644IILzGmnnXbc86sIBXUsXLiwxPMqz5071wwaNMhcddVV1fbxeKjVq1ebp556ypx66qkmJibG9O/f35x44ommffv25uuvvzbGFP0kR1UX6c8ZxkT+8wZzDXNNdRHp802kzzXGMN8Yw3xTHUT6XGNMzZhvjHHH0O/3mx49epj27dub//u///MuCwaDZs2aNZWYHQqjaYsKN2nSJNOtWzfvWyQLPxEV/gbN6ignJ8ecddZZ5uqrr/ZOjl/4hdRPP/1kGjRo4DXLqtOTsDEl11dQQ1pammnYsKGxLMusWLGiyGXVSSAQMBdccIE566yzvP94F67jl19+MZ07dzbLly8vdll1UVaNubm5pn379sayLPPjjz8Wuaw6OXDggBk4cKB56KGHvGWFXyx+++23xrKsavtYNKbkGgvmm61bt5ro6GhjWZZ3qo/qWOOh8vLyTEpKinn//ffN+PHjzV//+lfzww8/GGOKzrXVRaQ/ZxgT+c8bzDXMNdVFpM83kT7XGMN8w3xTPUT6XGNMzZhvli5dav7+97+bFStWmLvuust06NDB9OnTx/ukaYHq9EV5kYqmLSpMQUN227Zt5uKLLzb9+vUzO3furOSsKpbjOGbOnDmmSZMm5g9/+EOxy/Py8syiRYuMMdXzSfhw9T3++OPmkUceMcZUz/oKLF261Jx00knm6quvLnZZMBg0y5YtM8ZEbo3Tp083Tz31lDGm+tYYDAbNc889Z2JjY83TTz9d7PLc3FzvI1mRWKPjOGbs2LHm3nvvNcZU3xoLi8QXhZH+nGFM5D9vMNcw11QXkT7fRPpcYwzzDfNN9RDpc40xNWO+KWzfvn1m/vz5ZvTo0SYpKckMGTKk2JHUqDw0bXFUSpukMjIyzOmnn2769u1brf+TWJoFCxaY9u3bm8mTJ5tgMFjso0vV3aH1FdSWnp7una+nutf7zTffmFatWpkbbrjBO8dydfwvaVlKqtFxHHPw4EFv/Kr7OE6dOtWcdNJJ3rndQqFQsZqq+wvmQ2ss2E93795t8vLyjDHVdxwL8nYcp8Rxqu5jVyDSnzOMifznDeYa5prqItLnm0ifa4xhvmG+qR4ifa4xpmbMN4Vt2rTJvPnmm6ZPnz5FvkAPlYumLY7azp07zcUXX2xeffVVs2jRIvPdd98ZY9yP8IwcOdLcf//9lZxhxcvJyTGTJk0y7dq1Mx999JG3PFKehEurL9LMnDnTdO/evcg32EbKGBYoqcZIsmvXLnPNNdeYpKQkk5yc7C2PpHEsrcbqqDzjMmfOHHPZZZeZ7du3H4eMjo9If84wJvKfN5hrqpeaOtcYE/nzTaTPNeb/27vv+KjK9G3g1ySEYqRLCVUMLYIEQRBDUUEpi8CKCCiKi7D6A1QUUMC1oKCCoqLIAqL0tiAKSBSkSjUWpDcVCCKhE0kgpMxc7x+8MyYQFJKYk/M81/ef1Znj53Nf+8x9Z+Y5M+dQ88ZtbJ03ps8a0o55czGfzxe4vJ7kDdq0lWybP38+27dvz/Lly7NevXqsUKECIyIi2L9/f9aoUYMej4czZ850usws8Z85W7lyJVesWMHly5dnuDPk8OHDWbZsWUZHRztVYraYno/8443DF198wQ8//JAzZszg77//zoSEBJIXbthVoEABTpkyxbVvMmzI6H+tfvPNN9yzZw+3b9+e4ZvRffv2ZWRkZOCmDm5kesb0r71FixaxXbt27NevH999990MN11ZvHgxS5QowSVLljhRZrbYMFNNz2h6H5LmZ7Rh1pD29KKp+Ujze5E0P6MN88amXjQ5o7iXhyQhkkPWr1+PQoUKYfHixTh48CBSU1OxYsUK7N27F9dcc43T5WVZq1atsGnTJlSqVAn79u3DnXfeiXLlyqFixYpYsGABrrnmGsyfPx/FihVzutQsMT0fADz44IPYunUrgoODsWfPHjRp0gQFChRAs2bNMH/+fMTHx2PNmjUoW7as06VmmekZSaJp06bYuHEjoqKicPToUdx+++2oWbMmzp49i6VLl6JatWqYMGECChQo4HS5WWJyRq/Xi+DgYDz33HNYsmQJmjdvjq1bt2Lbtm1YtWoVateuDZLweDw4ceIErrvuOqdLzjIbZqrJGU3uQz+TM9o0awCzexEwP5/Jvehnckab5o3pvQjYkVHcR5u2ki379+9HQkICSCIyMjLDc6mpqQgJCUFycrLr/gBnZt++fTh8+DCOHz+O6OhopKamIiYmBnv37kXbtm3x+eefO11itpieDwDi4uKQlpaG2NhYrFq1ComJifjqq69w7NgxNG7cGHPnznW6xGwzOWNaWhpSUlKwf/9+bN68GadPn8aiRYsQEhKCffv2Yc+ePbjzzjuxYsUKp0vNMlMz+j+wHDx4EDfeeCOWLl2Kxo0b49FHH0V8fDw+/fRTnDp1Ctu2bUPTpk0RFBTkdMnZZsNMNTWjqX2YnqkZbZw1gLm96GdyPlN7MT1TM9o4b0zuRT8bMorL5Pp3e8X1/D8VmDRpEiMiIliiRAk2a9aMjz/++CXXXXLrjZ3SX0A+MTGRJ0+evOSYc+fOkSR/+OEH193IyvR85B+1pqWl8bfffmNcXNxlj927dy8TExMz/HduYEPGiy/uf+bMmUuOiYuL49mzZ7lq1SoeP36cpDLmJel/OjhhwgQ2adKEJLl06VIWLVqUe/bsIUkuX76cHTp0CNy80k1smKmmZzS9D0nzM9owa0h7etHUfKT5vUian9GGeWNTL5qcUdxPm7aSJUlJSQwNDeUHH3zAzz77jEOHDmXz5s3ZsGFDDhs2jKdOnXK6xBwxZMgQ3nHHHSxdujR79erFgwcPBp5z67VB0zM9H0k+9dRTrF+/PkNCQti+fXuuWbOGZ8+edbqsHGVDxtdff50dOnRgw4YN+fzzz2d4zpS7tpqc0T9PYmJiWL9+fZJko0aN+OyzzwaOmT17NmvVqsXz5887UmNOsGGmmp7R5D70MzmjLbOGNL8XTc9Hmt2LfiZntGXe2NCLNmQU99KmrVwV/9Bav349e/TokeEC3atXr2a/fv3YuHFjRkZGctWqVQ5VmT3+jAsWLGDRokX5n//8h++88w5r1arFa6+9lsOGDQuccXMj0/ORf2T8/PPPWaRIEY4ZM4bz589n06ZNGRISwv/7v//j7t27Xf1m0YaM/to/++wzFi9enD179uSAAQNYqVIllihRgpMnTw4c69Yz3iZn3LlzJ7/66qsMJxAOHz7Mm266iVWrVmWpUqUCjx8/fpzh4eEcOXIkSXdltWmmmprR5D70MzmjLbOGNL8XTc9Hmt2LfiZntGXe2NCLNmQUM2jTVq7ab7/9xk6dOvG2224L/NzaLyEhgXPmzOGDDz7II0eOOFRhzhg4cCDfeuutwL/Hx8fz7bffZsmSJVmtWjVOnTrVweqyz/R8JPniiy8G3ij5zZs3j5UqVWLZsmX58ssvMykpyaHqcoYNGf/v//6P77zzDkkyOTmZ27dv59NPP82CBQuyXr16XLt2rcMVZp+JGVu1asUqVapwxIgR3LJlS+BD3I8//sg2bdowLCyMffv25QsvvMBmzZoxKirK4Yqzx4aZanpGE/vwYiZmtG3WkOb3oun5SDN78WImZrRt3tjQizZkFHdz/9WwJdetXr0aP//8M7Zv345+/fph3759geeuvfZadOnSBePGjUOZMmUcrDJrfD4fAGDp0qUIDQ3F2bNnA88VLVoU/fv3x6ZNm9CoUSOMHTvWqTKzzPR8wIWbAgDA4sWLcebMGRw+fDjwGAB06tQJsbGx6NGjB7766isULFjQqVKzzIaM/tfqqlWrEBYWhuDgYABA/vz5UatWLQwfPhxfffUVihQpgn79+jlZapaZnnHevHm4//778cEHH6B///6YPHkyYmNjUbduXYwYMQK9e/fGDz/8gMWLF6NTp06YPn06gAt3YnYLG2aq6RlN70PA/Iw2zBrAnl40NR9gfi8C5me0Yd7Y1IsmZxSDOLtnLG61ceNGPv3006xfvz5bt27NcePGGXMNTa/Xy7vuuosej4c333wzT5w4EXgu/TVtEhISSDLDJSLcwPR8fl26dKHH42GFChX43XffBR5Pn9GfTRnzptTUVDZt2pQej4ctW7bM9Jjjx48HXsPKmHek/wngzp072bFjR5YvX55du3ZldHR0hm9/u/2nZzbMVNMzmtqH6Zma0aZZQ5rfi6bnI83txfRMzWjTvLGhF23IKGbQpq1kWVpaGhcsWMAHH3yQDRs2ZNeuXfnJJ584XVaOOHz4MKdMmcLKlSvzuuuu40cffRR4zoSLkZuez2/lypWMjIxkaGgoX3755cCdaU1icsa0tDR+8803HDZsGIsXL86qVaty4cKFgedNeK2anDEtLS1D/Z999hlvueUWhoeHs3///tywYYMxJ/tsmKkmZzS5D/1MzmjTrCHN7kXS/Hwm96KfyRltmjem9yJpR0ZxP23ayl/yn1Xctm0bJ02axJ49e/L999/n4cOHSV44Uzp27Fg2bdqUjz76qJOl5rhDhw6xf//+LFiwIOvXr8+vv/7a6ZJylOn5/N55553Am8apU6e6+g6ul2NyxnPnznHdunV86KGHWLRoUbZp04Y7duxwuqwcZXLGc+fOZXjjO3LkSIaHh7NRo0YcOnQojx075mB1OcuGmWpyRpP70M/kjDbNGtLsXiTNz2dyL/qZnNGmeWN6L5J2ZBT30qat/Cn/xdVPnjzJiIgI1qhRg23atGGRIkUYFhaW4WzU9u3beejQIadKzRFHjhzhrFmz+NVXX/Hrr78O/Azi+++/5/3330+Px8O5c+c6XGXWmZ6PJPfs2cMxY8ZwypQpnD59emDz8sSJE3zyySfp8Xgy3LXWjWzImJCQwKVLl3LLli3ct28fSTIxMZHz5s1jy5Yt6fF4GB0d7XCV2WNqRv9c2bx5M5966in+4x//4B133MH3338/cMzx48fZo0cP1qlTx9XfZLBhppqe0dQ+TM/UjDbNGtL8XjQ9H2luL6Znakab5o0NvWhDRjGHNm3lT/n/4HTt2pXt2rXj6dOnSV749u2QIUPo8Xg4YsQIByvMPv+QXrhwIevVq8fixYuzZMmSbNSoEfv06cMtW7aQvHAnySVLljhZapaYno/8I+P8+fN54403smzZsoyIiAhcc3np0qWBY916ht+mjF988QUbN27MokWLskCBArz99tv51ltv8ejRo/T5fNy5c2eGE0ZuYkNGv4oVK7JVq1b817/+xT59+rBYsWK88cYb+e233waOOXXqFMmM14nL62yaqaZmtKEPbcjoZ+qsIe3pRVPzkXb0og0Z/UydNzb1oskZxUzatJW/dOrUKdatWzfwzb30F+F+/vnnGRUVZcSFuUuXLs2hQ4fy6NGjjIuL4+uvv85GjRqxS5culxzrxrOnpucjyTJlyvCNN97g2bNnGR8fz5kzZ7JDhw686667Aicc3M6GjKVLl+bgwYO5Y8cO7tmzh48//jjDw8P55JNPBo7xv0bd+lo1NaO/1rlz5/KWW24J/G04d+4cv/32W7Zt25b33HNPhps9uJUNM9X0jKb2YXqmZrRp1pB29KLJ+UhzezE9UzPaNG9s6UXTM4pZtGkrV+Suu+7KcL1a/5nDlStXskaNGty6datTpeWITz75hBEREUxOTs7w+Pr165k/f36+++67zhSWQ0zPR5JLlixhRETEJRuXO3bsYPHixfn88887U1gOsiHj9OnTGRERccnjCxcuZHBwMGfMmOFAVTnL1Iz+N7bJycl87bXX+K9//euSY7788ktee+21/Oqrr3K7vBxlw0w1PaOpfZieqRltmjWk+b1oej7S3F5Mz9SMNs0bG3rRhoxiniCIZMLr9QIAjh49ipSUFPTu3Rvz58/H888/jyNHjiA4OBgAsG/fPpw7dw433XSTk+VmW+nSpXH69GnExMQAAFJSUgAAUVFR6Nq1Kw4ePOhkedlmej4AKFOmDI4ePYply5YBAHw+HwDgxhtvxL///W8cPnwYaWlpTpaYbTZkLF68OJKSkrB3714AwLlz5wAA7du3x913341ffvnFyfJyhKkZPR4PAGD06NEYN24cvvzyy8DM8WvdujWqV6+OPXv2OFFijrFhppqe0dQ+TM/UjDbNGsD8XjQ9H2BuL6Znakab5o0NvWhDRjGPNm3lEiQDm7JPPfUU3nrrLdx666149tlnsX79enTs2BFPPvkkunfvjhdeeAGvvfaawxVnX40aNVCuXDlMnDgR586dQ/78+QPPHTlyBElJSQ5Wl32m5wOAmjVromnTppgyZQp27dqV4bmtW7cCAPLly+dEaTnGhozh4eFITU3FpEmTAADXXHNN4Lm0tDTEx8c7VFnOMT1jixYt0LZtWwQHB+Ppp5/GhAkTcPr0afh8Pnz22WfYsmUL7r33XgAX/t64kQ0z1fSMpvchYH5GG2YNYH4vmp4PML8XAfMz2jBvbOhFGzKKeTx061SRv43X60VwcDAGDBiAr7/+GnPmzEHVqlVx/vx5REdHY8OGDdi0aRNKliyJ++67Dw888IDTJV81f8akpCQUKlQIALB27Vp07twZ11xzDQYMGIBrrrkGW7ZswaRJk/DTTz+hdOnS8Pl8CArK++c6TM8H/JExISEBPp8PRYsWxbZt29C5c2ccO3YMjz32GAoVKoTY2FgsWLAAu3fvRqlSpZQxj/FnTElJCbxxmjdvHv7973+jSpUqGDZsGAoVKoQ1a9bg7bffxr59+1z7WjU5Y2aio6Mxbtw4HDhwACkpKShSpAiqVKmC++67D127dkVycjIKFCjgdJlXxKaZampGG/rQhoyZMWnWAPb0oqn5ADt60YaMmTFp3tjUiyZnFAs4eW0GybtOnDjBMmXKBK7Nk/4i3KdOneL58+edKi1H/etf/+Kbb74ZuMvnwYMHOWDAABYvXpzVq1dn27ZtOXfuXJLuugOon+n5SLJLly4cNmwYDx8+HHjs7bffZvXq1XnrrbfygQceYHR0NEllzMt69+7Nd999NzBb1q5dywcffJAFChRgWFgY77jjDk6dOpWkMuY1/lrPnDnDo0ePBu6+S164BtzYsWPZqFEjlitXjoMGDWJMTIxTpWabDTPV9Iym9mF6pma0adaQ5vei6flIc3sxPVMz2jRvbOhFGzKKubRpK5navHkza9euzTVr1gQe83q9JMnt27fzqaee4p49e5wqL1v8g3jChAmsUqUKly1blukxv/zyS4bH3HL3SNPzkRkzVq5cmRs3bsz0uItv2OUmNmX88MMPGR4ezvnz52d4HaakpPDkyZP89ttvM7yBcuNr1dSM6ets164dK1WqxIYNG/K+++7jihUrAs/9+uuv7N+/Pxs0aMDWrVtz2LBhPHTokBMlXzXbZqqJGU3vQ9L8jDbMGtKeXjQ1H2l+L5LmZ7Rh3tjUiyZnFDto01YylZCQwIiICI4YMYLkHxu2JLlgwQJWrVqVZ8+edaq8HFGlShV++OGHgX/3D/aUlBSmpKQEHnfr4DY9H0lGRERwwoQJgX/3v06Tk5MZFxcXyKaMeVuFChX48ccfB/49NTWVZMa5QypjXuSvd/Dgwbz55ps5Y8YMvvXWW2zfvj1r1KjBXr16ZXgzHBMTw27durFatWr86aefnCo7S2yYqaZnNLUP0zM1o02zhjS/F03PR5rbi+mZmtGmeWNDL9qQUcymC3XIJUiiUKFCaN++Pf7zn//g7bffRlJSEhISErBt2zYMGjQI3bp1y3CBebf59ddfUbp0aVSpUgVAxpuv/fTTTxg6dChiY2MB/HHXUDcxPR8AnDhxAsWLF0fhwoUBAD6fL5AlLi4O7777LrZv3w5AGfOyXbt2oWzZsoiMjARw4bXqv5na999/j5deegknT54EoIx5kcfjAUl4vV689NJL6NatGwYOHIjXXnsNPXr0wN69e9GuXTu8/PLLIImGDRtixowZmDFjBqpWrep0+VfMhplqekaT+9DP5Iy2zBrA/F40PR9gdi/6mZzRlnljQy/akFHMp01buYTH40FwcDBGjBiB1157Da+//jqqVKmCFi1aoEOHDqhatSqGDh3qdJnZUrx4cRw9ehTz5s0DkHFIJycnY/LkyfD5fE6Vl22m5wOAokWL4ty5c5g7dy4AICgoKJAzPj4ekydPDlxw3q1syFiqVCnExsZi0aJFADK+VoOCgjBt2jScPXvWqfJyhOkZPR4PypQpg5iYmMBjtWvXxoABAzB8+HBERUUhJCQEHo8HaWlpAICGDRs6VW6W2DBTTc9oeh8C5me0YdYA5vei6fkA83sRMD+jDfPGhl60IaNYIFe/1yuudPz4cY4ZM4YjR47kypUrefz4cadLyhEfffQRa9asyVGjRnHbtm0kyf379/OOO+5gt27dSF768x43MT0fSX755ZesXr06+/bty5UrV5K8cD3m22+/XRld5MUXX2RkZCQ/+eQTxsXFkbwwd5o3b86HHnqIpDLmNf6fQG7atIldunRh/fr1WaFCBS5fvvySY0+fPn3Zn0y6iQ0z1fSMpvVhZkzLaOOsIc3vRdPzkeb1YmZMy2jjvLGhF23IKGbzkKTTG8ciTjh16hT+85//YPny5ShXrhxOnz4Nn8+HAgUK4Ouvv8a1114Ln8+HoCB3fiHd9HwAcPbsWbz//vv44osvkJCQgOPHj6NgwYKoUKECvvjiC4SGhiqjC8TFxaFnz55Yt24dmjdvjqSkJMTHxyMxMRExMTFGvFZNzRgREYEqVaqgWrVq2LdvH1avXo2WLVvi3XffRaVKlZwuL0fZMFNNz2hqH6ZnakabZg1gfi+ang8wtxfTMzWjTfPGhl60IaOYTZu2Yr3169dj6dKlCAkJQeXKldGiRQuUL18eXq83cM0bNzM9HwBs27YN3333HdLS0lC+fHk0atQIJUuWVEYXIBn4qdIXX3yB2bNno0iRIqhatSruueceVKtWTRnzGP8b259//hmPPvooVq9ejaCgIBw+fBgrV67EuHHjsHfvXjz44IMYPXq0cdcIs2GmmpjRtD7MjGkZbZ81gJm9mJ6p+UzrxcyYltH2eWNqL6ZnQ0YxkzZtxRrp31wAMO6Mmun5MqOMZrDhzZJpGX0+H0aNGoVvv/0Ww4cPR82aNQFcyPnTTz9h4cKFGDlyJD777DPcfvvtDlebM2zoRdMzmtaHmTEto42zBjC/F03PB5jXi5kxLaON88aGXrQho5hNm7ZiFZIgiaCgoMAAT05ORoECBZwuLUeYmu/iDWk/f8bz58+jYMGCDlSWc2zIeDF/Nn92E16rFzM146pVq9CiRQsAwHvvvYcnn3wyw/Nnz57FoUOHUKNGDSfKy5bL9aJJa2hDxvRM7cP0TM1o8qwBzO9F0/NlxtReTM/UjCbPGxt60YaMYiedchCj7dy5E7NmzcLkyZMRExMDj8cT2ND0Gz9+POrVq4fdu3c7WGnWmJ7Pz+PxILPzS/6zpqNHj0bRokWxefPmTI9zA9MzxsXFYdmyZVi5ciV27NgB4EI2r9cbyDN16lTcf//9iI2NdbLULLMho1/Tpk3x448/ok+fPnjmmWfQsmXLDDMmNDTUlR9qAFz2J4/+x02ZqX/2uJsz2tCHNmT0M3nWAGb3ImB+Pht60YaMfibPG9N7EbAjo9hJm7ZiHK/XC+DCGdKHHnoI//73vzFu3Di0bNkSd911F3bu3ImgoKDAt1CLFSuGIkWKIDQ01OHKr4zp+YA/Mi5duhQzZsxAYmJi4A9u+g1p4MLZ00aNGqFp06YoVqyYa64xZVPG8ePHo2PHjmjVqhWeeOIJ3H///Xj44Ydx6NAhBAcHIygoCElJSTh69Cj279+P/PnzO1z5lbMhY2by5cuHyMhIjBw5EkuXLsX58+dRu3ZtPPPMM0hMTHS6vKviX8MNGzZg9erVSE5ODjx38QkSr9fr6plqakYb+tCGjJkxadYA9vSiqfkAO3rRhoyZMWne2NSLJmcUAQBQxEDx8fG85pprOGPGDJ48eZJ79+7lggULeMcddzAoKIhDhgzJcHxcXBxJ0uv1OlHuVTM9H0n+/vvvvOaaa1irVi0+88wzXLVqVYbnfT5fhn+Pj48nqYx5zenTpxkaGsqxY8fy0KFDXLt2Ld977z3eeuutDAsL4+jRozMc/9NPP5FUxrwmLS2NJJmcnMzDhw9z48aNGZ4/deoUJ02axNKlS7NKlSquyebvsfj4eBYqVIj16tXj6NGjuWvXrkyP8zt06BBJd6yhDRlJO/rQhoymzhrS/F40PV96NvSiDRlNnTc29KINGUX8tGkrRvrkk09Yp04d/v777xkeP3bsGN9//31Wq1aNw4cPd+3ANj0fSb788sts1KgRn332WTZs2JB33nknhw0bxu3bt2c47ueff3aowuyzIePkyZPZoEGDwBtjkkxJSeHmzZs5aNAg1qlThzNnznSwwuyzIaPfv//9b9apU4fXXXcdb7zxRs6ZM4dJSUmB5/ft28fvvvuOJDP8/5HXPf/887zlllvYvXt3li9fnm3btuWMGTP422+/ZTju4MGDDlWYfaZntKEPbcjoZ+qsIc3vRdPzkXb0og0Z/UydNzb0og0ZRbRpK0b68ccfGRoayuXLl1/y3NmzZ/nKK6+wVKlS/OWXXxyoLvtMz5eUlMQBAwawX79+JMndu3fzscce480338wOHTpwwoQJjIuLY3x8PMPCwhgdHe1swVlgQ0aSXL58OQsXLsxvv/32kueOHz/OHj16sGLFijxy5IgD1eUM0zP6P6CMHDmSERERXLBgAXfv3k2Px0OPx8OmTZty7dq1l3ybwS0SExPZu3dvPv/88yTJ77//ni1btmSlSpXYs2dPLl26lL///jtPnjzJ+vXr8+uvv3a44qtnQ0bT+5A0P6Pps4Y0vxdNz+dnei+S5mc0fd7Y0Is2ZBQhtWkrBnvooYd42223MTo6mikpKRmeO3bsGOvWrcslS5Y4VF32mZ7v6NGj3LJlS4bHlixZwg4dOrBevXrs1asXW7ZsyWrVqjlUYfbZkDExMZFt2rRh165duXnz5kve/P7yyy+sXbt24BsMbmRDxqSkJJYvX57z5s0jST799NO84447uGHDBoaHh7NQoULs3Llzhm+muIXX6+XWrVsZExOT4fFZs2axTp06jIiI4NChQ9mxY0eGh4c7VGX22JDRhj60IaPJs4Y0vxdNz+dnQy/akNHkeWNDL9qQUYTUpq0Y7JtvvmGLFi0YFRXFt99+m5s3bw48t3v3bl577bWZnj12C9PzpZeamhr4Z5/Px4kTJ7JevXr0eDyBs6bpj3EjkzMuWLCA4eHhbNy4MT/99FMePnw48NyuXbtYpEgRbtq0ycEKs8/0jKtWrWKXLl145swZxsbGMiwsLHAN5meffZbNmjXjs88+62yROST9STCv18thw4YxLCyMHo+Hq1evJuneXvQzNaPpfUian9GmWUOa24t+JuczvRdJ8zPaNG9M7kU/GzKKnTzkRbfWEzFIXFwcXnzxRSxfvhw1atRAyZIlUahQIWzbtg1hYWFYuHAhSMLj8ThdapaYnu9iXq8XwcHBAIAWLVqgePHi+OSTT5TRBWJjY/HYY49h9erVuOeee1CrVi2kpKRg48aNKFy4MBYvXqyMediRI0ewatUq/POf/8ScOXPw8ccfY/78+ShTpgw+++wzrFu3Dm+++SaCg4MzvIbdLC0tDfny5QMA3HnnnShVqhTmzp3r2jXMjIkZTe5DP5Mz2jhrADN7MT1T85nci34mZ7Rx3pjai+nZkFHsok1bMZbP50NQUBAA4JtvvsH//vc/HD9+HPv378cjjzyCzp07o1ixYq79I2x6vj+zfPlytGzZEgcOHEClSpWUMQ/jhV90BF6ry5Ytw9ixY5GcnIzY2Fh069YNvXv3RokSJZQxj7r4Te6XX36J7t27Y+3atbj22mvRsWNHNGnSBO+8846Rb4i/+uortG7d2vW9+GdMyGh6HwLmZ7R91gBm9OKfMSWf6b0ImJ/R9nljSi/+GRsyih20aStGS7+xmdm/u/2PsOn5/syGDRsQFRWV4WyqaUzK6PV6ERQUFHg9xsfHIzQ0FCEhIQDMeK2alNH/mluzZg0WLVqEnj17IiIiAiSRkJCAdu3aYe3atahSpQoKFiyI7du3w+PxuCrjlTp9+jS+++47tGzZ0ohezIxJGU3qw8sxKaNmTUYm9WJmTMtnUi9ejkkZNW/+YFovZsaGjGIHbdqK8S7eyDSN6fkAZTSFMrpD+g8nVatWRZcuXdCzZ0/ccMMNSE5ORoECBXDu3DksX74cZ8+eRVRUFCpXrmzMG2ITP5xdzPSMJvThXzEho+2zBjC/F03Ll1keE3oxPVMz2jRvMltDG3rRtIwifu6eviJXICgoCF6vF6+99ho2btzodDk5zvR8gDLmdenP/aWkpFz2OH/GSZMmYefOnblRWo6xIaOfP+urr76KggUL4rXXXkOVKlXw888/o1OnTmjWrBnWrFmD9u3b44EHHkDlypUBIM9/qLnSNfR4PK5dQ9Mz+vORxMmTJ3HmzBkkJCRccoyb+9CGjH6mzhrgytfR9F50az7gwkYlABw7dgyxsbEAEPjWpZ/be9GGjH6mzpsrXUMbetHNGUX+jDZtxfUOHToEr9f7p8d8//33mDlzJiZNmpRLVeUc0/MByujn1oz+s9ozZ85Ep06dUL9+fUyYMCHTY1euXIkXX3wRU6dOzc0Ss82GjH5BQUFITU3Ft99+i8ceewwAMG/ePDzzzDNISkpCuXLl0LlzZxw4cMDZQq+SDWtoekZ/vpEjR+Luu+9GrVq18O6772Z6zIoVK1yXD7Ajo5+pswYwfx1Nzwcg8M3STp064YknnsDMmTMRHx8fyOXz+QL/7MZ5CtiR0c/UeWPDGtqQUeRPUcSFvF4vSXL69OmsX78+586dyxMnTvzpf7NkyRL++uuvGf77vMr0fKQyXo7bMqalpZEkP/roI1arVo09evTgk08+yaCgIA4ZMoTkpTmmT5/OX375JdPn8iIbMmamf//+rFy5Mt9++23ecMMNHDp0KI8cOcJTp06xUaNGXL9+vdMlXjEb1tD0jP5848eP5/XXX8+xY8dy5MiR9Hg83Lp1K8+cOcMDBw4EjiPJmTNnuiYfaUfGzJg0a0jz19H0fOkdPHiQpUuXZoMGDRgVFcU+ffrwyy+/JEl+/fXXPH/+fOBYZXQH0+YNacca2pBR5HK0aSuu5fP5eMMNN7B8+fLMly8fO3TowDVr1vDs2bMZjkv/ptFNTM9HKmN6bs9YunRpzpo1K/DYxIkTWbt27QxvotzMhowX27ZtG7t27cpatWrx5ZdfZlJSEkly1apVLFGixF+egMhrbFhD0zP6fD6WKVOGc+bMCTzWvXt39urVi2XKlGGTJk342GOPMS4uzsEqs8eGjBczbdaQ5q+j6fn8vF4vn3nmGT777LMcO3YsGzRowObNm/O5556jx+PhokWLnC4x22zImJ6J88aGNbQho8jlaNNWXGvRokVs3Lgx9+3bx5iYGNapU4eFChXiM888w+3btzM1NZUkuWzZMo4YMYI+n8/hiq+O6flIZTQl40cffcTbb7+dp0+fDtR/8uRJli9fngsXLgwct3fvXn799ddOlZktNmTMTEpKSoYTCj/++CMjIyM5ePBgku462WDDGpqecezYsWzWrBl///33wGMlSpTgQw89xNmzZ3PEiBGsWLEiJ0yY4GCV2WNDxsyYNGtI89fR9HzprV27lo0aNWJiYiJ37tzJQYMGsUSJEixbtizHjBnDgwcPOl1ittmQMT3T5g1pxxrakFEkM7qmrbhWxYoV0bFjR4SEhKBhw4bYsmULPvjgA0ydOhWtW7fGuHHjsG3bNvTo0SPDtW7cwvR8gDKakJEkUlJSkC9fvkD9aWlpKFGiBG6//XZ8/vnngWNbtWqFLVu2OFht1tiQMS0tDQCwfv16PPLII2jdujVeeeUVrFu3DsHBwQCA3bt3Y8qUKShXrhzeeOMNAHDN3aRtWEPTM/p8PpQsWRLdu3dHSEgIAGD48OEoWrQoxo4di65du2LQoEGIjIzE1q1bHa42a2zIaPqsAcxfR9PzXaxJkyaoWbMmZs+ejYiICDz88MM4c+YMIiIiMG3aNPTp0wd79+51usxsMTWjDfPGz9Q1TM+GjCKZcnbPWCR7kpOTSV44Y5regAEDmD9/fhYrVowVKlQIPO62bzGano9URlMyxsTEkMxY+6RJk9igQQOS5Pvvv89SpUo5UltOsSFjqVKl2LlzZ95zzz2Miori7bffziFDhnDbtm0kyV27dgW+yeDGb6LYsIamZ0x/aZnt27dz586dJP+YrwMGDODDDz/sSG05xYaMps8a0vx1NDnfxe/Dpk2bxlq1apEkIyMj+dRTT5EkJ0yYoIwuYOq8sWENbcgo8le0aSuu4x/eKSkpnD9/fobn/D81Jy9clNzj8TA6OvqS5/Iy0/ORymhKRv/F/b1eLz/99FOSGd9c7dmzh+XLl+f27dtZsmRJzpw5k6Qy5jX+PHPnzuXdd98deHzr1q3s168fb7vtNt5zzz18++23eebMGafKzDIb1tD0jP58mc3T9I4fP86wsLDApSDcdAMSGzKaPmtI89fR9HzkH6/T8+fP85NPPiF5of6uXbvynnvuYcmSJbl///7A8f5Najdu9tmQ0dR5Y9MampxR5Epo01Zcxz/Ae/XqxcqVK/PcuXOXPJ+amsrOnTuzdu3aTpSYLabnI5XR/7wpGXv27HlJxtTUVHq9Xt5555285pprGBUV5VSZ2WJ6Rv8H6cTERA4fPpxPPvnkJccsWbKEXbp04U033cRff/01t0vMNtPXkDQ/41/NU5I8cOAAH3/8cTZu3Di3y8sRpme0YdaQ5q+j6fnISzMmJiaSJP/3v/+xePHifP/99wPHuWkzOj3TM9owb0xfQ9KOjCJXIp/Tl2cQuRo+nw9BQUFITk5GhQoVMGbMGBQqVCjDMR6PB8nJyShfvjxeeuklAIDX6w1cuygvMz0foIx+JmWsWLHiJRnz5bvw56V27dpYvXo1Jk+eDEAZ8xr/ddveeustjB07FgUKFECnTp3QrFmzwDGtWrXCHXfcgU2bNqFChQog6ZprL9uwhqZnvJJ5ev78eYwfPx6bN2/G9OnTAbgnH2BHRtNnDWD+OpqeD8g8Y2hoKADgvvvuQ3h4OGrVqgXgwvs4N70+/WzIaPq8sWENbcgocsUc3jQWyZIhQ4awdu3afPXVVwOPXXyGLSkpiaQ7rw9qej5SGf1Mz5iamhr4CaVbf65kasb0r7kDBw5wyJAhrFGjBps2bcrRo0dn+MmZ25m6humZnvGv8qWlpQWuqenWb9yYmtGmWUOau45+pucjM89oGlMz2jRvTF3D9GzIKPJX3HdrRLHeqVOncOjQISQlJWHcuHFYtmwZgAtnVX0+H3w+HwCgYMGCAOC6M2+m5wOU0ZaM/jvZd+zYMfC425ic0f+amz17NipXrozXX38dH330EcqVK4epU6fi+eefx5w5cxAfH+9sodlk8hr6mZ7xr/KlpqYiODgYERERgcfdxuSMtswawOx1BMzPB1w+I4AM79/czOSMtswbk9fQz4aMIlfE6V1jkaw4cOAAZ86cyRYtWrBKlSrs2bMnf/7558Dzbv3Wop/p+UhlJO3I6NZv2aRncsYtW7bQ4/Hw8ccfz3BTqpkzZ7J169asUaMGhw8f7mCFOcPkNfQzPaPmqbsz2jJrSLPXkTQ/H/nXGU1gckZb5o3Ja+hnQ0aRv+IhSac3jkX+CtNdZyj9tbEOHTqEOXPmYPHixYiPj0fbtm3x0ksvoUCBAk6We9VMzwcoozK6hw0Z01uxYgWee+45NGzYEM899xyqVKkCAEhISMCoUaPQtm1bNGzY0FXXe7NhDU3PaHo+wI6M6Zk4awDz19H0fIAympIxPRPnjQ1raENGkauWq1vEIlng/3ZQSkoKp02bxsaNG7Nbt24cNmwYjx07xrS0NK5du5bPPvssq1Spwh9++MHhiq+O6flIZVRG97ApI/nHtU0/++wzNmjQgIMHD3aqrBxj0xqamtH0fKRdGUkzZw1p/jqano9URtMykmbOG5vW0OSMIlmhTVvJ8/wDvHfv3rz55pv52muvsW3btixevDi3bdsWOO73339nTEyMU2Vmmen5SGVURvcwPaP/Z6v79u1jkyZNOHLkSE6bNo2JiYlcs2YNQ0ND+dBDDzE+Pt7hSrPO9DUkzc9oej7S/Iw2zBrS/HU0PR+pjCZktGHemL6GpB0ZRbJCm7aSp/n/CO/evZvXXnstt2zZQpLs3Lkzu3XrRpI8fPgwv/jiC8dqzA7T85HKqIzuYUNGv2nTptHj8bB58+bs168fr7vuOvbv359PP/00Q0JC2LVrVx45csTpMq+aDWtoekbT85F2ZPQzddaQ5q+j6flIZTQlo5+p88aGNbQho0hWadNWXOHDDz9ks2bNSJKffvopS5QowQMHDpAkV6xYwTZt2vDHH390sMLsMT0fqYzK6B42ZPz99985ePBg5s+fn+PHj+fPP//MJ554gt26deP111/P4sWLZ7h5h9vYsIamZzQ9H2lHRtNnDWn+Opqej1RGUzKaPm9sWEMbMopcrSCnr6krciXq1q2LhIQEAMCrr76Kp556CpUrVwYAHDx4EL/99hsiIyOdLDFbTM8HKKMyuofpGY8fP44iRYrgjTfewLhx47BgwQLs2LEDY8aMwdtvv43Vq1cjJiYG+fLlQ1pamtPlZonpawiYn9H0fID5GW2YNYD562h6PkAZTchow7wxfQ0BOzKKXDWnd41FMuP/iYTf8ePH2aBBA5YpU4Zly5YNPH7w4EFWqFCBY8eOJfnHhefzOtPzkcqojMqYF3366ads0aIF+/bty+3bt3Pv3r18/fXX2bNnT3777bdOl5dlNqyh6RlNz0fakdHP1FlDmr+OpucjldGUjH6mzhsb1tCGjCLZ5SFJpzeORS5GEh6PB9OnT0d4eDiioqKwe/duDB06FGvXrkWTJk1QokQJbNq0CcWKFcPSpUudLvmqmJ4PUEZldA8bMvrNmjULW7duxTfffIP169ejTZs2KFKkCL788kv4fD588MEH6Natm9NlXjUb1tD0jKbnA+zI6GfqrAHMX0fT8wHKaEpGP1PnjQ1raENGkezSpq3kaZ06dcKOHTswd+5c3HTTTdi0aRPWrVuHZcuWISEhAb169UKLFi0QFhYGr9eL4OBgp0u+KqbnA5RRGd3DhozAH2+Qd+/ejRkzZiAxMRHbtm3DqlWr0KNHD3z88cdOl5hlNqyh6RlNzwfYkREwe9YA5q+j6fkAZTQlI2D2vLFhDW3IKJJV2rSVPG3fvn149NFHER4eHvhj6x/U/j/ObmZ6PkAZldE9TMzor//8+fPYtWsXjh07htTUVNSpUweVKlUCAJw6dQohISH47rvvEBkZiZIlS8Ln8yEoyH2XvTdxDS9mekbT8wFmZrRt1gBmrmN6pucDlNGtGW2bNyau4cVsyCiSZX/bhRdEcsj69et53XXXsWfPnjx9+jRJuvrOnxczPR+pjKZQRvfq3r07b7zxRubPn5916tRh48aNOXr0aKakpDhdWo4zdQ3TMz2j6flIczPaNGtIc9fRz/R8pDK6mU3zxtQ1TM+GjCJZ4b5TTWIFn88X+OeoqChMnDgRP/74I1atWgUAyJcvn1Ol5QjT8wHKCCijW5ia0ev1ArhwrbcvvvgCH3zwAY4ePYrhw4fjlltuwdSpUzFv3jyHq8wZpq5heqZnND0fYG5Gm2YNYO46+pmeD1BGwL0ZbZo3pq5hejZkFMkudYHkGfz/P31Ys2YNli5dinLlyqF58+ZITU3FP//5T6xfvx6PPPIIChUqhNatW7vupxKm5wOUURndw4aM/ut9zZgxA3379sWdd94JAGjXrh0aN26MQYMGYeDAgWjevDnKli3rZKlZYsMamp7R9HyAHRlNnzWA+etoej5AGU3JaPq8sWENbcgokpN0TVvJc/r27Ytdu3bh8OHDSEhIQFhYGOLj49GqVSt8+OGHuPnmmxEdHY1SpUo5XWqWmJ4PUEZldA9TM3q93sAb3B49euDIkSNYvHgxQkJCAm9+f/75Z9x7772YPHkybrnlFocrzjpT1zA90zOang8wN6NNswYwdx39TM8HKKObM9o0b0xdw/RsyCiSE7RpK3nS6dOnUbx4cfzwww9ISEjA/v37ERMTg4IFC2LOnDmoXr065s+f79ohbno+QBmV0T1Myrhx40bceuutGW60MXfuXLzwwgt466238I9//AMhISEAgO3btyMqKgobN25ErVq1nCo5R5i0hpdjekbT8wFmZbR11gBmrWNmTM8HKKPbMto6b0xaw8uxIaNItv2N18sVuSI+ny/DP//VBccPHDjAevXqcerUqX93aTnC9HykMmZGGfMmkzP+/vvvrF27NqOiorhw4cLA44mJiezSpQuDg4P56KOPcsGCBXznnXfYvHlz/vOf/yRJer1ep8q+aiavoZ/pGU3PR5qd0ZZZQ5q9jqT5+UhlzIybMtoyb0xeQz8bMor8HXQjMskzZs6ciV69eqFu3bp44YUXEBsbG3jOf5Fyr9eLypUro379+pg2bRrS0tKcKveqmZ4PUEZldA8TMwYFBWHAgAG4/vrrMWTIEHTp0gU7duxAaGgo5syZg3nz5mHDhg0YMGAA/vvf/6J69eqYMmWK02VnmYlreDHTM5qeDzAzo22zBjBzHdMzPR+gjG7NaNu8MXENL2ZDRpEc5fSusdgtLS2NJLl69WpWqFCBDz/8MMeNG0ePx8OKFSvyrbfe4qlTpy757+69914+9dRTuV3uVTM9H6mMyqiMec3Bgwf57rvvslKlSqxYsSIHDx7MY8eOBZ7fsWMHT58+HfiGg5u+iWLDGpqe0fR8pB0ZSbNnDWn+Opqej1RGUzKSZs8bG9bQhowifxdt2opj/MObJCMjI/nSSy+RJOfNm8eKFSty4MCBDAkJYYsWLThnzpzATyrOnz/P5cuXO1Lz1TA9H6mMyqiMecn58+dJkkuXLmWnTp3YrFkz1q9fnxEREaxXrx7Hjx/vcIXZY8Mamp7R9HykHRlNnzWk+etoej5SGU3JaPq8sWENbcgo8nfSpq04bu3atbztttt4+PBhkuQNN9zA999/nyT5yCOP0OPxMCIiwskSs8X0fKQyKqN7mJ7R5/OxcOHCHD9+PM+fP8+0tDQuWLCAHTt2ZOHChdm+fXsuWrTI6TKzxfQ1JM3PaHo+0vyMNswa0vx1ND0fqYwmZLRh3pi+hqQdGUX+DrqmreS6hQsX4rnnnsP27dsBAJUqVUL79u0RGhqK2bNno2jRoujcuTMAoG3bthg1ahRiYmIAwBXXszE9H6CMyqiMedWyZctQpkwZ3HvvvShQoACCg4PRoUMHvPfee7jhhhsQExODL774wukyr4oNa2h6RtPzAXZkTM/EWQOYv46m5wOU0ZSM6Zk4b2xYQxsyiuQGbdpKrtu/fz8+//xzvPTSS5g4cSKKFCmCgQMHokiRIihYsCDS0tJQoEABAMCKFSuwbt06FC5cGACQL18+J0u/IqbnA5RRGZUxr7r++utx8uRJzJo1K8PjFSpUQMeOHfHwww9jxIgRAC7c5MENbFhD0zOang+wI2N6Js4awPx1ND0foIymZEzPxHljwxrakFEkVzj9VV+x0+HDh/noo4+ybt267N69OxcsWMCzZ89y3759LFy4MBs0aMD77ruPBQsW5I4dO0i664LypucjlVEZ3cOGjOkNHDiQt912Gz/99FMePXo08Hjr1q356quvOlhZ1tmwhqZnND0faUfG9EycNaT562h6PlIZTcmYnonzxoY1tCGjyN/NQ5JObxyLvb7//nsMGTIEZ8+eRVRUFHr27Inz58/jgw8+QGpqKjp06ID77rsPXq8XwcHBTpd71UzPByijMrqHDRkB4PDhw+jduzdWr16N1q1bI1++fDh16hS++eYbHDx4EIULFwZJeDwep0u9ajasoekZTc8H2JERMHvWAOavo+n5AGU0JSNg9ryxYQ1tyCjyd9GmrTjO6/Vi3rx5GDVqFIoVK4Y2bdqga9euKF++fOAYt/4RBszPByijnzLmfaZnTF97dHQ0pkyZgpCQEJQuXRqdOnVCkyZNkJaW5uqfnZm+hoD5GU3PB5if0YZZA5i/jqbnA5TRz80ZbZg3pq8hYEdGkb+DNm0lz0hMTMSbb76J6OhoFC5cGBMnTkS1atWcLivHmJ4PUEZTKKO7XfwtheTk5MA1w0xi8hr6mZ7R9HyA2RltmTWA2esImJ8PUEa3s2XemLyGfjZkFMlJ2rSVPGffvn2YOHEi3njjDadL+VuYng9QRlMoo7v4fD4EBQUFvqVg6geai5m0hpdjekbT8wFmZbR11gBmrWNmTM8HKKPb2DpvTFrDy7Eho0hO0Kat5GmmX9fG9HyAMppCGfOWuLg4bN++HcHBwShTpgxq1aoF4EIGj8eDoKAgfPjhh1i2bBlGjRqFypUrO1xx7nDTGmaV6RlNzwe4K6NmzeW5aR2zwvR8gDLmNZo3mXPTGmaVDRlFssq9F34RK5g+vE3PByijKZTRef43tOPHj8fUqVMRExODmjVrAgDq16+PN954AxUqVAAAJCUl4ejRo9i/fz/y58/vZNm5Kq+vYU4wPaPp+YC8n1Gz5srk9XXMLtPzAcqYF2je/LW8voY5wYaMIlmlb9qKiIi4RHx8PCpUqIA333wTHTp0wP79+7Fp0ybMmjULBw8exKBBg9CvX7/A8T///DOqVq0a+HmhiMiV0KwRkdyieSMicnnatBUREXGJKVOm4L///S82btwY+FZCamoqdu7cidmzZ+PLL7/EoEGD8OCDDzpcqYi4mWaNiOQWzRsRkcvTqSkRERGXqFixInbv3o1NmzYFHgsJCUFkZCQGDhyI+vXrY/DgwTh69KiDVYqI22nWiEhu0bwREbk8bdqKiIi4RKNGjdCkSRO888472LJlC9L/WOa6667DCy+8gKJFi+LXX391sEoRcTvNGhHJLZo3IiKXp01bERERlwgNDcXjjz+O7777Dn379sWCBQsQFxcXeD4lJQUHDx7UDR1EJFs0a0Qkt2jeiIhcnq5pKyIi4jKxsbF47LHHsHr1atxzzz2oVasWUlJSsHHjRhQuXBiLFy8GSXg8HqdLFREX06wRkdyieSMicilt2oqIiLgESZAM3C152bJlGDt2LJKTkxEbG4tu3bqhd+/eKFGiBLxer76VIiJZolkjIrlF80ZE5PK0aSsiIuIyXq8XQUFBgW+bxMfHIzQ0FCEhIQCgb6KISI7QrBGR3KJ5IyJyKW3aioiIuJDP5wt8K0VE5O+iWSMif4fMNmE1b0REMtJEFBERyUPSn0tNSUm57HFBQUHwer2YNGkSdu7cmRuliYhB/LOGJE6ePIkzZ84gISHhkmM0a0Qku3w+HwDg2LFjiI2NBQB4PJ4M73k0b0RELqVNWxERkTzE/62TmTNnolOnTqhfvz4mTJiQ6bErV67Eiy++iKlTp+ZmiSJiAP+sGTlyJO6++27UqlUL7777bqbHrFixQrNGRLLM/+3ZTp064YknnsDMmTMRHx8fmDE+ny/wz3pvIyLyB10eQUREJI/w32Dj448/xsiRI9GkSRNce+21GDt2LAYNGoTXX3/9kp8OzpgxA1FRUbjhhhv0s0IRuSL+WTNhwgSMGDECzz77LBITEzF48GBs2bIF119/PU6dOoUKFSoEbvoza9YsNGrUSLNGRLLk119/xS233ILKlSsjJCQEdevWRbt27dC6dWusWbMGt956KwoUKABA80ZExE+btiIiInkISZQtWxajR4/GAw88AAD46KOP8N577+H7778PfKAREckOkggLC8N7772HLl26AAAeeeQR5M+fH59//jmqVauGG2+8Ea+88grKli3rcLUi4nY+nw8DBw5Evnz5cP3112PKlCkoXLgwbrnlFrz11ltYuHAh2rVr53SZIiJ5ik5ZiYiI5CGTJk1CREQE2rRpE7jWW8eOHXH69GksXbo0cNxPP/2ENWvWOFWmiLjcuHHjUKNGDbRp0ybw2OLFi3H+/HmMHj0a99xzD7788kssWrTIwSpFxBRBQUHo2LEj1q5di0ceeQRTp05FgwYN8NFHH6FMmTKIjY3Fr7/+6nSZIiJ5ijZtRURE8giSSElJQb58+QLXd0tLS0OJEiVw++234/PPPw8c26pVK2zZssXBakXErXw+H0qWLInu3bsjJCQEADB8+HAULVoUY8eORdeuXTFo0CBERkZi69atDlcrIqZo0qQJatasidmzZyMiIgIPP/wwzpw5g4iICEybNg19+vTB3r17nS5TRCTP0KatiIhIHuHxeNC7d2+8/vrrKFGiBEgiX758AIC77rorsEk7ZswYJCYm4sknn3SyXBFxqaCgIHTp0gUPPPAAChUqBAC49957ER0djSJFiiA1NRUAUKNGDZw5c8bJUkXE5S6+GmPz5s0xevRoAEC3bt3Qp08frFy5Er169ULx4sVRvXp1B6oUEcmbtGkrIiKSB/h8vsD//vbbb5c837hxYxw+fBg7duzAK6+8EvjAk5aWlptliojL+WdNamoqlixZEni8Vq1aiIiIAACEhITgxIkTmDVrFjp16pThvxMRuVIk4fF4kJycjPnz5wO4sFF70003oV27djh06BCeeeYZAMBjjz2Gjz/+GMCFmyWKiIg2bUVERPIEj8cD4MKHlmeeeQZJSUmBx9LS0lC1alVUr14dDRs2RI0aNfDggw8CQOCbuCIiV8I/V/r06YP+/fsjKSnpkmNiY2Pxwgsv4IYbbkD79u0BQHdvF5Ese+KJJzBgwACcPXsWQUFBuPfee7F+/Xq8/PLLuP7660ESPp8vcLmW4OBghysWEckb9ElPRETEYT6fD0FBQUhOTkbFihUxZsyYwE+WgT82ZmvXro3Vq1dj8uTJAC58E0UfbETkSqWfNRUqVLhk1gDA+fPnMX78eGzevBnTp08HoFkjIlcvs3kTGhoKALjvvvsQHh6OWrVqAbhwMsl/QklERP6gTVsRERGH+b/B9sorr+Dzzz9HUFAQ2rVrB+CPDz0A8M477+COO+5A9erVtYkiIlftSmZNwYIFMXz4cOzduxfVqlWDz+fTrBGRq/Zn8yY4OBj169d3sjwREVfQ75xERETygFOnTuHQoUNISkrCuHHjsGzZMgAXPvT4fD6kpKQgX7586NixY+BxEZGr9VezJjU1FcHBwYHr22rWiEhWXW7eABdOFOla2SIif87Di2/nKCIiIo6IjY3F+vXrMWnSJOzbtw/NmzfHkCFDEB4eDiDjt25FRLLqr2aN/+ZBIiLZ9VfzRkRELk+btiIiIg5JvzGS/nIHhw4dwpw5c7B48WLEx8ejbdu2eOmll1CgQAEnyxURl9KsEZHconkjIpJztGkrIiLiAP+3ZlNTUzFnzhxMmDAB119/PWrWrInHH38cJUqUwMaNG7Fo0SJ88skn+OSTT1CvXj2nyxYRl9GsEZHconkjIpKztGkrIiLiAP8Hmz59+uCbb75Bp06dsGHDBmzYsAFr1qxB7dq1AQBnzpzB7t270bBhQ4crFhE30qwRkdyieSMikrO0aSsiIpLL/D8d3LNnD2655RasX78ederUQZcuXRASEoIZM2YgLi4OmzdvRps2bZwuV0RcSrNGRHKL5o2ISM7L53QBIiIitvFf623NmjWoV68e6tSpg88++wzLly/Hpk2bAAC7du3CmDFjEBYWhrp16zpYrYi4lWaNiOQWzRsRkZynW1CLiIg4pG7dukhISAAAvPrqq3jqqadQuXJlAMDBgwfx22+/ITIy0skSRcQAmjUikls0b0REco4ujyAiIpJL0t9RGQBOnDiBf/zjHzh48CA8Hg/i4uIAAL/++iuioqIwZMgQ9OnTJ8Pdl0VE/opmjYjkFs0bEZG/jzZtRUREcon/g8306dMRHh6OqKgo7N69G0OHDsXatWvRpEkTlChRAps2bUKxYsWwdOlSp0sWERfSrBGR3KJ5IyLy99GmrYiISC7r1KkTduzYgblz5+Kmm27Cpk2bsG7dOixbtgwJCQno1asXWrRogbCwMH0TRUSyTLNGRHKL5o2ISM7Tpq2IiEgu27dvHx599FGEh4fj448/BoDAB5iLf2YoIpJVmjUikls0b0REcp42bUVERBywYcMGdOjQAR06dMCoUaNQrFgxpKWlIV++fE6XJiIG0awRkdyieSMikrOCnC5ARETEFj6fL/DPUVFRmDhxIn788UesWrUKAPShRkRyhGaNiOQWzRsRkb+PJqiIiMjfyP+TwDVr1mDp0qUoV64cmjdvjtTUVPzzn//E+vXr8cgjj6BQoUJo3bq1fkIoIlmiWSMiuUXzRkQkd+jyCCIiIrmgb9++2LVrFw4fPoyEhASEhYUhPj4erVq1wocffoibb74Z0dHRKFWqlNOlioiLadaISG7RvBER+Xtp01ZERCSXnD59GsWLF8cPP/yAhIQE7N+/HzExMShYsCDmzJmD6tWrY/78+fpwIyLZolkjIrlF80ZE5O+jTVsREZG/QfqfApKE1+v90+u6xcbGomPHjujXrx+6d++eW2WKiMtp1ohIbtG8ERHJXboRmYiIyN9o5syZ6NWrF+rWrYsXXngBsbGxgef8N+/wer2oXLky6tevj2nTpiEtLc2pckXEpTRrRCS3aN6IiOQObdqKiIjkMK/XC4/Hg6+//hqDBw9GamoqnnjiCbz++uto2rQpRo0ahdOnTyMo6MKf4eDgYADAiRMnUKtWLd1pWUSuiGaNiOQWzRsRkdynTVsREZEc5PV6Ax9U+vXrh0cffRTTpk3DddddhwoVKqBLly54/vnncf/99+N///sf/FcpSk5ORt++ffHee+85Wb6IuIRmjYjkFs0bERFn6Jq2IiIif4N169bhueeew/z58xEWFobw8HA8/fTTePLJJ/Gvf/0L06ZNQ82aNbFz506nSxURF9OsEZHconkjIpK79E1bERGRHLBw4UI899xz2L59OwCgUqVKaN++PUJDQzF79mwULVoUnTt3BgC0bdsWo0aNQkxMDADoOm8icsU0a0Qkt2jeiIg4S5u2IiIiOWD//v34/PPP8dJLL2HixIkoUqQIBg4ciCJFiqBgwYJIS0tDgQIFAAArVqzAunXrULhwYQDQdd5E5Ipp1ohIbtG8ERFxli6PICIikkPi4uLwwgsvYNOmTahTpw46duyIu+++G0ePHkVkZCRq1qyJSpUqITo6Gj/88ANuvPFG+Hy+wE07RESuhGaNiOQWzRsREedo01ZERCSHff/99xgyZAjOnj2LqKgo9OzZE+fPn8cHH3yA1NRUdOjQAffdd1+GG3uIiFwtzRoRyS2aNyIiuU+btiIiIn8Dr9eLefPmYdSoUShWrBjatGmDrl27onz58oFjSMLj8ThYpYi4nWaNiOQWzRsRkdylTVsREZG/UWJiIt58801ER0ejcOHCmDhxIqpVq+Z0WSJiGM0aEcktmjciIrlDm7YiIiK5YN++fZg4cSLeeOMNp0sREYNp1ohIbtG8ERH5e2nTVkREJJfpem8ikhs0a0Qkt2jeiIjkPG3aioiIiIiIiIiIiOQhQU4XICIiIiIiIiIiIiJ/0KatiIiIiIiIiIiISB6iTVsRERERERERERGRPESbtiIiIiIiIiIiIiJ5iDZtRURERERERERERPIQbdqKiIiIiIiIiIiI5CHatBURERERERERERHJQ7RpKyIiIiIiIiIiIpKH/D8JsOJe36Dj3QAAAABJRU5ErkJggg==", - "text/plain": [ - "
" - ] - }, - "metadata": {}, - "output_type": "display_data" - } - ], - "source": [ - "result.plot(figsize=(14, 5))" - ] - }, - { - "cell_type": "markdown", - "id": "schedule-header", - "metadata": {}, - "source": [ - "## 5. Creating Non-Uniform Per-Layer Targets\n", - "\n", - "The key insight from sensitivity analysis is that you shouldn't compress all layers equally. Use `to_layer_targets()` to generate non-uniform per-layer compression targets:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "schedule", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "conv1: 57.59%\n", - "layer1.0.conv1: 67.72%\n", - "layer1.0.conv2: 67.72%\n", - "layer1.1.conv1: 67.72%\n", - "layer1.1.conv2: 10.0%\n" - ] - } - ], - "source": [ - "targets = result.to_layer_targets(\n", - " model,\n", - " target_pct=50, # Target 50% average compression\n", - " min_pct=10, # No layer below 10%\n", - " max_pct=80, # No layer above 80%\n", - " gamma=1.5, # Higher = more differentiation between layers\n", - ")\n", - "\n", - "# Show a few entries\n", - "for name, sparsity in list(targets.items())[:5]:\n", - " print(f\"{name}: {sparsity}%\")" - ] - }, - { - "cell_type": "markdown", - "id": "apply-schedule-header", - "metadata": {}, - "source": [ - "## 6. Applying Per-Layer Targets with Sparsifier\n", - "\n", - "The targets generated by `to_layer_targets()` can be directly used with fasterai's `Sparsifier` to apply non-uniform compression:" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "apply-schedule-code", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Computing baseline accuracy... 1.1848\n", - "Analyzing 21 layers for sparsity @ 50% (granularity=weight, criteria=abs)\n", - " [1/21] conv1... Δ=+0.0064\n", - " [2/21] layer1.0.conv1... Δ=-0.0987\n", - " [3/21] layer1.0.conv2... Δ=-0.0649\n", - " [4/21] layer1.1.conv1... Δ=-0.0779\n", - " [5/21] layer1.1.conv2... Δ=+0.0648\n", - " [6/21] layer2.0.conv1... Δ=-0.1699\n", - " [7/21] layer2.0.conv2... Δ=-0.1183\n", - " [8/21] layer2.0.downsample.0... Δ=-0.2051\n", - " [9/21] layer2.1.conv1... Δ=-0.1259\n", - " [10/21] layer2.1.conv2... Δ=+0.0256\n", - " [11/21] layer3.0.conv1... Δ=-0.0785\n", - " [12/21] layer3.0.conv2... Δ=+0.0398\n", - " [13/21] layer3.0.downsample.0... Δ=-0.0064\n", - " [14/21] layer3.1.conv1... Δ=-0.0353\n", - " [15/21] layer3.1.conv2... Δ=-0.0569\n", - " [16/21] layer4.0.conv1... Δ=-0.0521\n", - " [17/21] layer4.0.conv2... Δ=+0.0452\n", - " [18/21] layer4.0.downsample.0... Δ=+0.0108\n", - " [19/21] layer4.1.conv1... Δ=-0.0230\n", - " [20/21] layer4.1.conv2... Δ=-0.0772\n", - " [21/21] fc... Δ=+0.0573\n", - "✓ Analysis complete\n" - ] - }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "/home/nathan/miniconda3/envs/dev/lib/python3.12/site-packages/torchvision/models/_utils.py:208: UserWarning: The parameter 'pretrained' is deprecated since 0.13 and may be removed in the future, please use 'weights' instead.\n", - " warnings.warn(\n", - "/home/nathan/miniconda3/envs/dev/lib/python3.12/site-packages/torchvision/models/_utils.py:223: UserWarning: Arguments other than a weight enum or `None` for 'weights' are deprecated since 0.13 and may be removed in the future. The current behavior is equivalent to passing `weights=ResNet18_Weights.IMAGENET1K_V1`. You can also use `weights=ResNet18_Weights.DEFAULT` to get the most up-to-date weights.\n", - " warnings.warn(msg)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "Sparsity Report:\n", - "--------------------------------------------------------------------------------\n", - "Layer Type Params Zeros Sparsity \n", - "--------------------------------------------------------------------------------\n", - "conv1 Conv2d 9,408 5,473 58.17%\n", - "layer1.0.conv1 Conv2d 36,864 23,998 65.10%\n", - "layer1.0.conv2 Conv2d 36,864 23,998 65.10%\n", - "layer1.1.conv1 Conv2d 36,864 23,998 65.10%\n", - "layer1.1.conv2 Conv2d 36,864 3,687 10.00%\n", - "layer2.0.conv1 Conv2d 73,728 47,997 65.10%\n", - "layer2.0.conv2 Conv2d 147,456 95,994 65.10%\n", - "layer2.0.downsample.0 Conv2d 8,192 5,333 65.10%\n", - "layer2.1.conv1 Conv2d 147,456 95,994 65.10%\n", - "layer2.1.conv2 Conv2d 147,456 55,267 37.48%\n", - "layer3.0.conv1 Conv2d 294,912 191,988 65.10%\n", - "layer3.0.conv2 Conv2d 589,824 130,646 22.15%\n", - "layer3.0.downsample.0 Conv2d 32,768 21,332 65.10%\n", - "layer3.1.conv1 Conv2d 589,824 383,975 65.10%\n", - "layer3.1.conv2 Conv2d 589,824 383,975 65.10%\n", - "layer4.0.conv1 Conv2d 1,179,648 767,951 65.10%\n", - "layer4.0.conv2 Conv2d 2,359,296 385,037 16.32%\n", - "layer4.0.downsample.0 Conv2d 131,072 70,071 53.46%\n", - "layer4.1.conv1 Conv2d 2,359,296 1,535,902 65.10%\n", - "layer4.1.conv2 Conv2d 2,359,296 1,535,902 65.10%\n", - "--------------------------------------------------------------------------------\n", - "Overall all 11,166,912 5,788,518 51.84%\n" - ] - } - ], - "source": [ - "from fasterai.sparse.all import Sparsifier\n", - "\n", - "# 1. Run sensitivity analysis\n", - "result = analyze_sensitivity(model, sample, eval_fn, compression=\"sparsity\", level=50)\n", - "\n", - "# 2. Generate per-layer targets (layer_name -> sparsity %)\n", - "targets = result.to_layer_targets(model, target_pct=50, min_pct=10, max_pct=80)\n", - "\n", - "# 3. Create a fresh model and Sparsifier\n", - "model = resnet18(pretrained=True)\n", - "sparsifier = Sparsifier(model, granularity='weight', context='local', criteria=large_final)\n", - "\n", - "# 4. Apply non-uniform sparsity using the targets\n", - "for name, module in model.named_modules():\n", - " if name in targets:\n", - " sparsifier.sparsify_layer(module, targets[name])\n", - " \n", - "# 5. Check the results\n", - "sparsifier.print_sparsity()" - ] - }, - { - "cell_type": "markdown", - "id": "apply-pruner-header", - "metadata": {}, - "source": [ - "### Note on Structural Pruning\n", - "\n", - "The `to_layer_targets()` method generates **per-layer compression targets**, which works directly with `Sparsifier`. \n", - "\n", - "For **structural pruning** with `Pruner`, the current API uses a **uniform ratio** across all layers (torch-pruning handles dependency graphs automatically). You can still use sensitivity analysis to:\n", - "1. Identify which layers to **exclude** from pruning (fragile layers)\n", - "2. Choose an appropriate **global pruning ratio** based on the most sensitive layers" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "apply-pruner-code", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Fragile layers to protect: ['layer1.1.conv2', 'fc', 'layer4.0.conv2']\n", - "\n", - "Pruning Report:\n", - "-------------------------------------------------------------------------------------\n", - "Layer Type In Ch Out Ch Params \n", - "-------------------------------------------------------------------------------------\n", - "conv1 Conv2d 3 64 9,408 \n", - "layer1.0.conv1 Conv2d 64 64 36,864 \n", - "layer1.0.conv2 Conv2d 64 64 36,864 \n", - "layer1.1.conv1 Conv2d 64 64 36,864 \n", - "layer1.1.conv2 Conv2d 64 64 36,864 \n", - "layer2.0.conv1 Conv2d 64 128 73,728 \n", - "layer2.0.conv2 Conv2d 128 128 147,456 \n", - "layer2.0.downsample.0 Conv2d 64 128 8,192 \n", - "layer2.1.conv1 Conv2d 128 128 147,456 \n", - "layer2.1.conv2 Conv2d 128 128 147,456 \n", - "layer3.0.conv1 Conv2d 128 256 294,912 \n", - "layer3.0.conv2 Conv2d 256 256 589,824 \n", - "layer3.0.downsample.0 Conv2d 128 256 32,768 \n", - "layer3.1.conv1 Conv2d 256 249 573,696 \n", - "layer3.1.conv2 Conv2d 249 256 573,696 \n", - "layer4.0.conv1 Conv2d 256 338 778,752 \n", - "layer4.0.conv2 Conv2d 338 512 1,557,504 \n", - "layer4.0.downsample.0 Conv2d 256 512 131,072 \n", - "layer4.1.conv1 Conv2d 512 1 4,608 \n", - "layer4.1.conv2 Conv2d 1 512 4,608 \n", - "fc Linear 512 1000 513,000 \n", - "-------------------------------------------------------------------------------------\n", - "Total 5,735,592 \n", - "Original 11,689,512 \n", - "Reduction 50.93%\n" - ] - } - ], - "source": [ - "from fasterai.prune.all import Pruner\n", - "\n", - "# Use sensitivity to identify layers to EXCLUDE from pruning\n", - "fragile_layers = [layer.name for layer in result.top(3, most_sensitive=True)]\n", - "print(f\"Fragile layers to protect: {fragile_layers}\")\n", - "\n", - "# Get the actual module references for ignored_layers\n", - "model = resnet18(pretrained=True)\n", - "ignored = []\n", - "for name, module in model.named_modules():\n", - " if name in fragile_layers:\n", - " ignored.append(module)\n", - "\n", - "# Create Pruner with uniform ratio, but protect fragile layers\n", - "pruner = Pruner(\n", - " model,\n", - " example_inputs=sample,\n", - " pruning_ratio=30, # Uniform 30% pruning\n", - " ignored_layers=ignored, # Protect sensitive layers!\n", - " context='global',\n", - " criteria=large_final,\n", - ")\n", - "\n", - "pruner.prune_model()\n", - "pruner.print_sparsity()" - ] - }, - { - "cell_type": "markdown", - "id": "workflow-tip", - "metadata": {}, - "source": [ - "> **Workflow Summary**:\n", - "> - **Sparsifier**: Use `to_layer_targets()` for per-layer sparsity targets\n", - "> - **Pruner**: Use `top(most_sensitive=True)` to identify layers to protect via `ignored_layers`\n", - "> - **Both**: Fine-tune after compression to recover accuracy" - ] - }, - { - "cell_type": "markdown", - "id": "analyzer-header", - "metadata": {}, - "source": [ - "## 7. Using SensitivityAnalyzer for More Control" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "analyzer", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Computing baseline accuracy... 0.6909\n", - "Analyzing 21 layers for sparsity @ 50% (granularity=weight, criteria=abs)\n", - " [1/21] conv1... Δ=+0.0001\n", - " [2/21] layer1.0.conv1... Δ=-0.0006\n", - " [3/21] layer1.0.conv2... Δ=-0.0046\n", - " [4/21] layer1.1.conv1... Δ=-0.0052\n", - " [5/21] layer1.1.conv2... Δ=+0.0020\n", - " [6/21] layer2.0.conv1... Δ=-0.0021\n", - " [7/21] layer2.0.conv2... Δ=-0.0031\n", - " [8/21] layer2.0.downsample.0... Δ=-0.0055\n", - " [9/21] layer2.1.conv1... Δ=-0.0014\n", - " [10/21] layer2.1.conv2... Δ=-0.0021\n", - " [11/21] layer3.0.conv1... Δ=+0.0017\n", - " [12/21] layer3.0.conv2... Δ=+0.0051\n", - " [13/21] layer3.0.downsample.0... Δ=+0.0010\n", - " [14/21] layer3.1.conv1... Δ=+0.0031\n", - " [15/21] layer3.1.conv2... Δ=+0.0005\n", - " [16/21] layer4.0.conv1... Δ=+0.0053\n", - " [17/21] layer4.0.conv2... Δ=-0.0020\n", - " [18/21] layer4.0.downsample.0... Δ=-0.0050\n", - " [19/21] layer4.1.conv1... Δ=+0.0036\n", - " [20/21] layer4.1.conv2... Δ=-0.0001\n", - " [21/21] fc... Δ=-0.5034\n", - "✓ Analysis complete\n" - ] - } - ], - "source": [ - "analyzer = SensitivityAnalyzer(\n", - " model,\n", - " sample,\n", - " eval_fn,\n", - " criteria=large_final, # Importance scoring method\n", - " higher_is_better=True, # Higher metric = better\n", - " metric_name=\"accuracy\", # For display\n", - ")\n", - "\n", - "# Analyze sparsity sensitivity\n", - "sparsity_result = analyzer.analyze(\n", - " compression=\"sparsity\",\n", - " level=50,\n", - " granularity=\"weight\", # or \"filter\", \"kernel\", etc.\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "compression-types-header", - "metadata": {}, - "source": [ - "## 8. Different Compression Types" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "compression-sparsity", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Computing baseline accuracy... 0.6909\n", - "Analyzing 21 layers for sparsity @ 50% (granularity=weight, criteria=abs)\n", - " [1/21] conv1... Δ=+0.0001\n", - " [2/21] layer1.0.conv1... Δ=-0.0006\n", - " [3/21] layer1.0.conv2... Δ=-0.0046\n", - " [4/21] layer1.1.conv1... Δ=-0.0052\n", - " [5/21] layer1.1.conv2... Δ=+0.0020\n", - " [6/21] layer2.0.conv1... Δ=-0.0021\n", - " [7/21] layer2.0.conv2... Δ=-0.0031\n", - " [8/21] layer2.0.downsample.0... Δ=-0.0055\n", - " [9/21] layer2.1.conv1... Δ=-0.0014\n", - " [10/21] layer2.1.conv2... Δ=-0.0021\n", - " [11/21] layer3.0.conv1... Δ=+0.0017\n", - " [12/21] layer3.0.conv2... Δ=+0.0051\n", - " [13/21] layer3.0.downsample.0... Δ=+0.0010\n", - " [14/21] layer3.1.conv1... Δ=+0.0031\n", - " [15/21] layer3.1.conv2... Δ=+0.0005\n", - " [16/21] layer4.0.conv1... Δ=+0.0053\n", - " [17/21] layer4.0.conv2... Δ=-0.0020\n", - " [18/21] layer4.0.downsample.0... Δ=-0.0050\n", - " [19/21] layer4.1.conv1... Δ=+0.0036\n", - " [20/21] layer4.1.conv2... Δ=-0.0001\n", - " [21/21] fc... Δ=-0.5034\n", - "✓ Analysis complete\n" - ] - } - ], - "source": [ - "sparsity_result = analyzer.analyze(\n", - " compression=\"sparsity\",\n", - " level=50, # 50% of weights zeroed\n", - " granularity=\"weight\", # Unstructured sparsity\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "compression-pruning-header", - "metadata": {}, - "source": [ - "### Structural Pruning (Filter Removal)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "compression-pruning", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Computing baseline accuracy... 0.6909\n", - "Analyzing 21 layers for pruning @ 30% (structural, criteria=abs)\n", - " [1/21] conv1... Δ=0.0000\n", - " [2/21] layer1.0.conv1... Δ=-0.0016\n", - " [3/21] layer1.0.conv2... Δ=0.0000\n", - " [4/21] layer1.1.conv1... Δ=-0.0001\n", - " [5/21] layer1.1.conv2... Δ=0.0000\n", - " [6/21] layer2.0.conv1... Δ=-0.0045\n", - " [7/21] layer2.0.conv2... Δ=0.0000\n", - " [8/21] layer2.0.downsample.0... Δ=0.0000\n", - " [9/21] layer2.1.conv1... Δ=-0.0187\n", - " [10/21] layer2.1.conv2... Δ=0.0000\n", - " [11/21] layer3.0.conv1... Δ=+0.0001\n", - " [12/21] layer3.0.conv2... Δ=0.0000\n", - " [13/21] layer3.0.downsample.0... Δ=0.0000\n", - " [14/21] layer3.1.conv1... Δ=-0.0012\n", - " [15/21] layer3.1.conv2... Δ=0.0000\n", - " [16/21] layer4.0.conv1... Δ=-0.0063\n", - " [17/21] layer4.0.conv2... Δ=0.0000\n", - " [18/21] layer4.0.downsample.0... Δ=0.0000\n", - " [19/21] layer4.1.conv1... Δ=0.0000\n", - " [20/21] layer4.1.conv2... Δ=0.0000\n", - " [21/21] fc... Δ=+0.0301\n", - "✓ Analysis complete\n" - ] - } - ], - "source": [ - "pruning_result = analyzer.analyze(\n", - " compression=\"pruning\",\n", - " level=30, # 30% of filters removed\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "compression-quant-header", - "metadata": {}, - "source": [ - "### Quantization (Precision Reduction)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "compression-quant", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Computing baseline accuracy... 0.6909\n", - "Analyzing 21 layers for quantization @ 8bits (per-channel, weights only)\n", - " [1/21] conv1... Δ=-0.0002\n", - " [2/21] layer1.0.conv1... Δ=+0.0003\n", - " [3/21] layer1.0.conv2... Δ=+0.0001\n", - " [4/21] layer1.1.conv1... Δ=-0.0002\n", - " [5/21] layer1.1.conv2... Δ=-0.0003\n", - " [6/21] layer2.0.conv1... Δ=-0.0001\n", - " [7/21] layer2.0.conv2... Δ=-0.0002\n", - " [8/21] layer2.0.downsample.0... Δ=-0.0002\n", - " [9/21] layer2.1.conv1... Δ=-0.0004\n", - " [10/21] layer2.1.conv2... Δ=-0.0000\n", - " [11/21] layer3.0.conv1... Δ=+0.0000\n", - " [12/21] layer3.0.conv2... Δ=+0.0001\n", - " [13/21] layer3.0.downsample.0... Δ=-0.0000\n", - " [14/21] layer3.1.conv1... Δ=-0.0001\n", - " [15/21] layer3.1.conv2... Δ=+0.0002\n", - " [16/21] layer4.0.conv1... Δ=-0.0002\n", - " [17/21] layer4.0.conv2... Δ=-0.0000\n", - " [18/21] layer4.0.downsample.0... Δ=-0.0002\n", - " [19/21] layer4.1.conv1... Δ=+0.0001\n", - " [20/21] layer4.1.conv2... Δ=+0.0000\n", - " [21/21] fc... Δ=-0.0009\n", - "✓ Analysis complete\n" - ] - } - ], - "source": [ - "quant_result = analyzer.analyze(\n", - " compression=\"quantization\",\n", - " level=8, # 8-bit quantization\n", - " quant_per_channel=True, # Per-channel quantization\n", - " quant_activations=False, # Weights only\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "sweep-header", - "metadata": {}, - "source": [ - "## 9. Sweeping Multiple Compression Levels" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "sweep", - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "\n", - "============================================================\n", - "Sweep: sparsity @ 25%\n", - "============================================================\n", - "Computing baseline accuracy... 0.6909\n", - "Analyzing 21 layers for sparsity @ 25% (granularity=weight, criteria=abs)\n", - " [1/21] conv1... Δ=-0.0001\n", - " [2/21] layer1.0.conv1... Δ=-0.0003\n", - " [3/21] layer1.0.conv2... Δ=+0.0005\n", - " [4/21] layer1.1.conv1... Δ=-0.0006\n", - " [5/21] layer1.1.conv2... Δ=+0.0015\n", - " [6/21] layer2.0.conv1... Δ=+0.0010\n", - " [7/21] layer2.0.conv2... Δ=+0.0019\n", - " [8/21] layer2.0.downsample.0... Δ=-0.0014\n", - " [9/21] layer2.1.conv1... Δ=+0.0009\n", - " [10/21] layer2.1.conv2... Δ=+0.0000\n", - " [11/21] layer3.0.conv1... Δ=-0.0012\n", - " [12/21] layer3.0.conv2... Δ=+0.0007\n", - " [13/21] layer3.0.downsample.0... Δ=+0.0008\n", - " [14/21] layer3.1.conv1... Δ=+0.0004\n", - " [15/21] layer3.1.conv2... Δ=-0.0005\n", - " [16/21] layer4.0.conv1... Δ=+0.0010\n", - " [17/21] layer4.0.conv2... Δ=-0.0006\n", - " [18/21] layer4.0.downsample.0... Δ=-0.0011\n", - " [19/21] layer4.1.conv1... Δ=+0.0011\n", - " [20/21] layer4.1.conv2... Δ=-0.0000\n", - " [21/21] fc... Δ=-0.0011\n", - "✓ Analysis complete\n", - "\n", - "============================================================\n", - "Sweep: sparsity @ 50%\n", - "============================================================\n", - "Computing baseline accuracy... 0.6909\n", - "Analyzing 21 layers for sparsity @ 50% (granularity=weight, criteria=abs)\n", - " [1/21] conv1... Δ=+0.0001\n", - " [2/21] layer1.0.conv1... Δ=-0.0006\n", - " [3/21] layer1.0.conv2... Δ=-0.0046\n", - " [4/21] layer1.1.conv1... Δ=-0.0052\n", - " [5/21] layer1.1.conv2... Δ=+0.0020\n", - " [6/21] layer2.0.conv1... Δ=-0.0021\n", - " [7/21] layer2.0.conv2... Δ=-0.0031\n", - " [8/21] layer2.0.downsample.0... Δ=-0.0055\n", - " [9/21] layer2.1.conv1... Δ=-0.0014\n", - " [10/21] layer2.1.conv2... Δ=-0.0021\n", - " [11/21] layer3.0.conv1... Δ=+0.0017\n", - " [12/21] layer3.0.conv2... Δ=+0.0051\n", - " [13/21] layer3.0.downsample.0... Δ=+0.0010\n", - " [14/21] layer3.1.conv1... Δ=+0.0031\n", - " [15/21] layer3.1.conv2... Δ=+0.0005\n", - " [16/21] layer4.0.conv1... Δ=+0.0053\n", - " [17/21] layer4.0.conv2... Δ=-0.0020\n", - " [18/21] layer4.0.downsample.0... Δ=-0.0050\n", - " [19/21] layer4.1.conv1... Δ=+0.0036\n", - " [20/21] layer4.1.conv2... Δ=-0.0001\n", - " [21/21] fc... Δ=-0.5034\n", - "✓ Analysis complete\n", - "\n", - "============================================================\n", - "Sweep: sparsity @ 75%\n", - "============================================================\n", - "Computing baseline accuracy... 0.6909\n", - "Analyzing 21 layers for sparsity @ 75% (granularity=weight, criteria=abs)\n", - " [1/21] conv1... Δ=+0.0020\n", - " [2/21] layer1.0.conv1... Δ=-0.0142\n", - " [3/21] layer1.0.conv2... Δ=-0.0076\n", - " [4/21] layer1.1.conv1... Δ=-0.0119\n", - " [5/21] layer1.1.conv2... Δ=-0.0053\n", - " [6/21] layer2.0.conv1... Δ=-0.0100\n", - " [7/21] layer2.0.conv2... Δ=-0.0100\n", - " [8/21] layer2.0.downsample.0... Δ=-0.0090\n", - " [9/21] layer2.1.conv1... Δ=-0.0107\n", - " [10/21] layer2.1.conv2... Δ=-0.0007\n", - " [11/21] layer3.0.conv1... Δ=-0.0200\n", - " [12/21] layer3.0.conv2... Δ=+0.0011\n", - " [13/21] layer3.0.downsample.0... Δ=+0.0008\n", - " [14/21] layer3.1.conv1... Δ=+0.0050\n", - " [15/21] layer3.1.conv2... Δ=-0.0010\n", - " [16/21] layer4.0.conv1... Δ=+0.0087\n", - " [17/21] layer4.0.conv2... Δ=-0.0084\n", - " [18/21] layer4.0.downsample.0... Δ=-0.0028\n", - " [19/21] layer4.1.conv1... Δ=+0.0074\n", - " [20/21] layer4.1.conv2... Δ=-0.0006\n", - " [21/21] fc... Δ=-2.6806\n", - "✓ Analysis complete\n" - ] - } - ], - "source": [ - "results = analyzer.sweep(\n", - " compression=\"sparsity\",\n", - " levels=[25, 50, 75],\n", - ")" - ] - }, - { - "cell_type": "markdown", - "id": "dataframe-header", - "metadata": {}, - "source": [ - "## 10. Exporting Results" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "dataframe", - "metadata": {}, - "outputs": [ - { - "data": { - "text/html": [ - "
\n", - "\n", - "\n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - " \n", - "
namelayer_typeparamsbaseline_metriccompressed_metricdelta
0conv1Conv2d94081.1848461.1784270.006419
1layer1.0.conv1Conv2d368641.1848461.283498-0.098653
2layer1.0.conv2Conv2d368641.1848461.249760-0.064914
3layer1.1.conv1Conv2d368641.1848461.262704-0.077859
4layer1.1.conv2Conv2d368641.1848461.1200000.064846
\n", - "
" - ], - "text/plain": [ - " name layer_type params baseline_metric compressed_metric \\\n", - "0 conv1 Conv2d 9408 1.184846 1.178427 \n", - "1 layer1.0.conv1 Conv2d 36864 1.184846 1.283498 \n", - "2 layer1.0.conv2 Conv2d 36864 1.184846 1.249760 \n", - "3 layer1.1.conv1 Conv2d 36864 1.184846 1.262704 \n", - "4 layer1.1.conv2 Conv2d 36864 1.184846 1.120000 \n", - "\n", - " delta \n", - "0 0.006419 \n", - "1 -0.098653 \n", - "2 -0.064914 \n", - "3 -0.077859 \n", - "4 0.064846 " - ] - }, - "execution_count": null, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "df = result.to_dataframe()\n", - "df.head()" - ] - }, - { - "cell_type": "markdown", - "id": "summary-header", - "metadata": {}, - "source": [ - "---\n", - "\n", - "## Summary\n", - "\n", - "| Function/Method | Purpose |\n", - "|-----------------|--------|\n", - "| `analyze_sensitivity()` | One-line sensitivity analysis |\n", - "| `SensitivityAnalyzer` | Full control over analysis |\n", - "| `result.summary()` | Print formatted summary |\n", - "| `result.top(n, most_sensitive)` | Get top N layers |\n", - "| `result.plot()` | Visualize sensitivity |\n", - "| `result.to_layer_targets()` | Generate non-uniform per-layer compression targets |\n", - "| `result.to_dataframe()` | Export to pandas |\n", - "| `analyzer.sweep()` | Test multiple compression levels |" - ] - }, - { - "cell_type": "markdown", - "id": "see-also", - "metadata": {}, - "source": [ - "---\n", - "\n", - "## See Also\n", - "\n", - "- [Sparsifier](../sparse/sparsifier.html) - Apply sparsity to models\n", - "- [Pruner](../../prune/pruner.html) - Structural pruning\n", - "- [Criteria](../../core/criteria.html) - Importance scoring methods\n", - "- [Schedules](../../core/schedules.html) - Gradual compression schedules" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "python3", - "language": "python", - "name": "python3" - } - }, - "nbformat": 4, - "nbformat_minor": 5 -}