diff --git a/playbooks/ucm-config-analyzer/APPHUB.yaml b/playbooks/ucm-config-analyzer/APPHUB.yaml new file mode 100644 index 0000000..9a541fd --- /dev/null +++ b/playbooks/ucm-config-analyzer/APPHUB.yaml @@ -0,0 +1,126 @@ +# APPHUB.yaml — Playbook metadata for Webex App Hub +# Copy this file into your Playbook folder under playbooks// +# Fill in all required fields. See CONTRIBUTING.md for field rules. + +# ----------------------------------------------------------------------------- +# friendly_id — Unique identifier. Must end with -playbook (e.g. epic-ehr-playbook) +# ----------------------------------------------------------------------------- +friendly_id: "ucm-config-analyzer-playbook" + +# ----------------------------------------------------------------------------- +# title — Display name for the Playbook (matches ContentStack field) +# ----------------------------------------------------------------------------- +title: "UCM Config Analyzer + Webex Calling Migration" + +# ----------------------------------------------------------------------------- +# tag_line — Short tagline for App Hub detail page (required, max 128 chars) +# ----------------------------------------------------------------------------- +tag_line: "Analyze Cisco UCM configuration exports to identify migration clusters before moving to Webex Calling" + +# ----------------------------------------------------------------------------- +# description — App Hub supports Markdown. Use a block scalar (description: |) with +# blank lines between sections: opening paragraph (bold key terms), **Why use this +# playbook** (3–5 outcome bullets), **What it does** (concrete behaviors/endpoints). +# Do not put upstream repo URLs or install-only instructions here—use README and +# src/README.md for reference playbooks. See docs/commands/import_playbook.md. +# ----------------------------------------------------------------------------- +description: | + A **Python** tool that parses **Cisco UCM** bulk-export TAR files offline and + produces interactive **Plotly** visualizations to guide your **Webex Calling** + migration: user dependency clusters, phone readiness, and dial-plan analysis. + + **Why use this playbook** + + - **Know before you migrate:** Identify tightly coupled users — shared lines, + shared phones, hunt groups, BLF subscriptions — so you can sequence migration + waves without breaking calling workflows. + - **Device gap in minutes:** Get a supported-vs-unsupported **Cisco phone** breakdown + from your actual export data before ordering or planning replacements. + - **Dial plan clarity:** Visualize **translation patterns**, **CSS combinations**, + and **route patterns** as interactive charts instead of raw CSV rows. + - **No live system access needed:** Runs entirely offline against a standard **UCM + BAT export**; no AXL credentials or network access to UCM are required. + - **Scriptable foundation:** The `ucmexport.Proxy` class exposes typed containers for + every UCM object type, making it easy to write custom migration queries. + + **What it does** + + - Parses **UCM BAT export** TAR archives into typed Python objects via the + `ucmexport` library (phones, end users, hunt pilots, DNs, CSS, locations, etc.). + - Builds a **user dependency graph** using **NetworkX** — connects users via shared + phones, shared lines, hunt pilots, BLF subscriptions, and call pickup groups. + - Renders **Plotly** interactive sunburst, Sankey, and 3D graph charts in the local + browser for migration readiness review. + - Provides a CLI utility to **strip sensitive columns** from export CSVs + (`transform_tar.py`) before sharing archives across teams. + +# ----------------------------------------------------------------------------- +# product_types — Where this Playbook appears. Pick one or more. +# Valid: teams | meetings | calling | rooms | contact_center +# ----------------------------------------------------------------------------- +product_types: + - "calling" + +# ----------------------------------------------------------------------------- +# categories — App Hub category slugs. Pick one or more. +# Verticals: healthcare | financial-services | retail-ecommerce +# App categories (use kebab-case slugs): +# ai-agent-testing-observability | agent-supervisor-tools | analytics | +# calendar-scheduling | collaboration-management | customer-relations | +# customer-support | developer-tools | doc-management | education | +# finance | government | healthcare | human-resources | internet-of-things | +# marketing-sales | orchestration | platform | productivity | +# project-management | recording-transcriptions | security-compliance | +# self-service-bots | social-and-fun | strategy-team-planning | +# workflow-automation | workforce-optimization | other +# ----------------------------------------------------------------------------- +categories: + - "developer-tools" + - "workflow-automation" + +# ----------------------------------------------------------------------------- +# company_name — Your company or team name +# ----------------------------------------------------------------------------- +company_name: "Webex for Developers" + +# ----------------------------------------------------------------------------- +# company_url — Your company or project URL +# ----------------------------------------------------------------------------- +company_url: "https://developer.webex.com" + +# ----------------------------------------------------------------------------- +# support_url — Issues or support link (e.g. GitHub issues) +# ----------------------------------------------------------------------------- +support_url: "https://github.com/webex/WebexPlaybooks/issues" + +# ----------------------------------------------------------------------------- +# product_url — Link to this Playbook in the repo (required) +# ----------------------------------------------------------------------------- +product_url: "https://github.com/webex/WebexPlaybooks/tree/main/playbooks/ucm-config-analyzer" + +# ----------------------------------------------------------------------------- +# logo — (Optional) URL to your logo image. If not provided, defaults to the +# standard Webex Playbook logo. +# ----------------------------------------------------------------------------- +logo: "https://images.contentstack.io/v3/assets/bltd14fd2a03236233f/blta2de9daa773c6604/60f71f81e2de935fc7e35dbe/download" + +# ----------------------------------------------------------------------------- +# estimated_implementation_time — e.g. "2-4 hours", "1 day" +# ----------------------------------------------------------------------------- +estimated_implementation_time: "2-4 hours" + +# ----------------------------------------------------------------------------- +# third_party_tool — (Optional) The tool being integrated (e.g. Salesforce, Epic) +# Omit for generic playbooks (e.g. "any CMS") +# ----------------------------------------------------------------------------- +third_party_tool: "Cisco UCM" + +# ----------------------------------------------------------------------------- +# privacy_url — Privacy policy URL (required; use Cisco default for Webex-authored) +# ----------------------------------------------------------------------------- +privacy_url: "https://www.cisco.com/c/en/us/about/legal/privacy-full.html" + +# ----------------------------------------------------------------------------- +# submission_date — (Optional) ISO date (e.g. 2025-03-01) +# ----------------------------------------------------------------------------- +submission_date: "2026-04-02" diff --git a/playbooks/ucm-config-analyzer/README.md b/playbooks/ucm-config-analyzer/README.md new file mode 100644 index 0000000..69c899f --- /dev/null +++ b/playbooks/ucm-config-analyzer/README.md @@ -0,0 +1,172 @@ +# UCM Config Analyzer for Webex Calling Migration + +This Playbook is adapted from the [ucmmigration](https://github.com/jeokrohn/ucmmigration) sample on GitHub. + +## Use Case Overview + +Organizations preparing to migrate from Cisco Unified Communications Manager (UCM) to **Webex Calling** often face a discovery problem: they do not have a clear picture of what is in their UCM deployment before migration planning begins. Users share lines, phones are shared across workers, hunt groups cross team boundaries, and dial plans carry years of accumulated complexity — all of which affect how users must be grouped and sequenced during migration. + +This Playbook packages the **UCM Config Analyzer**, a Python tool that reads a standard UCM **Bulk Administration Tool (BAT) export** TAR file and produces interactive visualizations and analysis reports. A calling administrator or migration engineer can run it against their own export data in under an hour to answer: + +- Which users are tightly coupled (shared lines, shared phones, BLF pickup, hunt groups) and must migrate together? +- Which phone models are supported by Webex Calling, and how many need replacement? +- What does the dial plan look like (translation patterns, route patterns, CSS combinations)? +- Where are the location clusters in the network? + +**Target persona:** Calling administrator or UC migration engineer preparing a Cisco UCM-to-Webex Calling migration. + +**Estimated implementation time:** 2–4 hours (tool setup and initial analysis run; deeper dial-plan interpretation may take additional time). + +## Architecture + +The UCM Config Analyzer operates entirely **offline** — it does not connect to a live UCM server or to any Webex API. The workflow is: + +1. An administrator exports the UCM configuration using the **Bulk Administration Tool (BAT)** within CUCM, producing a `.tar` archive. +2. The archive is placed in the project directory on the analyst's workstation. +3. The Python tool parses the CSV files inside the archive through the `ucmexport` library. +4. The `App` module runs an interactive CLI menu of analysis functions. +5. Results are displayed as console output and **Plotly** interactive browser charts. +6. The analyst uses the output to build a Webex Calling migration plan. + +See the Mermaid diagram in [/diagrams/architecture-diagram.md](diagrams/architecture-diagram.md) for a visual overview. + +The `ucmexport` library supports the following UCM object types: phones, end users, directory numbers, CSS, device pools, hunt pilots, hunt lists, line groups, call park, call pickup groups, route patterns, translation patterns, device profiles, locations, and remote destinations. + +## Prerequisites + +**Cisco UCM requirements:** +- Access to a Cisco UCM deployment with **Bulk Administration Tool (BAT)** access (typically requires CCM Admin or BAT Admin role). +- Ability to perform a full UCM configuration export — this generates a `.tar` file containing multiple CSV files (phones, end users, directory numbers, etc.). +- UCM 10.x or later is recommended; the tool was built against UCM CSV export formats current as of UCM 12.x. + +**Webex requirements:** +- No Webex account is required to run the analysis. Webex Calling is the migration *target* — you will use the output of this tool to plan your Webex Calling onboarding. +- To act on the analysis output, you will need a Webex Calling organization with the appropriate licenses (Webex Calling — Professional or similar) and access to Control Hub. + +**Developer environment:** +- Python 3.8 or later (tested smoke import on **Python 3.13**) +- `pip` package manager +- The playbook’s `requirements.txt` bumps **numpy** and **scipy** slightly above the upstream pins so **Python 3.13** can use prebuilt wheels (upstream `numpy==2.0.2` may try to compile from source and fail on 3.13). +- A virtual environment tool (`venv`, `virtualenv`, or `virtualenvwrapper`) — strongly recommended to isolate dependencies +- A modern browser (for Plotly interactive visualizations) +- Approximately 500 MB of disk space for Python packages (numpy, scipy, matplotlib, plotly) + +**Network/firewall:** +- No inbound connectivity required — the tool is fully local. +- Plotly renders charts in the local browser; no data is sent to Plotly's servers in the default offline-rendering mode. + +## Code Scaffold + +The source code under `/src/` is vendored directly from the upstream repository and demonstrates how to parse UCM bulk export archives and drive analysis visualizations. + +**Entry points:** + +| File | Purpose | +|------|---------| +| `src/main.py` | Primary entry point. Discovers all `*.tar` files in the current directory, loads them via `App`, and starts the interactive menu. | +| `src/simple.py` | Minimal one-file demo: loads `sample.tar`, counts phones, and prints phones with multiple lines. Good starting point for scripting custom queries. | + +**Core library — `src/ucmexport/`:** + +- `proxy/__init__.py` — `Proxy` class: the main facade. Instantiate with a TAR filename; exposes typed container attributes (`phones`, `end_user`, `hunt_pilot`, `translation_pattern`, etc.). +- `objects/` — One file per UCM object type (`phone.py`, `enduser.py`, `directorynumber.py`, etc.), each defining a container and a typed model class. + +**Analysis modules:** + +- `src/app/__init__.py` — `App` class with an interactive text menu (~840 lines). Key analyses: user dependency graph construction, supported vs. unsupported device breakdown, phone/user consistency, DN/dial-plan Sankey, CSS combinations, translation pattern treemap, abbreviated dialing, hunt group dump. +- `src/digit_analysis/` — Digit tree analysis: builds and traverses dial-plan node trees (`DaNode`) to evaluate routing logic. +- `src/user_dependency_graph/` — `UserGraph` class: constructs a NetworkX graph of user coupling via shared phones, shared lines, hunt pilots, BLF subscriptions, and call pickup groups. Used to identify migration clusters. + +**Utility scripts:** + +| File | Purpose | +|------|---------| +| `src/transform_tar.py` | CLI tool to strip noisy/sensitive columns from `phone.csv` and `enduser.csv` inside a TAR and write a cleaned `*_transformed.tar`. Useful for reducing export size before analysis. | +| `src/type_user_association.py` | Reads `enduser.csv` from TAR files and prints all distinct `TYPE USER ASSOCIATION` values across one or more exports. | +| `src/reduce_tar.py` | Reduces a TAR archive to a smaller subset; useful for testing with large production exports. | + +**What the code does NOT do:** +- It does not connect to a live UCM server or call the UCM AXL API. +- It does not call any Webex API — migration execution is out of scope. +- It is not production-hardened; it is an analysis sandbox intended for use on analyst workstations. +- Secrets and credentials are not required for the analysis phase, but `src/env.template` documents the `TAR_FILE` variable used by `type_user_association.py`. + +For upstream context and additional notes, see [docs/upstream-overview.md](docs/upstream-overview.md). + +## Deployment Guide + +### Part A — Export your UCM configuration + +1. Log in to your Cisco UCM Administration interface. +2. Navigate to **Bulk Administration → Export Configuration**. +3. Select all object types you want to analyze (recommended: phones, end users, directory numbers, hunt pilots, hunt lists, line groups, CSS, route patterns, translation patterns, device pools, locations). +4. Click **Export** and wait for the job to complete under **Bulk Administration → Job Scheduler**. +5. Download the resulting `.tar` file to your analyst workstation. + + + +### Part B — Set up the Python environment + +6. Install Python 3.8 or later from [python.org](https://www.python.org/downloads/) if not already present. Confirm with: + ```bash + python3 --version + ``` +7. Clone or download this playbook's `src/` directory to a working folder on your workstation: + ```bash + git clone https://github.com/webex/WebexPlaybooks.git + cd WebexPlaybooks/playbooks/ucm-config-analyzer/src + ``` +8. Create and activate a Python virtual environment: + ```bash + python3 -m venv .venv + source .venv/bin/activate # macOS/Linux + .venv\Scripts\activate.bat # Windows + ``` +9. Install all required packages: + ```bash + pip install -r requirements.txt + ``` +10. Optional — verify the environment (no UCM `.tar` needed): + ```bash + python -c "from ucmexport import Proxy; from app import App; from user_dependency_graph import UserGraph; import digit_analysis; print('All core imports OK')" + ``` + +### Part C — Run the analysis + +11. Copy the UCM export `.tar` file(s) into the `src/` directory (the same directory as `main.py`). The tool automatically discovers all `*.tar` files in the current working directory. +12. Run the main analysis tool: + ```bash + python main.py + ``` +13. The tool will log startup information and then display an interactive text menu. Use the numbered options to: + - Select which TAR file to analyze (if multiple are present) + - Toggle user dependency relation types (hunt pilots, shared phones, shared lines, BLF, call pickup groups) + - Run individual analyses (user clusters, device type breakdown, dial plan Sankey, translation pattern treemap, etc.) +14. For each visualization option, a Plotly chart will open in your default browser. Close the browser tab to return to the menu. + +### Part D — Quick scripted query (optional) + +15. To run a minimal custom query without the interactive menu, edit `simple.py` and replace `'sample.tar'` with your TAR filename, then run: + ```bash + python simple.py + ``` + This demonstrates how to use the `Proxy` class directly for scripted analysis. + +### Part E — Clean and slim the TAR before analysis (optional) + +16. If the export is very large or contains sensitive fields you want to strip before sharing, use `transform_tar.py`: + ```bash + python transform_tar.py + ``` + This writes a `_transformed.tar` with noisy columns removed from `phone.csv` and `enduser.csv`. + +## Known Limitations + +- **Offline analysis only:** The tool reads static UCM export TAR files. It does not connect to a live UCM server, does not use the AXL API, and does not call any Webex API. Migration execution requires separate tooling (see the [UCM to Webex Calling Migration playbook](../ucm-to-wxc-migration/README.md) for migration execution). +- **Export format dependency:** The parser is tuned to UCM BAT export CSV formats from UCM 10.x–12.x. Column name changes in newer UCM releases may require updates to the object models in `ucmexport/objects/`. +- **Python 3.8 baseline:** The upstream code targets Python 3.8. It runs on 3.9+ in practice (including **3.13** with the playbook’s `numpy`/`scipy` pins), but type annotations use older union syntax (`Optional[X]`) and some walrus operator patterns. Test against your Python version before use. +- **No authentication or credential handling:** This tool requires no credentials to run. The `TAR_FILE` environment variable in `env.template` is only used by the `type_user_association.py` utility script. +- **Large exports:** Very large UCM deployments (tens of thousands of phones) may take several minutes to parse and may require significant RAM for NetworkX graph operations. The `reduce_tar.py` and `transform_tar.py` utilities can help reduce export size for initial testing. +- **Plotly display:** Visualizations open in the system default browser. In headless or server environments, `fig.show()` may fail; redirect to `fig.write_html()` for file output. +- **No license file in upstream repo:** The upstream code at [jeokrohn/ucmmigration](https://github.com/jeokrohn/ucmmigration) is provided "as is" with no explicit open-source license. Review the upstream repository before any commercial or redistributed use. +- **Standard Webex disclaimer:** This Playbook is provided as a starting point. Webex does not guarantee the functional accuracy of the source code. Test thoroughly before use in a production environment. diff --git a/playbooks/ucm-config-analyzer/diagrams/architecture-diagram.md b/playbooks/ucm-config-analyzer/diagrams/architecture-diagram.md new file mode 100644 index 0000000..a161649 --- /dev/null +++ b/playbooks/ucm-config-analyzer/diagrams/architecture-diagram.md @@ -0,0 +1,49 @@ +# Architecture — UCM Config Analyzer and Webex Calling migration + +This diagram shows how the offline **UCM Config Analyzer** fits into a **Webex Calling** migration planning workflow. No live Webex or UCM API calls are made during analysis. + +```mermaid +flowchart LR + subgraph ucmSide [UCM Administrator] + Admin[UCM Admin] + end + + subgraph export [Offline export] + BAT[UCM BAT export job] + TarFile[".tar archive CSV files"] + end + + subgraph analyzer [Analyst workstation] + MainPy[main.py] + Proxy[ucmexport.Proxy] + AppMod[app.App menu] + Graph[user_dependency_graph] + Digit[digit_analysis] + Plotly[Plotly charts] + end + + subgraph planning [Migration planning] + Report[Console reports and visualizations] + Plan[Webex Calling migration waves] + end + + subgraph webex [Webex Calling target] + WxC[Control Hub Webex Calling] + end + + Admin --> BAT + BAT --> TarFile + TarFile --> MainPy + MainPy --> Proxy + Proxy --> AppMod + AppMod --> Graph + AppMod --> Digit + AppMod --> Plotly + Graph --> Plotly + Digit --> Plotly + Plotly --> Report + Report --> Plan + Plan --> WxC +``` + +**Authentication:** The analyzer needs no tokens. UCM authentication applies only when the admin runs the BAT export in CUCM. Webex Control Hub authentication applies later when executing the migration itself. diff --git a/playbooks/ucm-config-analyzer/docs/upstream-overview.md b/playbooks/ucm-config-analyzer/docs/upstream-overview.md new file mode 100644 index 0000000..79a386e --- /dev/null +++ b/playbooks/ucm-config-analyzer/docs/upstream-overview.md @@ -0,0 +1,37 @@ +# Upstream repository notes + +This playbook's `/src/` code is adapted from [jeokrohn/ucmmigration](https://github.com/jeokrohn/ucmmigration) (Python 3.8). + +## Original project description + +The upstream README states the project is a playground to analyze UCM configuration data from TAR files containing a UCM config export. Code is provided as-is. + +## Installing dependencies (upstream) + +1. Install Python from [python.org](https://www.python.org). +2. Create and activate a virtual environment ([venv tutorial](https://docs.python.org/3/tutorial/venv.html)). +3. Run `pip install -r requirements.txt` from the directory containing `requirements.txt`. + +## Running the main application (upstream) + +- Place one or more UCM bulk-export `.tar` files in the working directory. +- Run `python main.py`. The tool discovers all `*.tar` files in the current directory and starts the interactive `App` menu. + +## Running the simple example (upstream) + +- Ensure `sample.tar` (or edit the filename in `simple.py`) exists in the working directory. +- Run `python simple.py`. + +## Additional utilities (upstream) + +- `transform_tar.py` — Strip selected columns from `phone.csv` and `enduser.csv` inside a TAR; writes `*_transformed.tar`. +- `type_user_association.py` — Scans `enduser.csv` in each local `*.tar` for distinct `TYPE USER ASSOCIATION` values (see `env.template` for `main()` mode). +- `reduce_tar.py` — Reduce TAR size for testing. + +## Pipenv (upstream) + +The upstream repo includes `Pipfile` and `Pipfile.lock` for users who prefer Pipenv. This playbook standardizes on `requirements.txt` only. + +## License + +The upstream repository does not include a `LICENSE` file. Verify licensing with the upstream maintainer before redistribution. diff --git a/playbooks/ucm-config-analyzer/src/app/__init__.py b/playbooks/ucm-config-analyzer/src/app/__init__.py new file mode 100644 index 0000000..2207b9c --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/app/__init__.py @@ -0,0 +1,917 @@ +from ucmexport import * +from user_dependency_graph import UserGraph +import logging +from typing import List, Dict, Set, Tuple, Iterable, Optional +from itertools import chain +from collections import defaultdict, Counter, namedtuple +import plotly.graph_objects as go +import plotly.express as px +import digit_analysis + +__all__ = ['App'] + +log = logging.getLogger(__name__) + +""" +Aspects to consider to identify groups: + shared line - X + line group - X + hunt list - X + directed/indirect park + intercom (phones with a phone button template that has a intercom line) + IPMA + blf - X + Call-Park; All phones with access to call park partition can park calls and all users who can dial that number + range (have access to the same partition) can pick up a acall + call pickup (within group), DNs have a call pickup group asignment + call pickup outside group + user +""" + +SUPPORTED_DEVICES = {'Cisco 8831', 'Cisco 8841', 'Cisco 8851', 'Cisco 8861', 'Cisco 8821', 'Cisco 8811', 'Cisco 8845', + 'Cisco 8865', 'Cisco 8851NR', 'Cisco 8865NR', 'Cisco 8832', 'Cisco 8832NR'} + + +class SunBurstHelper: + def __init__(self): + self.ids = [] + self.labels = [] + self.parents = [] + self.values = [] + + def add_entry(self, parent_id, label, value=1) -> int: + new_id = len(self.ids) + self.ids.append(new_id) + self.labels.append(label) + self.parents.append(parent_id) + self.values.append(value) + return new_id + + def add_phone(self, parent_id: int, phone: Phone, phone_childs): + phone_id = self.add_entry(parent_id=parent_id, label=phone.device_name) + if phone_childs: + phone_childs(self, phone_id, phone) + + def add_phones(self, parent_id: int, phones: Iterable[Phone], details=False, phone_childs=None): + phones_by_type: Dict[str, List[Phone]] = defaultdict(list) + for phone in phones: + phones_by_type[phone.device_type].append(phone) + phone_types = sorted(phones_by_type, key=lambda pt: len(phones_by_type[pt]), reverse=True) + for phone_type in phone_types: + phone_type_id = self.add_entry(parent_id=parent_id, label=phone_type, + value=len(phones_by_type[phone_type])) + if details: + for phone in sorted(phones_by_type[phone_type], key=lambda x: x.device_name): + self.add_phone(parent_id=phone_type_id, phone=phone, phone_childs=phone_childs) + return + + def fig(self): + fig = go.Figure(go.Sunburst( + ids=self.ids, + labels=self.labels, + parents=self.parents, + values=self.values + )) + return fig + + +class App: + class menu_register: + """" + Decorate methods as menu items + """ + menu_items = [] + + def __init__(self, text): + self.text = text + + def __call__(self, f): + self.menu_items.append((self.text, f)) + return f + + def menu_quit(self): + self.stopped = True + + def run(self): + self.menu_register.menu_items.append(('Quit', App.menu_quit)) + while not self.stopped: + # print menu + print(f'selected tar file: {self.tar_files[self.tar_file_index]}') + user_grouping = ", ".join(self.USER_RELATION_TEXT[r] + for r in sorted(self.phone_relations_check)) + print(f'user grouping based on: {user_grouping}') + print(f'only new relations: {self.only_new_relations}') + for i, (text, _) in enumerate(self.menu_register.menu_items, 1): + print(f'{i}) {text}') + + # get user input + # noinspection PyUnboundLocalVariable + choice = input(f'Enter your choice [1-{i}] : ') + try: + choice = int(choice) + except ValueError: + continue + if choice < 1 or choice > i: + continue + + # execute the registered method + self.menu_register.menu_items[choice - 1][1](self) + print() + print() + return + + # device types to ignore when looking at user assignments + ANONYMOUS_DEVICE_TYPES = { + 'Cisco VGC Phone', + 'CTI Port', + 'Cisco ATA 186', + 'Analog Phone', + 'Cisco ATA 187', + 'Cisco Cius', + 'Universal Device Template'} + + USER_RELATION_SHARED_PHONES = 0 + USER_RELATION_SHARED_LINES = 1 + USER_RELATION_HUNT_PILOT = 2 + USER_RELATION_BLF = 3 + USER_RELATION_CPG = 4 + USER_RELATION_ALL = {USER_RELATION_SHARED_PHONES, + USER_RELATION_SHARED_LINES, + USER_RELATION_HUNT_PILOT, + USER_RELATION_BLF, + USER_RELATION_CPG} + USER_RELATION_TEXT = { + USER_RELATION_SHARED_PHONES: 'shared phones', + USER_RELATION_SHARED_LINES: 'shared lines', + USER_RELATION_HUNT_PILOT: 'hunt pilots', + USER_RELATION_BLF: 'blfs', + USER_RELATION_CPG: 'call pickup groups' + } + + def __init__(self, tar_files: List[str]): + self.stopped = False + self.tar_files = tar_files + self.tar_file_index = 0 + self.proxy = Proxy(tar=self.tar_files[self.tar_file_index]) + self.phone_relations_check = self.USER_RELATION_ALL + self.only_new_relations = False + self.user_graph: Optional[UserGraph] = None + + @menu_register('Switch tar file') + def menu_switch_tar_file(self): + self.user_graph = None + self.tar_file_index += 1 + if self.tar_file_index >= len(self.tar_files): + self.tar_file_index = 0 + self.proxy = Proxy(tar=self.tar_files[self.tar_file_index]) + + @menu_register('Toggle "only new relations"') + def toggle_only_new_relations(self): + self.only_new_relations = not self.only_new_relations + + def toggle_check(self, check): + self.user_graph = None + if check in self.phone_relations_check: + self.phone_relations_check.remove(check) + else: + self.phone_relations_check.add(check) + + @menu_register('Toggle "shared phones"') + def menu_toggle_shared_phones(self): + self.toggle_check(self.USER_RELATION_SHARED_PHONES) + + @menu_register('Toggle "shared lines"') + def menu_toggle_shared_lines(self): + self.toggle_check(self.USER_RELATION_SHARED_LINES) + + @menu_register('Toggle "hunt pilot"') + def menu_toggle_hunt_pilot(self): + self.toggle_check(self.USER_RELATION_HUNT_PILOT) + + @menu_register('Toggle "blf"') + def menu_toggle_blf(self): + self.toggle_check(self.USER_RELATION_BLF) + + @menu_register('Toggle "call pickup groups"') + def menu_toggle_cpg(self): + self.toggle_check(self.USER_RELATION_CPG) + + def assert_user_graph(self): + """ + Assert that the user graph has been created. + """ + if self.user_graph is not None: + return + self.user_graph = UserGraph() + if self.USER_RELATION_HUNT_PILOT in self.phone_relations_check: + # Now let's look at all hunt pilots, and link all users of phones with dns in any hunt-list + users_added = self.user_graph.related_users_hunt_pilot(proxy=self.proxy) + print(f'{users_added} based on hunt pilots') + + if self.USER_RELATION_SHARED_PHONES in self.phone_relations_check: + # Shared phones + users_added = self.user_graph.related_users_shared_phones(self.proxy, + only_new_relations=self.only_new_relations) + print(f'{users_added} users based on shared phones') + + if self.USER_RELATION_SHARED_LINES in self.phone_relations_check: + # we now want to check all dns of all phones and if that dn might be shared with phones belonging + # to other users + users_added = self.user_graph.related_users_shared_lines(self.proxy, + only_new_relations=self.only_new_relations) + print(f'{users_added} based on shared lines') + + # also users with BLFs on other users' DNs are related + if self.USER_RELATION_BLF in self.phone_relations_check: + users_added = self.user_graph.related_users_blf(proxy=self.proxy, + only_new_relations=self.only_new_relations) + print(f'{users_added} user relations based on BLF') + + if self.USER_RELATION_CPG in self.phone_relations_check: + users_added = self.user_graph.related_users_cpg(proxy=self.proxy, + only_new_relations=self.only_new_relations) + print(f'{users_added} user relations based on CPG') + self.user_graph.simplify() + + @menu_register('User dependency') + def menu_user_dependency(self): + self.user_graph = None + self.assert_user_graph() + # print stats about clusters + print() + print('Summary:') + related_users = self.user_graph.related_users_by_len() + for cluster_len in sorted(related_users): + print(f'{len(related_users[cluster_len])} groups of users with {cluster_len} users each') + print(f'total: {sum(map(len, related_users.values()))} groups') + no_users = sum(sum(len(cluster) + for cluster in clusters) + for clusters in related_users.values()) + print(f'total: {no_users} users') + + @menu_register('Draw user dependency') + def menu_draw_user_clusters(self): + """ + Create a graph of users + :return: None + """ + self.assert_user_graph() + self.user_graph.draw() + + @menu_register('Draw 3D user dependency') + def menu_draw_user_clusters3d(self): + """ + Create a graph of users + :return: None + """ + self.assert_user_graph() + self.user_graph.draw3d() + + @menu_register('Phones without lines') + def menu_phones_no_lines(self): + phones_no_lines = [phone for phone in self.proxy.phones.list if len(phone.lines) == 0] + device_types = Counter(phone.device_type for phone in phones_no_lines) + print('Phones w/o lines') + ll = max(len(f'{c}') for c in device_types.values()) + dtl = max(len(dt) for dt in device_types) + for device_type in sorted(device_types): + output = ", ".join(phone.device_name for phone in phones_no_lines if phone.device_type == device_type) + print(f'{device_types[device_type]:{ll}} x {device_type:{dtl}}: {output}') + + @menu_register('Device type stats') + def menu_device_type_stats(self): + device_type_counter = Counter(p.device_type for p in self.proxy.phones.list) + device_types = list(device_type_counter) + device_types.sort() + max_len = max(len(dt) for dt in device_types) + for device_type in device_types: + print(f'{device_type:{max_len}}: {device_type_counter[device_type]:5} ' + f'{"supported" if device_type in SUPPORTED_DEVICES else "unsupported"}') + print('Supported/unsupported classification only based on device type') + + @menu_register('Supported vs. unsupported phones for migration') + def menu_supported_phones(self): + phones = self.proxy.phones.list + # we can ignore phones with certain device types + relevant_phones = [phone for phone in phones if phone.device_type not in self.ANONYMOUS_DEVICE_TYPES] + + sb = SunBurstHelper() + sb_data = { + 'supported': defaultdict(list), + 'unsupported': defaultdict(list) + } + for phone in relevant_phones: + if phone.device_type in SUPPORTED_DEVICES: + sb_key = 'supported' + else: + sb_key = 'unsupported' + sb_data[sb_key][phone.device_type].append(phone.device_name) + for sb_key in sb_data: + parent = sb.add_entry(parent_id='', label=sb_key, value=len(sb_data[sb_key])) + for device_type in sb_data[sb_key]: + device_parent = sb.add_entry(parent_id=parent, label=device_type, + value=len(sb_data[sb_key][device_type])) + for device_name in sb_data[sb_key][device_type]: + sb.add_entry(parent_id=device_parent, label=device_name) + if len(sb.ids) > 10000: + print(f'figure with {len(sb.ids)} nodes might be hard to render...') + fig = sb.fig() + fig.update_layout(margin=dict(t=0, l=0, r=0, b=0)) + fig.show() + + @menu_register('Analyze phone/user assignment') + def menu_analyze_phone_user_assignment(self): + # phones have owners and USER ID X (x=1...N) + phones = self.proxy.phones.list + + relevant_phones = [] + ignored_phones = [] + for phone in phones: + if phone.device_type in self.ANONYMOUS_DEVICE_TYPES: + ignored_phones.append(phone) + else: + relevant_phones.append(phone) + + sb = SunBurstHelper() + phones_id = sb.add_entry(parent_id='', label='Phones') + relevant_id = sb.add_entry(parent_id=phones_id, label='relevant', value=len(relevant_phones)) + ignored_id = sb.add_entry(parent_id=phones_id, label='ignored', value=len(ignored_phones)) + sb.add_phones(ignored_id, ignored_phones) + + # we can ignore phones with certain device types + relevant_phones = [phone for phone in phones if phone.device_type not in self.ANONYMOUS_DEVICE_TYPES] + + # classifications per phone + # '0': no user + # '1': exactly one user + # 'n': more than one user + # 'u': at least unknown user + phone_classification: Dict[str, Set[str]] = defaultdict(set) + for phone in relevant_phones: + device_name = phone.device_name + users = phone.user_set + if len(users) == 0: + phone_classification[device_name].add('0') + elif len(users) == 1: + phone_classification[device_name].add('1') + else: + phone_classification[device_name].add('n') + # see if all users ids are valid + if any((self.proxy.end_user.get(u) is None) for u in users): + phone_classification[device_name].add('u') + print(f'# of phones: {len(self.proxy.phones.list)}') + print(f'# of ignored phones ({", ".join(self.ANONYMOUS_DEVICE_TYPES)}): ' + f'{len(self.proxy.phones.list) - len(relevant_phones)}') + + print('Phones with issues') + print('0 : no user, n: more than one user, u: at least unknown user') + phones_with_issues = {device_name: classification + for device_name, classification in phone_classification.items() + if classification & {'u', 'n', '0'}} + + ok_id = sb.add_entry(parent_id=relevant_id, label='Ok', + value=len(relevant_phones) - len(phones_with_issues)) + sb.add_phones(ok_id, phones=(p + for p in relevant_phones + if not phone_classification[p.device_name] & {'u', 'n', '0'} + )) + + w_issues_id = sb.add_entry(parent_id=relevant_id, label='With issues', value=len(phones_with_issues)) + + classification_to_text = { + '0': 'No user', + 'n': 'Multiple users', + 'u': 'unknown user' + } + + def sunburst_users(sb, parent_id, phone: Phone): + users = phone.user_set + if not users: + return + for user in users: + sb.add_entry(parent_id=parent_id, label=user) + + for issue in '0nu': + phones_with_this_issue = [self.proxy.phones[device_name] + for device_name, classification in phones_with_issues.items() + if issue in classification] + class_id = sb.add_entry(parent_id=w_issues_id, label=classification_to_text[issue], + value=len(phones_with_this_issue)) + sb.add_phones(parent_id=class_id, phones=phones_with_this_issue, details=True, + phone_childs=sunburst_users) + + phones_with_issue_by_device_type: Dict[str, List[Phone]] = defaultdict(list) + for phone in phones_with_this_issue: + phones_with_issue_by_device_type[phone.device_type].append(phone) + + print(f' Phones classified as "{issue}"') + device_types = sorted(phones_with_issue_by_device_type) + for device_type in device_types: + output = (f'{phone.device_name}' for phone in phones_with_issue_by_device_type[device_type]) + print(f' {len(phones_with_issue_by_device_type[device_type])} x {device_type}: ' + f'{", ".join(output)}') + + fig = sb.fig() + fig.update_layout(margin=dict(t=0, l=0, r=0, b=0)) + fig.show() + + log.debug('Start looking at phones per user') + # Check whether the 1st line on all phones of a user has the same DN + # get phones per user + phones_per_user: Dict[str, List[Phone]] = defaultdict(list) + for phone in self.proxy.phones.list: + users = list(phone.user_set) + if len(users) != 1: + # we only want to look at phones with one user + continue + user = users[0] + phones_per_user[user].append(phone) + log.debug('Done looking at phones per user') + + def dn_and_partition1(phone: Phone) -> Optional[str]: + try: + return phone.lines[1].dn_and_partition + except KeyError: + return None + + log.debug('Start looking at all lines of all phones') + dn_issues = {user: phones + for user, phones in phones_per_user.items() + if not (dn := dn_and_partition1(phones[0])) or + any(dn != dn_and_partition1(phone) for phone in phones[1:])} + log.debug('Done looking at all lines of all phones') + print('Users with inconsistent DN assignments') + for user in sorted(dn_issues): + phones = dn_issues[user] + output = ', '.join(f'{phone.device_name}({dn_and_partition1(phone)})' for phone in phones) + print(f'{user}: {output}') + + @menu_register('Users with more than 10 phones') + def menu_users_with_phone_count(self): + phones_per_user: dict[str, List[Phone]] = defaultdict(list) + for ph in self.proxy.phones.list: + for user in ph.user_set: + phones_per_user[user].append(ph) + user_phone_count = {user: len(phones) for user, phones in phones_per_user.items()} + user_phone_count = sorted(user_phone_count.items(), key=lambda x: x[1], reverse=True) + print('Users with more than 10 phones:') + for user, phone_count in user_phone_count: + if phone_count < 10: + continue + print(f'{user}: {phone_count}') + + @menu_register('External phone number masks') + def menu_external_phone_number_masks(self): + phones_by_mask: Dict[str, List[Phone]] = defaultdict(list) + for phone in self.proxy.phones.list: + line = next((line for line in phone.lines.values()), None) + if not line: + continue + mask = line.external_phone_number_mask + if not mask: + continue + phones_by_mask[mask].append(phone) + sb = SunBurstHelper() + root = sb.add_entry(parent_id='', label='External Phone Number Masks') + for mask in sorted(phones_by_mask): + phones_this_mask = phones_by_mask[mask] + print(f'mask {mask}: {len(phones_this_mask)} phones') + mask_id = sb.add_entry(parent_id=root, label=mask, value=len(phones_this_mask)) + sb.add_phones(parent_id=mask_id, phones=phones_this_mask) + fig = sb.fig() + fig.show() + + @menu_register('DN analyis') + def menu_dn_analysis(self): + """ + get all DNs of all 1st lines of all phones and analyze the structure + :return: None + """ + DNP = namedtuple('DNP', ['dn', 'partition']) + dnps = [DNP._make((line1.directory_number, line1.partition)) + for phone in self.proxy.phones.list + if (line1 := phone.lines.get(1))] + + def do_analysis(dnps: List[DNP]): + """ + Analysis of a set of DNs + :param dnps: + :return: + """ + # group DNs by len + dn_by_len: Dict[int, List[str]] = defaultdict(list) + for dnp in dnps: + dn_by_len[len(dnp.dn)].append(dnp.dn) + + DNCluster = namedtuple('DNCluster', ['prefix', 'dns']) + + def find_clusters(prefix: str, digit_strings: List[str], total_count=None) -> List[Tuple[str, List[str]]]: + if not prefix: + total_count = len(digit_strings) + if len(digit_strings[0]) <= 1: + return [] + + # determine DNs per next level digit + first_digits = set() + next_level_dns: Dict[str, List[str]] = defaultdict(set) + for ds in digit_strings: + first_digit = ds[0] + first_digits.add(first_digit) + next_level_dns[first_digit].add(ds[1:]) + first_digits = sorted(first_digits) + total_count /= len(first_digits) + for fd in first_digits: + nld = sorted(next_level_dns[fd])[:10] + output = [f'{prefix}{fd}-{ds}' for ds in nld] + if len(next_level_dns[fd]) > 10: + output.append('...') + remaining_length = len(next(dn for dn in next_level_dns[fd])) + density = 9 ** remaining_length + + print( + f'prefix {prefix}-{fd}: {int(total_count)} {len(next_level_dns[fd])}/{density} digit strings: ' + f'{", ".join(output)}') + for fd in first_digits: + find_clusters(prefix=f'{prefix}{fd}', digit_strings=list(next_level_dns[fd]), + total_count=total_count) + + return [] + + for dn_len in dn_by_len: + print(f' len({dn_len}):') + find_clusters('', dn_by_len[dn_len]) + return [] + + # analysis of all DNS + print('All DNs') + do_analysis(dnps) + + dn_by_partition: Dict[str, List[DNP]] = defaultdict(list) + for dnp in dnps: + dn_by_partition[dnp.partition].append(dnp) + + # analysis by partition + for partition in dn_by_partition: + print(f'Partition \'{partition}\'') + do_analysis(dn_by_partition[partition]) + + @menu_register('Find Locations based on users phone numbers') + def menu_locations_based_on_users_phones(self): + # assert CSVs are read + # noinspection PyStatementEffect + self.proxy.phones.list + # noinspection PyStatementEffect + self.proxy.end_user.list + + print(f' # of phones: {len(self.proxy.phones.list)}') + + # Ignore "anonymous" phones + relevant_phones = [phone + for phone in self.proxy.phones.list + if phone.device_type not in self.ANONYMOUS_DEVICE_TYPES] + print(f' w/o anonymous phones: {len(relevant_phones)}') + + # ignore phones w/o users or with multiple users + relevant_phones = [phone + for phone in relevant_phones + if len(phone.user_set) == 1] + print(f' only phones with one user: {len(relevant_phones)}') + + # ignore phones where the users don't exist + relevant_phones = [phone + for phone in relevant_phones + if self.proxy.end_user.get(list(phone.user_set)[0])] + print(f' only phones where the user exists: {len(relevant_phones)}') + + # ignore phones w/o lines + relevant_phones = [phone + for phone in relevant_phones + if len(phone.lines)] + print(f'only phones with at least one line: {len(relevant_phones)}') + + users = list(chain.from_iterable(phone.user_set + for phone in relevant_phones)) + # check if + device_name_len = max(len(phone.device_name) for phone in relevant_phones) + uid_len = max(len(u) for u in users) + for phone in relevant_phones: + user_id = list(phone.user_set)[0] + user = self.proxy.end_user[user_id] + try: + user_phone_number = user.phone.replace(' ', '') + except AttributeError: + user_phone_number = None + if not user_phone_number: + user_phone_number = 'empty' + + try: + dn = phone.lines[1].directory_number + except KeyError: + dn = 'no line!' + print(f'{phone.device_name:{device_name_len}} {user_id:{uid_len}} {user_phone_number} {dn}') + + dns = [line1.directory_number for phone in relevant_phones if (line1 := phone.lines.get(1))] + dns_by_len: Dict[int, List[str]] = defaultdict(list) + for dn in dns: + dns_by_len[len(dn)].append(dn) + + for dn_len, dns_work in dns_by_len.items(): + print(f'len {dn_len}: {len(dns_work)}') + + for prefix_len in range(1, dn_len + 1): + prefixes = sorted(list(set(dn[:prefix_len] for dn in dns_work))) + if len(prefixes) > 15: + prefixes_output = prefixes[:15] + ['...'] + else: + prefixes_output = prefixes + print(f' len {prefix_len}: {len(prefixes)} prefixes: {", ".join(prefixes_output)}') + + def search_location_prefixes(dn_len, prefix_len, dns): + prefixes = sorted(list(set(dn[:prefix_len] for dn in dns))) + + pass + + for dn_len, dns_work in dns_by_len.items(): + search_location_prefixes(dn_len=dn_len, prefix_len=1, dns=dns_work) + + @menu_register('Sankey Diagram of DNs') + def menu_sankey_dn(self): + # get all DNs from the 1st line of all phones + dns = [line1.directory_number + for phone in self.proxy.phones.list + if (line1 := phone.lines.get(1))] + + node_label = [] + links_source = [] + link_target = [] + link_value = [] + + def add_to_sankey(numbers: List[str], parent_node_id: int = None): + first_digits = sorted(set(n[0] for n in numbers)) + for first_digit in first_digits: + node_id = len(node_label) + label = first_digit + if parent_node_id is not None: + label = f'{node_label[parent_node_id].strip("X")}{label}' + # noinspection PyUnboundLocalVariable + next_level_numbers = [s for n in numbers if n[0] == first_digit and (s := n[1:])] + if not next_level_numbers: + node_label.append(label) + continue + max_len = max(map(len, next_level_numbers)) + node_label.append(f'{label}{"X" * max_len}') + if max_len < 2: + continue + if next_level_numbers and parent_node_id is not None: + links_source.append(parent_node_id) + link_target.append(node_id) + link_value.append(len(next_level_numbers)) + add_to_sankey(next_level_numbers, parent_node_id=node_id) + + add_to_sankey(numbers=dns) + fig = go.Figure(data=[go.Sankey( + node=dict( + pad=15, + thickness=20, + line=dict(color='black', + width=0.5), + label=node_label, + color='blue' + ), + link=dict( + source=links_source, + target=link_target, + value=link_value + ) + )]) + fig.update_layout(title_text='Sankey', font_size=10) + fig.show() + + @menu_register('Find intra-site dialing translation patterns') + def menu_find_intrasite_translation_patterns(self): + tps = self.proxy.translation_pattern.list + print(f'Found {len(tps)} translation patterns') + tp_by_len: Dict[int, List[TranslationPattern]] = defaultdict(list) + names = [] + parents = [] + for tp in tps: + if tp.block: + # ignore blocking translation pattern + continue + tp_by_len[tp.length].append(tp) + names.append(tp.pattern_and_partition) + parents.append(str(tp.length)) + # add tp lengths as parents + names.extend(map(str, tp_by_len)) + # .. w/o parents themselves + parents.extend([''] * len(tp_by_len)) + fig = px.treemap( + names=names, + parents=parents + ) + fig.show() + + @menu_register('CSS Combinations on first lines') + def menu_css_combinations(self): + css_combinations = Counter((first_line.css, phone.css) + for phone in self.proxy.phones.list + if (first_line := next(iter(phone.lines.values()), None))) + frequency_len = len(f'{max(css_combinations.values())}') + css_len = max(len(f'{", ".join(css_combination)}') for css_combination in css_combinations) + for css_combination in sorted(css_combinations, + key=lambda x: css_combinations[x], + reverse=True): + frequency = css_combinations[css_combination] + combined_partitions = list(chain.from_iterable(map(self.proxy.css.partition_names, css_combination))) + print( + f'frequency: {frequency:{frequency_len}}: {", ".join(css_combination):{css_len}} -> ' + f'{":".join(combined_partitions)}') + + @menu_register('Dial Plan Analysis') + def menu_dial_plan_analysis(self): + da_tree = digit_analysis.DaNode.from_proxy(self.proxy, first_line_only=True) + # after adding all patterns we now want to find out how to dial on-net + # traverse the tree breadth first and consider all sub trees which potentially can get us to a DN + # - only TPs and DNs + # - look out for blocking TPs + + # get CSS combinations on phones + # noinspection PyShadowingNames + css_count = Counter((first_line.css, phone.css) + for phone in self.proxy.phones.list + if (first_line := next(iter(phone.lines.values()), None))) + + css_combinations = sorted(css_count, key=lambda c: css_count[c], reverse=True) + for line_css_name, device_css_name in css_combinations: + print(f'Looking at line_css:device_css: {line_css_name}:{device_css_name}') + combined_partitions = list(chain(self.proxy.css.partition_names(line_css_name), + self.proxy.css.partition_names(device_css_name))) + # add partition at the end if not already present somewhere in the partition list + if next((p for p in combined_partitions if not p), None) is None: + print(f'appending NONE partition') + combined_partitions.append('') + print(f'{line_css_name} + {device_css_name}: {", ".join(combined_partitions)}') + leaves = list(da_tree.find_leaves(depth=30, + pattern_types={digit_analysis.PatternType.DN, + digit_analysis.PatternType.TP}, + stop_decent={digit_analysis.PatternType.DN}, + partitions=combined_partitions)) + print(f' {len(leaves)} leaves') + # list all blocking TPs + blocking_tps = [tp for tp in self.proxy.translation_pattern.list + if tp.block] + print("\n".join(f'{tp}' for tp in blocking_tps)) + # print(da_tree.pretty()) + + @menu_register('Translation pattern overview') + def menu_translation_pattern_overview(self): + tps = self.proxy.translation_pattern.list + rows = [('pattern', 'partition', 'Urgent', 'Block', 'discard digits', 'prefix digits', 'mask')] + for tp in tps: + rows.append((f'{tp.pattern}', f'{tp.partition}', f'{tp.urgent}', f'{tp.block}', f'{tp.discard_digits}', + f'{tp.called_party_prefix_digits}', f'{tp.called_party_mask}')) + transposed = list(zip(*rows)) + column_lens = [max(map(len, col)) for col in transposed] + print("\n".join(' '.join(f'{c:{column_lens[i]}}' for i, c in enumerate(r)) for r in rows)) + + @menu_register('Find abbreviated on-net dialing') + def menu_abbreviated_on_net(self): + """ + + :return: + """ + + """For a given CSS find all TP terminal nodes for each terminal TP node apply TP's translation to TP's + pattern and then see if that transformed digit string can hit DNs """ + da_tree = digit_analysis.DaNode.from_proxy(self.proxy) + + css_count = Counter((first_line.css, phone.css) + for phone in self.proxy.phones.list + if (first_line := next(iter(phone.lines.values()), None))) + + css_combinations = sorted(css_count, key=lambda c: css_count[c], reverse=True) + for line_css_name, device_css_name in css_combinations: + print(f'Looking at line_css:device_css: {line_css_name}+{device_css_name}') + combined_partitions = list(chain(self.proxy.css.partition_names(line_css_name), + self.proxy.css.partition_names(device_css_name))) + # add partition at the end if not already present somewhere in the partition list + if next((p for p in combined_partitions if not p), None) is None: + print(f'appending NONE partition') + combined_partitions.append('') + print(f'{line_css_name}+{device_css_name}: {":".join(combined_partitions)}') + # get all TPs that can be dialed with this CSS + leaves = list(da_tree.find_leaves(depth=30, + pattern_types={digit_analysis.PatternType.TP}, + partitions=combined_partitions)) + combined_css = ':'.join(combined_partitions) + for da_node, dial_string in leaves: + tps = [tp + for tp in da_node.terminal_pattern.values() + if isinstance(tp, digit_analysis.TranslationPattern)] + for tp in tps: + print(f'{dial_string}: {tp.pattern}') + lookup_result = da_tree.lookup(digits=dial_string, css=combined_css) + print(f'Dial string {dial_string} lookup led to {lookup_result}') + # translated_dial_string, css = tp.translate(digits=tp.pattern.replace('.', ''), + # css=combined_partitions) + foo = 1 + pass + + @menu_register('Dump call hunt info') + def menu_line_group_dump(self): + # get hunt pilots + hunt_pilots = self.proxy.hunt_pilot.list + print('Hunt pilots:') + for hp in hunt_pilots: + phones = hp.phones_or_device_profiles(hunt_pilot_container=self.proxy.hunt_pilot, + container=self.proxy.phones) + device_profiles = hp.phones_or_device_profiles(hunt_pilot_container=self.proxy.hunt_pilot, + container=self.proxy.device_profile) + phones |= device_profiles + print(f'{hp.pilot_and_partition}{"" if phones else " (no phones)"}, {hp.description}, hunt lists: ' + f'{", ".join(f"{hl}" for hl in hp.hunt_lists)}') + + # list hunt lists + hunt_lists = self.proxy.hunt_list.list + print() + print('Hunt Lists') + for hl in hunt_lists: + phones = hl.phones_or_device_profiles(hunt_list_container=self.proxy.hunt_list, + container=self.proxy.phones) + device_profiles = hl.phones_or_device_profiles(hunt_list_container=self.proxy.hunt_list, + container=self.proxy.device_profile) + phones |= device_profiles + members = hl.members + members.sort(key=lambda m: m.selection_order) + print(f'{hl.name}{"" if phones else " (no phones)"}, {hl.description}, ' + f'members: {", ".join(f"{m}" for m in hl.members)}') + users_by_pe: Dict[str, List[EndUser]] = self.proxy.end_user.by_attribute('primary_extension') + users_by_pe.pop(None, None) + line_groups = self.proxy.line_group.list + print() + print('Line Groups') + for line_group in line_groups: + phones = line_group.phones_or_device_profiles(container=self.proxy.phones) + device_profiles = line_group.phones_or_device_profiles(container=self.proxy.device_profile) + phones |= device_profiles + if not phones: + # skip line groups w/o phones + continue + print(f'Line group "{line_group.name}"') + lg_members = line_group.members + # sort by selection order + lg_members.sort(key=lambda m: m.selection_order) + for lg_member in lg_members: + dnp = lg_member.pattern_and_partition + print(f' {lg_member.selection_order}) DN: {dnp}', end='') + users_pe = users_by_pe.get(dnp) + if users_pe: + print(f', primary extension for user(s) {", ".join(f"{u.user_id}" for u in users_pe)}', end='') + phones = self.proxy.phones.by_dn_and_partition.get(dnp, set()) + device_profiles = self.proxy.device_profile.by_dn_and_partition.get(dnp, set()) + phones |= device_profiles + if phones: + print(', phones or device profiles: ', end='') + owners: List[List[str]] = [] # list of owners per phone + for phone_or_dp in phones: + if isinstance(phone_or_dp, Phone): + phone_or_dp: Phone + owners.append([phone_or_dp.owner]) + else: + # for a device profile we need to find users with this device profile name set in + # their device associations + phone_or_dp: DeviceProfile + users = self.proxy.end_user.by_em_profile_name.get(phone_or_dp.device_profile_name, []) + owners.append([user.user_id for user in users]) + + print(', '.join(f'{phone} ({", ".join(users)})' for phone, users in zip(phones, owners)), end='') + all_owners = list(set(chain.from_iterable(owners))) + all_owners.sort() + print(f', owner(s): {", ".join(all_owners)}', end='') + print() + if False: + """ + Deprecated output format + """ + + print(f' DNs: {", ".join(members)}') + users = list(chain.from_iterable(users_by_pe.get(m, []) for m in members)) + users.sort() + if users: + print(f' Users by primary extension: {", ".join(u.user_id for u in users)}') + # collect all phones with any of these members as dn + phones = line_group.phones(phone_container=self.proxy.phones) + if phones: + print(f' phones: {", ".join(p.device_name for p in phones)}') + owners = list(set(p.owner for p in phones if p.owner)) + owners.sort() + if owners: + print(f' owners: {", ".join(owners)}') + # users associated with these phones + user_ids = set(chain.from_iterable(phone.user_set for phone in phones)) + # .. and we only want to look at users which actually exist as end users + users = [user for user_id in user_ids if (user := self.proxy.end_user.get(user_id))] + users: List[EndUser] + users.sort(key=lambda u: u.user_id) + if users: + print(f' users: {", ".join(u.user_id for u in users)}') diff --git a/playbooks/ucm-config-analyzer/src/digit_analysis/__init__.py b/playbooks/ucm-config-analyzer/src/digit_analysis/__init__.py new file mode 100644 index 0000000..0cc0815 --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/digit_analysis/__init__.py @@ -0,0 +1,3 @@ +from digit_analysis.node import * +from digit_analysis.base import * + diff --git a/playbooks/ucm-config-analyzer/src/digit_analysis/base.py b/playbooks/ucm-config-analyzer/src/digit_analysis/base.py new file mode 100644 index 0000000..67c996c --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/digit_analysis/base.py @@ -0,0 +1,153 @@ +from enum import unique, Enum +from itertools import zip_longest +from typing import List, Tuple, Set +import logging + +__all__ = ['PatternType', 'Pattern', 'TranslationPattern', 'DnPattern', 'RoutePattern', 'ALL_PATTERN_TYPES', + 'TranslationError'] + + +@unique +class PatternType(Enum): + DN = 0 + TP = 1 + RP = 2 + + def __str__(self): + return self.name + + +ALL_PATTERN_TYPES = {PatternType.DN, PatternType.TP, PatternType.RP} + +log = logging.getLogger(__name__) + + +class Pattern: + __slots__ = ['type', 'pattern', 'partition'] + + def __init__(self, pattern: str, partition: str, type: PatternType): + self.type = type + self.pattern = pattern + self.partition = partition + + def __str__(self): + return f'{self.dn_and_partition}({self.type})' + + @property + def dn_and_partition(self): + partition = self.partition or 'NONE' + return f'{self.pattern}:{partition}' + + +class TranslationError(Exception): + pass + + +class TranslationPattern(Pattern): + __slots__ = ['css', 'block', 'urgent', 'use_originators_calling_search_space', 'discard_digits', + 'called_party_mask', 'called_party_prefix_digits', 'route_next_hop_by_calling_party_number'] + + def __init__(self, pattern: str = '', partition: str = '', block: bool = False, + css: List[str] = None, urgent: bool = True, use_originators_calling_search_space: bool = False, + discard_digits: int = 0, called_party_mask: str = '', called_party_prefix_digits: str = '', + route_next_hop_by_calling_party_number: bool = False): + super(TranslationPattern, self).__init__(pattern=pattern, partition=partition, type=PatternType.TP) + self.css = css + self.block = block + self.urgent = urgent + self.use_originators_calling_search_space = use_originators_calling_search_space + self.discard_digits = discard_digits + self.called_party_mask = called_party_mask + self.called_party_prefix_digits = called_party_prefix_digits + self.route_next_hop_by_calling_party_number = route_next_hop_by_calling_party_number + + def translate(self, digits: str, css: str) -> Tuple[str, str]: + """ + Apply translation to given digit string + :param digits: digit string + :param css: activating css + :return: Tuple[digit string, css for secondary lookup] + """ + + def discard_digits(digits: str) -> str: + # find dot in pattern + dot_pos = self.pattern.find('.') + if dot_pos == -1: + raise TranslationError(f'discard PreDot requires "." in pattern: {self}') + return digits[dot_pos:] + + def str_to_set_list(digits: str) -> List[Set[str]]: + digits = iter(digits) + result = [] + for digit in digits: + if digit == '[': + # start of enumeration + dset = set() + p_digit = None + digit = next(digits) + while digit != ']': + if digit == '-': + digit = next(digits) + for o in range(ord(p_digit), ord(digit) + 1): + dset.add(chr(o)) + else: + dset.add(digit) + p_digit = digit + digit = next(digits) + # while + result.append(dset) + else: + result.append({digit}) + # if + # for + return result + + def set_list_to_str(dset_list: List[Set[str]]) -> str: + result = '' + for dset in dset_list: + if not dset: + continue + if len(dset) == 1: + append = next(iter(dset)) + if not append: + continue + else: + append = ''.join(sorted(dset)) + if append == '0123456789': + append = 'X' + else: + append = f'[{append}]' + result = f'{result}{append}' + return result + + new_digits = str_to_set_list(digits) + if self.use_originators_calling_search_space: + new_css = css + else: + new_css = ':'.join(self.css) + if self.discard_digits: + new_digits = discard_digits(new_digits) + if self.called_party_prefix_digits: + new_digits = [{d} for d in self.called_party_prefix_digits] + new_digits + if self.route_next_hop_by_calling_party_number: + raise NotImplementedError + + if self.called_party_mask: + masked = list(d if m == 'X' else {m} + for m, d in zip_longest(self.called_party_mask[::-1], + new_digits[::-1], + fillvalue='')) + new_digits = masked[::-1] + new_digits = set_list_to_str(new_digits) + log.debug(f'{self} translate {digits}->{new_digits}') + return new_digits, new_css + + +class DnPattern(Pattern): + def __init__(self, pattern: str, partition: str = ''): + super(DnPattern, self).__init__(pattern=pattern, partition=partition, type=PatternType.DN) + + +class RoutePattern(Pattern): + def __init__(self, pattern: str, partition: str = ''): + super(RoutePattern, self).__init__(pattern=pattern, partition=partition, type=PatternType.RP) diff --git a/playbooks/ucm-config-analyzer/src/digit_analysis/node.py b/playbooks/ucm-config-analyzer/src/digit_analysis/node.py new file mode 100644 index 0000000..a03d678 --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/digit_analysis/node.py @@ -0,0 +1,482 @@ +from .base import Pattern, PatternType, ALL_PATTERN_TYPES, DnPattern, TranslationPattern, RoutePattern + +from typing import Dict, Optional, Iterator, List, Set, Generator, Deque, Iterable, Tuple, Any, Callable, Union +from itertools import takewhile, tee, chain +from collections import defaultdict, deque +import re +from logging import getLogger +from ucmexport import Proxy +from time import perf_counter + +__all__ = ['DaNode'] + +log = getLogger(__name__) +traversal_log = getLogger(f'{__name__}.traversal') +match_log = getLogger(f'{__name__}.match') + +MAX_TRANSLATION_DEPTH = 5 + + +class DaNode: + """ + Da node represents a digit in the tree + """ + __slots__ = ['childs', 'terminal_pattern', 'depth', 'representation', 'partitions', 'pattern_types', 'parent', + 'full_representation', 'matching_digits'] + + SINGLE_DIGIT_RE = re.compile(r'\[((?:\d|(?:\d-\d))+)]') + + ALL_DIGITS = set('*0123456789') + + def __init__(self, representation: str = '', + matching_digits: int = 1, + parent: 'DaNode' = None): + """ + :param representation: representation for this node. Can be a wildcard ("X", "[1-5]", ...) + :param parent: parent node + """ + self.childs: Dict[str, List['DaNode']] = defaultdict(list) + self.representation = representation + self.parent = parent + if parent is None: + self.depth = 0 + self.full_representation = '' + else: + self.depth = parent.depth + 1 + self.full_representation = f'{parent.full_representation}{self.representation}' + self.terminal_pattern: Dict[str, Pattern] = {} + self.partitions: Set[str] = set() + self.pattern_types: Set[PatternType] = set() + self.matching_digits = matching_digits + + @staticmethod + def from_proxy(proxy: Proxy, first_line_only=False) -> 'DaNode': + """ + Get a DA tree based on TPs, RPs and DNs + :param proxy: + :param first_line_only: + :return: + """ + da_tree = DaNode() + + def tp_from_proxy_and_translation_pattern(proxy: Proxy, + translation_pattern: TranslationPattern) -> \ + TranslationPattern: + return TranslationPattern(pattern=translation_pattern.pattern, + partition=translation_pattern.partition, + block=translation_pattern.block, + css=proxy.css.partition_names(css_name=translation_pattern.css), + urgent=translation_pattern.urgent, + use_originators_calling_search_space=translation_pattern.use_originators_calling_search_space, + discard_digits=translation_pattern.discard_digits, + called_party_mask=translation_pattern.called_party_mask, + called_party_prefix_digits=translation_pattern.called_party_prefix_digits, + route_next_hop_by_calling_party_number=translation_pattern + .route_next_hop_by_calling_party_number) + + # add all translation patterns to DA tree + tps = proxy.translation_pattern.list + start = perf_counter() + for tp in tps: + da_tree.add_pattern(tp_from_proxy_and_translation_pattern(proxy=proxy, translation_pattern=tp)) + log.debug(f'adding {len(tps)} translation patterns: {(perf_counter() - start) * 1000:.2f}ms') + + if first_line_only: + # dnps of all 1st lines + dnps = set(first_line.dn_and_partition + for phone in proxy.phones.list + if (first_line := next(iter(phone.lines.values()), None))) + else: + # All dnps of all lines + dnps = set(chain.from_iterable((line.dn_and_partition + for line in phone.lines.values()) + for phone in proxy.phones.list)) + start = perf_counter() + for dnp in dnps: + dn, partition = dnp.split(':') + da_tree.add_pattern(DnPattern(pattern=dn, partition=partition)) + log.debug(f'adding {len(dnps)} DNs: {(perf_counter() - start) * 1000:.2f}ms') + + # finally all route patterns + rps = proxy.route_pattern.list + start = perf_counter() + for rp in rps: + da_tree.add_pattern(RoutePattern(pattern=rp.pattern, partition=rp.partition)) + log.debug(f'adding {len(rps)} Route Patterns: {(perf_counter() - start) * 1000:.2f}ms') + + return da_tree + + @property + def all_child_nodes(self) -> Set['DaNode']: + return set(chain.from_iterable(childs + for childs in self.childs.values())) + + def add_pattern(self, pattern: Pattern) -> None: + """ + Add a pattern to the DA tree starting at this node + :param pattern: pattern to be added + """ + self.add_digits(digits=iter(pattern.pattern), pattern=pattern) + + def repr_and_matching_set(self, digit: str, digits: Iterator[str]) -> Tuple[str, Set[str]]: + # determine representation for new node + # for something like "[1-9]" we need to collect everything until the closing bracket + representation = digit + if not digit: + return '', set() + if digit == '[': + digits_matched = set() + inner = takewhile(lambda x: x != ']', digits) + for c in inner: + if c == '-': + representation = f'{representation}{c}' + # noinspection PyUnboundLocalVariable + # digit after "-" + c = next(inner) + # noinspection PyUnboundLocalVariable + if c < previous_c: + raise ValueError + for i in range(int(previous_c), int(c)): + digits_matched.add(f'{i + 1}') + else: + digits_matched.add(c) + # store previous digit in case the next "digit" is a "-" + previous_c = c + # if + # for + # normalized representation without "-" + representation = f'[{"".join(sorted(digits_matched))}]' + elif digit in self.ALL_DIGITS: + digits_matched = {digit} + elif digit in '!X@': + digits_matched = self.ALL_DIGITS + elif digit == '+': + if self.depth != 0: + raise ValueError + digits_matched = {'+'} + else: + raise ValueError + return representation, digits_matched + + def add_digits(self, digits: Iterator[str], pattern: Pattern) -> None: + """ + Add digit string as child of node + :param pattern: pattern to be added at the terminating node + :param digits: iterator over the digits of the digit string to be added + :raises ValueError: illegal digit string + """ + self.partitions.add(pattern.partition) + self.pattern_types.add(pattern.type) + # skip over separator + while (first_digit := next(digits, None)) and first_digit in '.#\\': + if first_digit == '\\' and self.depth != 0: + raise ValueError + if first_digit is None: + # Reached the end og the digit string + # put the terminal pattern into this node + # same pattern can exist in multiple partitions and we want to keep track of all terminal + # patterns + self.terminal_pattern[pattern.partition] = pattern + return + representation, digits_matched = self.repr_and_matching_set(digit=first_digit, digits=digits) + # we need a child node for every digit matched + # a new child node is not needed for a given matched digit if one of the existing child nodes for that digit + # has the same representation + # we might be forced to climb down multiple trees... + new_node = None + add_to_nodes: Set[DaNode] = set() + for digit_matched in digits_matched: + child_nodes_for_digit_matched = self.childs[digit_matched] + child_node_with_matching_representation = next((child + for child in child_nodes_for_digit_matched + if child.representation == representation), + None) + if child_node_with_matching_representation: + # need to climb down that node + add_to_nodes.add(child_node_with_matching_representation) + else: + # new node needs to be added with the new representation + new_node = new_node or DaNode(representation=representation, + parent=self, + matching_digits=len(digits_matched)) + self.childs[digit_matched].append(new_node) + # now continue to climb down all nodes + if new_node: + add_to_nodes.add(new_node) + if len(add_to_nodes) == 1: + digit_iters = [digits] + else: + digit_iters = tee(digits, len(add_to_nodes)) + for child_node, digit_iter in zip(add_to_nodes, digit_iters): + child_node.add_digits(digits=digit_iter, pattern=pattern) + + def terminal_nodes(self) -> Generator['DaNode', None, None]: + """ + Traverse through the tree breadth 1st and yield all nodes with terminal patterns + """ + for node in self.breadth_first_traversal(): + if node.terminal_pattern: + yield node + # for + # while + + def terminal_patterns(self) -> Generator[Pattern, None, None]: + for node in self.terminal_nodes(): + for tp in node.terminal_pattern: + yield tp + + def find_leaves(self, depth: int, partitions: List[str] = None, partition_set: Set[str] = None, + pattern_types: Set[PatternType] = None, + stop_decent: Set[PatternType] = None) -> Generator[Tuple['DaNode', str], None, None]: + partitions = partitions or list(self.partitions) + partition_set = partition_set or set(partitions) + pattern_types = pattern_types or ALL_PATTERN_TYPES + stop_decent = stop_decent or {PatternType.DN} + traversal_log.debug( + f'find_leaves: depth {depth} partitions {":".join(partition_set)} pattern_types: ' + f'{", ".join(str(pt) for pt in pattern_types)} stop_decent: {", ".join(str(s) for s in stop_decent)}') + + def dial_string_context(node: DaNode, dial_string: str) -> str: + return f'{dial_string}{node.representation}' + + traversal = self.breadth_first_traversal_with_context(start_context='', context_func=dial_string_context) + + def next_node(dial_string: str = '', descend: List[DaNode] = None) -> Tuple[Optional[DaNode], str]: + descend = descend or [] + try: + node = traversal.send((n, dial_string_context(node=n, dial_string=dial_string)) + for n in descend) + except StopIteration: + node = (None, '') + return node + + node, dial_string = next(traversal) + while node: + traversal_log.debug(f'find_leaves: depth {node.depth} representation {node.full_representation} ' + f'pattern types {", ".join(f"{pt}" for pt in node.pattern_types)} ' + f'partitions: {", ".join(node.partitions)}') + if not pattern_types & node.pattern_types: + # There are no pattern types on the current nodes common with the nodes we are interested in + # no need to look at this + traversal_log.debug(f'find_leaves: depth {node.depth} representation {node.full_representation} ' + f'pattern types {", ".join(f"{pt}" for pt in node.pattern_types)} ' + f'partitions: {", ".join(node.partitions)} -> ignore, stop descend, no matching ' + f'pattern ' + f'types') + node, dial_string = next_node() + continue + + if not partition_set & node.partitions: + # no common partitions: can skip this node + traversal_log.debug(f'find_leaves: depth {node.depth} representation {node.full_representation} ' + f'pattern types {", ".join(f"{pt}" for pt in node.pattern_types)} ' + f'partitions: {", ".join(node.partitions)} -> ignore, stop descend, ' + f'no common partitions') + node, dial_string = next_node() + continue + + if node.terminal_pattern: + traversal_log.debug(f'find_leaves: terminal pattern: {node.terminal_pattern}') + + # yield if we reached the max. depth or if we have a terminal pattern + # also yield if the pattern types below this node are identified by stop_decent + if node.depth == depth or \ + node.terminal_pattern or \ + node.pattern_types == stop_decent: + traversal_log.debug(f'find_leaves: depth {node.depth} representation {node.full_representation} ' + f'pattern types {", ".join(f"{pt}" for pt in node.pattern_types)} ' + f'partitions: {", ".join(node.partitions)} -> yield') + yield node, dial_string + # no deed to climb further down if we reached the maximum depth or the subtree only has patterns + # included in the stop decent set + if node.depth == depth or node.pattern_types == stop_decent: + # no need to climb further down + traversal_log.debug(f'find_leaves: depth {node.depth} representation {node.full_representation} ' + f'pattern types {", ".join(f"{pt}" for pt in node.pattern_types)} ' + f'partitions: {", ".join(node.partitions)} -> stop descend') + node, dial_string = next_node() + continue + # Determine childs to climb down into: only childs with partition match + climb_down = [child + for child in node.all_child_nodes + if child.partitions & partition_set] + node, dial_string = next_node(dial_string=dial_string, descend=climb_down) + return + + def breadth_first_traversal_with_context(self, start_context: Any, + context_func: Callable[['DaNode', Any], Any] = None) -> \ + Generator[Tuple['DaNode', Any], Optional[Iterable[Tuple['DaNode', Any]]], None]: + node_queue: Deque[Tuple['DaNode', Any]] = deque() + node_queue.append((self, start_context)) + while node_queue: + node, context = node_queue.popleft() + traversal_log.debug(f'breadth_first_traversal: yield {node.depth}:{node.full_representation}') + descend = yield node, context + + if descend is None: + # descend into all child nodes + if context_func: + node_queue.extend((n, context_func(n, context)) for n in node.all_child_nodes) + else: + node_queue.extend((n, context) for n in node.all_child_nodes) + else: + node_queue.extend(descend) + + def breadth_first_traversal(self) -> Generator['DaNode', Optional[Iterable['DaNode']], None]: + """ + Traverse the tree breadth 1st + :return: Nodes of the tree in breadth 1st order + """ + for node, _ in self.breadth_first_traversal_with_context(None): + yield node + + def matching_nodes(self, digits: Union[str, Iterator[str]], + css: Union[str, Set[str]], + parent_digits: str = None, + parent_alternatives: int = 1) -> Generator[Tuple['DaNode', int], None, None]: + """ + Yield matching DA nodes for given digits and CSS + :param digits: digit string to match + :param css: CSS string + :param parent_digits: + :param parent_alternatives: + :return: Tuple of da node and match priority (the lower the better) + """ + if isinstance(digits, str): + digits = iter(digits) + if isinstance(css, str): + css = set(css.split(':')) + parent_digits = parent_digits or '' + digit = next(digits, '') + digit_repr, digit_set = self.repr_and_matching_set(digit=digit, digits=digits) + match_log.debug(f'{self} digits {parent_digits}:{digit_repr}') + parent_digits = f'{parent_digits}{digit_repr}' + parent_alternatives = parent_alternatives * self.matching_digits + if digit == '': + # last digit consumed + if self.terminal_pattern: + match_log.debug(f'matches({parent_digits}): reached end of digits: yield {self}') + yield self, parent_alternatives + else: + match_log.debug(f'matches({parent_digits}): reached end of digits: no terminal pattern {self}') + return + if self.representation == '!': + # match arbitrary digits -> consume further digits on this node w/o climbing down + yield from self.matching_nodes(digits=digits, css=css, parent_digits=parent_digits, + parent_alternatives=parent_alternatives) + return + matching_childs = set(chain.from_iterable(self.childs.get(d, []) for d in digit_set)) + # matching_childs = self.childs.get(digit, []) + # filter by CSS + matching_childs = [mc + for mc in matching_childs + if css & mc.partitions] + if not matching_childs: + match_log.debug(f'{self} no match on {digit_repr}') + return + + match_log.debug(f'{len(matching_childs)} matching childs: {", ".join(f"{mc}" for mc in matching_childs)}') + if len(matching_childs) == 1: + yield from next(iter(matching_childs)).matching_nodes(digits=digits, css=css, parent_digits=parent_digits, + parent_alternatives=parent_alternatives) + else: + for digits, child in zip(tee(digits, len(matching_childs)), matching_childs): + yield from child.matching_nodes(digits=digits, css=css, parent_digits=parent_digits, + parent_alternatives=parent_alternatives) + # for + # if + + def lookup(self, digits: str, css: str, tp_depth=0) -> List[Pattern]: + """ + DA Lookup and return matched Pattern + :param digits: digit string to consume + :param css: string representation of the css; colon separated list of partition names + :param tp_depth: translation recursion depth + :return: pattern found as result of DA lookup + """ + if isinstance(css, str): + css_set = set(css.split(':')) + else: + css_set = set(css) + matches = list(self.matching_nodes(digits=digits, css=css)) + if not matches: + return [] + best_match_quality = min(m[1] for m in matches) + best_matches = [match[0] + for match in matches + if match[1] == best_match_quality] + # sorting by match priority makes sure that the best pattern(s) are at the start + matches.sort(key=lambda x: x[1]) + # best_match = matches[0][0] + # get all terminal patterns associated with these best matching nodes where the partition is part of the CSS + terminal_patterns = defaultdict(list) + for best_match in best_matches: + for partition, pattern in best_match.terminal_pattern.items(): + if partition in css_set: + terminal_patterns[partition].append(pattern) + if len(terminal_patterns) == 1: + patterns = next(iter(terminal_patterns.values())) + else: + # we have multiple patterns with the same matching quality in various partitions + # in that case the partition order in the CSS has to be used as a tie_breaker + # iterate through all partitions until terminal_patterns has an entry in that partition + patterns = next(patterns + for partition in css.split(':') + if (patterns := terminal_patterns.get(partition)) is not None) + # pattern is now a list of patterns that match + patterns_after_translation = [] + for pattern in patterns: + if isinstance(pattern, TranslationPattern): + if tp_depth == MAX_TRANSLATION_DEPTH: + continue + # translate and try again + pattern: TranslationPattern + digits, css = pattern.translate(digits, css) + if lookup_after_translation := self.lookup(digits, css, tp_depth=tp_depth + 1): + patterns_after_translation.extend(lookup_after_translation) + else: + patterns_after_translation.append(pattern) + return patterns_after_translation + + def __str__(self): + return f'{self.depth}:{self.full_representation}' + + def __repr__(self): + return f'{self.__class__.__name__}({self})' + + def pretty_lines(self, parent_representation: str = '') -> Generator[str, None, None]: + indent = ' ' * (self.depth * 2) + yield f'{indent}partitions: {", ".join(p or "NONE" for p in sorted(self.partitions))}' + yield f'{indent}pattern types: {", ".join(sorted(f"{pt}" for pt in self.pattern_types))}' + if self.terminal_pattern: + if len(self.terminal_pattern) > 1: + for partition in sorted(self.terminal_pattern): + pattern = self.terminal_pattern[partition] + partition = partition or 'NONE' + yield f'{indent}{self.depth}:{parent_representation}{self.representation}' \ + f': terminal: {pattern}' + pass + else: + pattern = next(iter(self.terminal_pattern.values())) + yield f'{indent}{self.depth}:{parent_representation}{self.representation}' \ + f' terminal: {pattern}' + child_nodes = self.all_child_nodes + for child_node in child_nodes: + digits = sorted(digit + for digit, childs in self.childs.items() + if child_node in childs) + full_representation = f'{parent_representation}{self.representation}' + if len(digits) == 1: + matching = f'{digits[0]}' + else: + if self.ALL_DIGITS - set(digits): + matching = f'[{"".join(digits)}]' + else: + matching = 'X' + matching = f'{full_representation}{matching}' + yield f'{indent}{self.depth}:{full_representation}' \ + f' matching: {matching}' + yield from child_node.pretty_lines(parent_representation=f'{parent_representation}{self.representation}') + + def pretty(self) -> str: + return "\n".join(self.pretty_lines()) diff --git a/playbooks/ucm-config-analyzer/src/env.template b/playbooks/ucm-config-analyzer/src/env.template new file mode 100644 index 0000000..9b06b0b --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/env.template @@ -0,0 +1,6 @@ +# Copy to .env and adjust values. Used by type_user_association.py when run via main(). +# main.py and simple.py do not require environment variables. + +# Absolute or relative path to a UCM bulk-export .tar file. +# Example: /path/to/ucm_export.tar +TAR_FILE= diff --git a/playbooks/ucm-config-analyzer/src/main.py b/playbooks/ucm-config-analyzer/src/main.py new file mode 100755 index 0000000..a05fa92 --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/main.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python +""" +UCM Config Analyzer — entry point for the interactive analysis menu. + +What this does: + Discovers all *.tar UCM BAT exports in the current working directory and runs + app.App with an interactive CLI of migration-oriented analyses and Plotly charts. + +What this does NOT do: + Does not connect to live UCM or Webex APIs. Not hardened for production use. + +Environment: + No required variables. Run from the directory that contains your export .tar file(s). +""" +import glob +import logging +import os + +from app import App + +log = logging.getLogger(__name__) + + +def main(): + logging.basicConfig(level=logging.DEBUG, format='%(asctime)s: %(message)s') + logging.getLogger('digit_analysis.node.traversal').setLevel(logging.INFO) + logging.getLogger('user_dependency_graph').setLevel(logging.INFO) + logging.getLogger('ucmexport').setLevel(logging.INFO) + cur_dir = os.getcwd() + tar_files = glob.glob(os.path.join(cur_dir, '*.tar')) + tar_files = sorted(map(os.path.basename, tar_files)) + assert tar_files, f'No TAR files found in {cur_dir}' + app = App(tar_files=tar_files) + app.run() + return + + +if __name__ == '__main__': + main() diff --git a/playbooks/ucm-config-analyzer/src/reduce_tar.py b/playbooks/ucm-config-analyzer/src/reduce_tar.py new file mode 100755 index 0000000..8d4525e --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/reduce_tar.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python +""" +Analyse a UCM config export TAR file: look at enduser.csv and identify columns only used by very few users as +candidates for elimination +""" +import logging +import os +import re +from collections import defaultdict +from collections.abc import Iterable +from csv import DictReader +from functools import reduce +from io import TextIOWrapper +from itertools import chain +from tarfile import TarFile + +from dotenv import load_dotenv + + +def progress(items): + print(f'reading ', end='', flush=True) + i = 0 + for i, e in enumerate(items): + if i % 200 == 0: + print('.', end='', flush=True) + yield e + print(f', got {i + 1} records') + + +def column_repr(columns: Iterable[str]) -> str: + # group columm names that are something like 'fooo nnnnn' + col_name_re = re.compile(r'^(.+) (\d+)$') + columns_by_prefix: dict[str, list[str]] = defaultdict(list) + r = list() + for column_name in columns: + match = col_name_re.match(column_name) + if match: + columns_by_prefix[match.group(1)].append(match.group(2)) + else: + columns_by_prefix[column_name].append(column_name) + for column_name in sorted(columns_by_prefix): + suffixes = columns_by_prefix[column_name] + if len(suffixes) == 1: + try: + r.append(f'{column_name} {int(suffixes[0])}') + except ValueError: + r.append(column_name) + continue + suffixes: list[int] = sorted(map(int, suffixes)) + start_suffix = None + column_entry = f'{column_name}' + for suffix in suffixes: + if start_suffix is None: + start_suffix = suffix + elif suffix != prev_suffix + 1: + column_entry = f'{column_entry} {start_suffix}-{suffix}' + start_suffix = None + prev_suffix = suffix + # for + if start_suffix is not None: + column_entry = f'{column_entry} {start_suffix}' + if suffix != start_suffix: + column_entry = f'{column_entry}-{suffix}' + r.append(column_entry) + # for + return ', '.join(r) + + +def evaluate(): + load_dotenv() + tar_file = os.getenv('TAR_FILE') + if not tar_file or not os.path.isfile(tar_file): + raise ValueError(f'{tar_file} is not a file') + + csv_file = 'enduser.csv' + with TarFile(name=tar_file, mode='r') as tar: + file = TextIOWrapper(tar.extractfile(member=csv_file), encoding='utf-8') + + def upper_first_line(it): + first_line = next(it) + first_line_upper = first_line.upper() + if first_line != first_line_upper: + logging.warning(f'found lowercase header in {csv_file}') + return chain([first_line_upper], it) + + file = upper_first_line(file) + csv_reader = DictReader(file, delimiter=',', doublequote=True, escapechar=None, quotechar='"', + skipinitialspace=True, strict=True) + # try to identify empty columns + col_usage: dict[str, set[str]] = defaultdict(set) + key_field = 'USER ID' + for row_number, row in enumerate(progress(csv_reader)): + key = row[key_field] + for col in (col for col, value in row.items() if value): + col_usage[col].add(key) + print(f'Read {row_number + 1} records from {csv_file}') + print(f'{len(csv_reader.fieldnames)} columns in {csv_file}') + # group columns be number of users using them + col_by_usage: dict[int:list[str]] = reduce(lambda r, e: r[len(e[1])].append(e[0]) or r, + col_usage.items(), + defaultdict(list)) + col_by_usage: list[tuple[int, list[str]]] = sorted(col_by_usage.items(), key=lambda x: x[0]) + for usage, columns in col_by_usage: + if usage > 10: + break + print(f'{len(columns)} columns with {usage} users') + users = sorted(set(chain.from_iterable(col_usage[col] for col in columns))) + print(f' users: {", ".join(users)}') + user_len = max(len(user) for user in users) + for user in users: + user_columns = set(col + for col in columns + if user in col_usage[col]) + print(f' {user:{user_len}}: {column_repr(user_columns)}') + # determine the column indices for the columns + col_indices = set() + for i, column_name in enumerate(csv_reader.fieldnames): + if column_name in user_columns: + col_indices.add(i) + print(f' {" ":{user_len}} min field index: {min(col_indices)}, max field index: {max(col_indices)}') + # for + # for + +if __name__ == '__main__': + evaluate() diff --git a/playbooks/ucm-config-analyzer/src/requirements.txt b/playbooks/ucm-config-analyzer/src/requirements.txt new file mode 100644 index 0000000..0730321 --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/requirements.txt @@ -0,0 +1,29 @@ +-i https://pypi.org/simple +annotated-types==0.7.0 +contourpy==1.3.0 +cycler==0.12.1 +fonttools==4.54.1 +importlib-resources==6.4.5 +kiwisolver==1.4.7 +matplotlib==3.9.2 +networkx==3.2.1 +# numpy 2.0.x lacks stable wheels for Python 3.13; 2.2.x maintains API compatibility for this codebase. +numpy==2.2.4 +packaging==24.1 +pandas==2.2.3 +pillow==10.4.0 +plotly==5.24.1 +pydantic==2.9.2 +pydantic-core==2.23.4 +pyparsing==3.1.4 +python-dateutil==2.9.0.post0 +python-dotenv==1.0.1 +pytz==2024.2 +# scipy 1.13.x can require a source build on Python 3.13; 1.15.x ships wheels. +scipy==1.15.2 +six==1.16.0 +tenacity==9.0.0 +tqdm==4.66.5 +typing-extensions==4.12.2 +tzdata==2024.2 +zipp==3.20.2 diff --git a/playbooks/ucm-config-analyzer/src/simple.py b/playbooks/ucm-config-analyzer/src/simple.py new file mode 100644 index 0000000..24ccdb8 --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/simple.py @@ -0,0 +1,9 @@ +from ucmexport import Proxy + +if __name__ == '__main__': + proxy = Proxy('sample.tar') + phones = proxy.phones.list + print(f'{len(phones)} phones in TAR') + phones_w_multiple_lines = [phone for phone in phones + if len(phone.lines)> 1] + print(f'{len(phones_w_multiple_lines)} phones with multiple lines') \ No newline at end of file diff --git a/playbooks/ucm-config-analyzer/src/test/__init__.py b/playbooks/ucm-config-analyzer/src/test/__init__.py new file mode 100644 index 0000000..b1c2613 --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/test/__init__.py @@ -0,0 +1,8 @@ +import glob +import os + +__all__ = ['TAR_FILE'] + +tar_files = sorted(glob.glob(os.path.realpath(os.path.join(os.path.dirname(__file__), '..', '*.tar')))) +TAR_FILE = tar_files[min(1, len(tar_files))] +tar_files = None diff --git a/playbooks/ucm-config-analyzer/src/test/proxytestcase.py b/playbooks/ucm-config-analyzer/src/test/proxytestcase.py new file mode 100644 index 0000000..1461d68 --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/test/proxytestcase.py @@ -0,0 +1,21 @@ +from unittest import TestCase +from ucmexport import Proxy +import logging +from test import TAR_FILE + +from typing import Optional + + +class ProxyTestCase(TestCase): + proxy: Optional[Proxy] = None + + @classmethod + def setUpClass(cls) -> None: + cls.proxy = Proxy(tar=TAR_FILE) + + @classmethod + def tearDownClass(cls) -> None: + cls.proxy = None + + def setUp(self) -> None: + logging.basicConfig(level=logging.DEBUG) diff --git a/playbooks/ucm-config-analyzer/src/test/test_callpickupgroup.py b/playbooks/ucm-config-analyzer/src/test/test_callpickupgroup.py new file mode 100644 index 0000000..cd6d283 --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/test/test_callpickupgroup.py @@ -0,0 +1,44 @@ +from test.proxytestcase import ProxyTestCase +from ucmexport import Proxy, CallPickupGroup +from typing import Optional +from test import TAR_FILE +import logging + + +class TestPickupGroup(ProxyTestCase): + + def test_list(self): + cpgs = self.proxy.call_pickup_group.list + self.assertIsInstance(cpgs, list) + self.assertGreater(len(cpgs), 0) + for cpg in cpgs: + self.assertIsInstance(cpg, CallPickupGroup, f'Not a CallPickupGroup: {cpg}') + + def test_at_least_one_cpg_has_associated_cpgs(self): + cpgs = self.proxy.call_pickup_group.list + self.assertTrue(any(cpg.associated_cpgs + for cpg in cpgs)) + + def test_number_of_cpgs_with_associated_cpgs(self): + cpgs = self.proxy.call_pickup_group.list + with_associated_cpgs = [cpg for cpg in cpgs if cpg.associated_cpgs] + print(f'{len(cpgs)} call pickup groups') + print(f'{len(with_associated_cpgs)} call pickup groups with associated CPGs') + + def test_cpgs_wo_associated_cpgs(self): + cpgs = self.proxy.call_pickup_group.list + wo_assoc = [cpg + for cpg in cpgs + if not cpg.associated_cpgs] + if not wo_assoc: + return + print(f'{len(wo_assoc)} CPGs w/o associated CPGs') + for cpg in wo_assoc: + print(f'{cpg.name} {cpg.number_and_partition}') + + def test_associated_cpgs(self): + cpgs = self.proxy.call_pickup_group.list + for cpg in cpgs: + associated = set(cpg.associated_cpgs) + if associated: + self.assertIn(cpg.name, associated) diff --git a/playbooks/ucm-config-analyzer/src/test/test_cpg_and_dn.py b/playbooks/ucm-config-analyzer/src/test/test_cpg_and_dn.py new file mode 100644 index 0000000..778c8cc --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/test/test_cpg_and_dn.py @@ -0,0 +1,17 @@ +from test.proxytestcase import ProxyTestCase + +import logging + + +class TestCpgAndDn(ProxyTestCase): + + def setUp(self) -> None: + logging.basicConfig(level=logging.DEBUG) + + def test_dns(self): + dirns = self.proxy.directory_number.list + dirns_w_cpg = [dn for dn in dirns if dn.call_pickup_group] + by_cpg = {cpg: dn_list + for cpg, dn_list in self.proxy.directory_number.by_call_pickup_group.items() + if cpg and len(dn_list) > 1} + foo = 1 diff --git a/playbooks/ucm-config-analyzer/src/test/test_cpg_and_phone.py b/playbooks/ucm-config-analyzer/src/test/test_cpg_and_phone.py new file mode 100644 index 0000000..8c85944 --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/test/test_cpg_and_phone.py @@ -0,0 +1,32 @@ +from test.proxytestcase import ProxyTestCase + + +class TestCpgAndDn(ProxyTestCase): + + def test_by_cpg(self): + by_cpg = self.proxy.phones.by_call_pickup_group + self.assertTrue(by_cpg) + + def test_by_cpg_set(self): + by_cpg = self.proxy.phones.by_call_pickup_group + self.assertTrue(all(isinstance(phone_set, set) + for phone_set in by_cpg.values())) + + def test_by_cpg_set_len(self): + by_cpg = self.proxy.phones.by_call_pickup_group + self.assertTrue(all(phone_set + for phone_set in by_cpg.values())) + + def test_no_empty_cpg(self): + by_cpg = self.proxy.phones.by_call_pickup_group + self.assertIsNone(by_cpg.get('')) + + def test_dpgs_exist(self): + by_cpg = self.proxy.phones.by_call_pickup_group.keys() + cpg_names = set(cpg.name + for cpg in self.proxy.call_pickup_group.list) + undefined_cpgs_names = [cpg_name + for cpg_name in by_cpg + if cpg_name not in cpg_names] + self.assertFalse(undefined_cpgs_names, + f'CPGs used on phones but not defined in CPG csv: {", ".join(undefined_cpgs_names)}') diff --git a/playbooks/ucm-config-analyzer/src/test/test_da.py b/playbooks/ucm-config-analyzer/src/test/test_da.py new file mode 100644 index 0000000..acaab12 --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/test/test_da.py @@ -0,0 +1,202 @@ +# Digit analysis tests +from unittest import TestCase +from digit_analysis import DaNode, DnPattern, TranslationPattern, RoutePattern +from digit_analysis.node import MAX_TRANSLATION_DEPTH + +from itertools import chain +from collections import Counter +import re + +class TestDp(TestCase): + DNS = [ + '\\+14085551001', '\\+14085551002', '\\+14085551005', '\\+14085551010', '\\+14085551011', '\\+14085551021', + '\\+14085551022', + '\\+19195551001', '\\+19195551002', '\\+19195551005', '\\+19195551099' + ] + + TPS = [ + ('SJC', '10XX', '+14085551XXX'), + ('RTP', '10XX', '+19195551XXX'), + ('DN', '\\+19195551XXX', '+19195551002'), + ('RTP', '5XXX', '+19195551005'), + ('RTP', '51XX', '+19195551099'), + ('FREAKTP', '9001', '9002'), + ('FREAKTP', '9002', '9003'), + ('FREAKTP', '9003', '9004'), + ('FREAKTP', '9004', '9005'), + ('FREAKTP', '9005', '9006'), + ('FREAKTP', '9006', '9007'), + ('FREAKTP', '9007', '9008'), + ('FREAKTP', '9008', '5001') + ] + + RPS = [ + ('RP1', '5XXX'), + ('RP2', '5XXX') + ] + + @classmethod + def setUpClass(cls) -> None: + cls.da_tree = DaNode() + for dn in cls.DNS: + dn_pattern = DnPattern(pattern=dn, partition='DN') + cls.da_tree.add_pattern(dn_pattern) + for partition, pattern, called_party_transform in cls.TPS: + tp = TranslationPattern(pattern=pattern, + partition=partition, + use_originators_calling_search_space=True, + called_party_mask=called_party_transform) + cls.da_tree.add_pattern(tp) + for partition, pattern in cls.RPS: + rp = RoutePattern(pattern=pattern, partition=partition) + cls.da_tree.add_pattern(rp) + + def test_lookup_1001_sjc(self): + css = 'DN:SJC' + lookup_result = self.da_tree.lookup(digits='1001', css=css) + self.assertIsNotNone(lookup_result) + self.assertEqual(len(lookup_result), 1) + lookup_result = lookup_result[0] + self.assertIsInstance(lookup_result, DnPattern) + self.assertEqual(lookup_result.partition, 'DN') + self.assertEqual(lookup_result.pattern, '\\+14085551001') + + def test_lookup_1003_sjc(self): + """ + Dialing 1003 in SJC should fail b/c after translation there is no DN ...1003 + :return: + """ + css = 'DN:SJC' + lookup_result = self.da_tree.lookup(digits='1003', css=css) + self.assertIsInstance(lookup_result, list) + self.assertFalse(lookup_result) + + def test_lookup_1001_existing_dn(self): + css = 'DN:RTP' + for numeric_digits in [1001, 1002, 1005, 1099]: + digits = str(numeric_digits) + lookup_result = self.da_tree.lookup(digits=digits, css=css) + self.assertIsNotNone(lookup_result) + self.assertEqual(len(lookup_result), 1) + lookup_result = lookup_result[0] + self.assertIsInstance(lookup_result, DnPattern) + self.assertEqual(lookup_result.partition, 'DN') + self.assertEqual(lookup_result.pattern, f'\\+1919555{digits}') + + def test_lookup_1003_rtp(self): + css = 'DN:RTP' + lookup_result = self.da_tree.lookup(digits='1003', css=css) + self.assertIsNotNone(lookup_result) + self.assertEqual(len(lookup_result), 1) + lookup_result = lookup_result[0] + self.assertIsInstance(lookup_result, DnPattern) + self.assertEqual(lookup_result.partition, 'DN') + self.assertEqual(lookup_result.pattern, '\\+19195551002') + + def test_lookup_5101_rtp(self): + css = 'DN:RTP' + lookup_result = self.da_tree.lookup(digits='5101', css=css) + self.assertIsNotNone(lookup_result) + self.assertEqual(len(lookup_result), 1) + lookup_result = lookup_result[0] + self.assertIsInstance(lookup_result, DnPattern) + self.assertEqual(lookup_result.partition, 'DN') + self.assertEqual(lookup_result.pattern, '\\+19195551099') + + def test_lookup_incomplete_dn(self): + css = 'DN:SJC' + # dialing an incomplete dial string should fail + lookup_result = self.da_tree.lookup(digits='+1408555', css=css) + self.assertIsInstance(lookup_result, list) + self.assertFalse(lookup_result) + + def test_matching_nodes_1099(self): + css = 'DN:RTP' + # dialing should give us two DA Nodes + matches = list(self.da_tree.matching_nodes(digits='+19195551099', css=css)) + self.assertEqual(len(matches), 2) + # sort by quality + matches.sort(key=lambda x: x[1]) + self.assertEqual(matches[0][0].full_representation, '+19195551099') + self.assertEqual(matches[1][0].full_representation, '+19195551XXX') + + def test_partition_order(self): + lookup_result = self.da_tree.lookup(digits='5001', css='RP1:RP2') + self.assertIsNotNone(lookup_result) + self.assertEqual(len(lookup_result), 1) + lookup_result = lookup_result[0] + self.assertIsInstance(lookup_result, RoutePattern) + self.assertEqual(lookup_result.partition, 'RP1') + self.assertEqual(lookup_result.pattern, '5XXX') + + lookup_result = self.da_tree.lookup(digits='5001', css='RP2:RP1') + self.assertIsNotNone(lookup_result) + self.assertEqual(len(lookup_result), 1) + lookup_result = lookup_result[0] + self.assertIsInstance(lookup_result, RoutePattern) + self.assertEqual(lookup_result.partition, 'RP2') + self.assertEqual(lookup_result.pattern, '5XXX') + + def test_tp_recursion(self): + for i in range(MAX_TRANSLATION_DEPTH + 1): + pre_transform = f'{9008 - i}' + lookup_result = self.da_tree.lookup(digits=pre_transform, css='FREAKTP:RP2') + self.assertIsInstance(lookup_result, list) + if i < MAX_TRANSLATION_DEPTH: + self.assertIsNotNone(lookup_result) + self.assertEqual(len(lookup_result), 1) + lookup_result = lookup_result[0] + self.assertIsInstance(lookup_result, RoutePattern) + self.assertEqual(lookup_result.partition, 'RP2') + self.assertEqual(lookup_result.pattern, '5XXX') + else: + self.assertFalse(lookup_result) + + def test_terminal_nodes(self): + patterns = list(chain(self.DNS, + (pattern for _, pattern, _ in self.TPS), + (pattern for _, pattern in self.RPS))) + pattern_count = Counter(patterns) + for node in self.da_tree.terminal_nodes(): + pattern = node.full_representation + if pattern.startswith('+'): + pattern = f'\\{pattern}' + self.assertIsNotNone(pattern_count.get(pattern), f'No count for {pattern}') + pattern_count[pattern] -= len(node.terminal_pattern) + # is there any pattern count not zero + pattern_count = {p: c + for p, c in pattern_count.items() + if c} + self.assertFalse(pattern_count, f'Some pattern counts are not zero: {pattern_count}') + + def test_unique_terminal_nodes(self): + node_set = set() + repr_set = set() + for node in self.da_tree.terminal_nodes(): + self.assertTrue(node not in node_set, f'{node} already in {node_set}') + node_set.add(node) + self.assertTrue(node.full_representation not in repr_set, + f'{node.full_representation} already in {repr_set}') + + def test_breadth_first_traversal_each_node_once(self): + """ + Each node should only be traversed once + :return: + """ + nodes = set() + for node in self.da_tree.breadth_first_traversal(): + self.assertTrue(node not in nodes, f'{node} already in {nodes}') + nodes.add(node) + + + def test_lookup_w_wildcards(self): + lookup_result = self.da_tree.lookup(digits='10XX', css='SJC:DN') + self.assertIsInstance(lookup_result, list) + self.assertEqual(len(lookup_result), 7) + # All patterns should be DNs + not_dn = [dnp for dnp in lookup_result if not isinstance(dnp, DnPattern)] + self.assertFalse(not_dn, f'Some lookup results are not DNs: {not_dn}') + # All patterns should be in SJC number range + sjc_re = re.compile(r'\\\+14085551\d{3}') + dn_not_sjc = [dnp for dnp in lookup_result if not sjc_re.match(dnp.pattern)] + self.assertFalse(dn_not_sjc, f'Some DNs are not in SJC: {dn_not_sjc}' ) \ No newline at end of file diff --git a/playbooks/ucm-config-analyzer/src/test/test_phonebuttontemplate.py b/playbooks/ucm-config-analyzer/src/test/test_phonebuttontemplate.py new file mode 100644 index 0000000..2013397 --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/test/test_phonebuttontemplate.py @@ -0,0 +1,119 @@ +from test.proxytestcase import ProxyTestCase +from ucmexport import PhoneButtonTemplate, Css, SpeedDial +from digit_analysis import DaNode, DnPattern +from itertools import chain + +import logging + +from typing import List + + +class TestPbt(ProxyTestCase): + def test_1_list(self): + pbts = self.proxy.phone_button_template.list + self.assertIsInstance(pbts, list) + + def test_2_len(self): + pbts = self.proxy.phone_button_template.list + self.assertTrue(pbts) + + def test_3_pbt(self): + pbts = self.proxy.phone_button_template.list + self.assertTrue(all(isinstance(pbt, PhoneButtonTemplate) for pbt in pbts)) + + def test_4_pbt_button_types(self): + pbts = self.proxy.phone_button_template.list + feature_types = sorted(list(set(chain.from_iterable((button.feature_type + for button in pbt.buttons) + for pbt in pbts)))) + self.assertTrue(feature_types) + print(f'Phone button template feature types: {", ".join(feature_types)}') + + +class TestPbtAndPhone(ProxyTestCase): + + def setUp(self) -> None: + super(TestPbtAndPhone, self).setUp() + logging.getLogger('digit_analysis.node.match').setLevel(logging.INFO) + + def test_sd_blf(self): + # prepare DA lookup + da_tree = DaNode.from_proxy(self.proxy) + + # find phones with SD BLF + pbt_w_sd_blf = [pbt + for pbt in self.proxy.phone_button_template.list + if any(button.feature_type == 'Speed Dial BLF' + for button in pbt.buttons)] + phones_w_sd_blf = {pbt.name: phones + for pbt in pbt_w_sd_blf + if (phones := self.proxy.phones.by_phone_button_template.get(pbt.name))} + for pbt, phones in phones_w_sd_blf.items(): + for phone in phones: + # some speed dials are actually BLFs based on the phone button template + pbt = self.proxy.phone_button_template.get(phone.phone_button_template) + # all speed dials (or speed dial BLFs) in the phone button template + pbt_sds = (button + for button in pbt.buttons + if button.feature_type.startswith('Speed Dial')) + # Indices of speed dial BLFs in the list of speed dials + sd_blf_indices = (i + for i, sd in enumerate(pbt_sds, start=1) + if sd.feature_type == 'Speed Dial BLF') + # use these indices to pick the speed dials from the phone + # no all SD BLS in the PBT might actually be set on the phone + sd_blfs: List[SpeedDial] + sd_blfs = [speed_dial + for sd_index in sd_blf_indices + if (speed_dial := phone.speed_dials.get(sd_index))] + # css + + phone_css = phone.css + if phone_css: + phone_css = self.proxy.css.get(phone_css) + self.assertIsNotNone(phone_css) + phone_css = phone_css.partitions_string + else: + phone_css = '' + line_css = phone.lines[1].css + if line_css: + line_css = self.proxy.css.get(line_css) + self.assertIsNotNone(line_css) + line_css = line_css.partitions_string + else: + line_css = '' + effective_css = ':'.join([line_css, phone_css]) + for sd_blf in sd_blfs: + # noinspection PyTypeChecker + lookup = da_tree.lookup(digits=sd_blf.number, + css=effective_css) + print(f'Da lookup for SD BLF "{sd_blf}" yields: "{lookup}"') + self.assertIsNotNone(lookup) + self.assertIsInstance(lookup, list) + self.assertEqual(len(lookup), 1) + lookup = lookup[0] + print(f'Da lookup for SD BLF "{sd_blf}" yields: "{lookup}"') + self.assertIsInstance(lookup, DnPattern) + lookup: DnPattern + phones_with_dn = self.proxy.phones.by_dn_and_partition.get(lookup.dn_and_partition) + self.assertTrue(phones_with_dn) + users = set(chain.from_iterable(phone.user_set for phone in phones_with_dn)) + self.assertTrue(users) + print(f'Users with phones with that DN: {", ".join(users)}') + # for + # for + + def test_intercom(self): + """ + Not really a test: just trying to identify phones using a phone button template with an intercom line + :return: + """ + # PBTs with at least one Intercom button + pbts = [pbt + for pbt in self.proxy.phone_button_template.list + if any(button.feature_type=='Intercom' for button in pbt.buttons)] + print(f'Found {len(pbts)} phone button templates with intercom: {", ".join(f"{pbt}" for pbt in pbts)}') + for pbt in pbts: + phones_w_pbt = self.proxy.phones.by_phone_button_template.get(pbt.name, []) + print(f'{pbt}: {len(phones_w_pbt)} phones: {", ".join(f"{phone}" for phone in phones_w_pbt)}') + diff --git a/playbooks/ucm-config-analyzer/src/test/test_translation.py b/playbooks/ucm-config-analyzer/src/test/test_translation.py new file mode 100644 index 0000000..c2e8631 --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/test/test_translation.py @@ -0,0 +1,137 @@ +from unittest import TestCase +from test.proxytestcase import ProxyTestCase + +from digit_analysis import TranslationPattern, TranslationError +from ucmexport import TranslationPattern as CSVTranslationPattern + +import logging + + +class TestTpProxy(ProxyTestCase): + def setUp(self) -> None: + super(TestTpProxy, self).setUp() + logging.getLogger('digit_analysis.base').setLevel(logging.DEBUG) + + def test_0_list(self): + tps = self.proxy.translation_pattern.list + self.assertIsInstance(tps, list, f'{tps.__class__}') + + def test_1_not_empty(self): + tps = self.proxy.translation_pattern.list + self.assertTrue(tps) + + def test_2_istances(self): + tps = self.proxy.translation_pattern.list + self.assertTrue(all(isinstance(tp, CSVTranslationPattern) for tp in tps)) + + def test_3_discard_digits(self): + tps = self.proxy.translation_pattern.list + discard_digits = set(tp.discard_digits for tp in tps) + print(', '.join(f'"{dd}"' for dd in discard_digits)) + + def test_4_pre_dot_tps(self): + """ + Not really a test: print a list of TPs w/ "PreDot" digit discard instruction + :return: + """ + tps = [tp + for tp in self.proxy.translation_pattern.list + if tp.discard_digits == 'PreDot'] + for tp in tps: + print(f'{tp.pattern} prefix: {tp.called_party_prefix_digits} mask: {tp.called_party_mask}') + + +class TestTp(TestCase): + def setUp(self) -> None: + super(TestTp, self).setUp() + logging.basicConfig(level=logging.DEBUG) + logging.getLogger('digit_analysis.base').setLevel(logging.DEBUG) + + def test_translate_2234_to_1234(self): + tp = TranslationPattern(pattern='2XXX', + css=['translated'], + called_party_mask='1XXX') + translated, css = tp.translate(digits='2234', css='in') + self.assertEqual(translated, '1234') + self.assertEqual(css, 'translated') + + def test_translate_2345_to_1234_no_X_in_mask(self): + tp = TranslationPattern(pattern='2XXX', + css=['translated'], + called_party_mask='1234') + translated, css = tp.translate(digits='2345', css='in') + self.assertEqual(translated, '1234') + self.assertEqual(css, 'translated') + + def test_translate_ESN_to_e164(self): + tp = TranslationPattern(pattern='84969XXX', + css=['translated'], + called_party_mask='+4961007739XXX', + use_originators_calling_search_space=True) + translated, css = tp.translate(digits='84969764', css='in') + self.assertEqual(translated, '+4961007739764') + self.assertEqual(css, 'in') + + def test_translate_pre_dot(self): + tp = TranslationPattern(pattern='000.49!', + css=['translated'], + discard_digits='PreDot', + use_originators_calling_search_space=True) + translated, css = tp.translate(digits='000491', css='in') + self.assertEqual(translated, '491') + self.assertEqual(css, 'in') + + def test_translate_pre_dot_prefix(self): + tp = TranslationPattern(pattern='000.49!', + css=['translated'], + discard_digits='PreDot', + called_party_prefix_digits='+', + use_originators_calling_search_space=True) + translated, css = tp.translate(digits='000491', css='in') + self.assertEqual(translated, '+491') + self.assertEqual(css, 'in') + + def test_missing_dot(self): + tp = TranslationPattern(pattern='00049!', + css=['translated'], + discard_digits='PreDot', + called_party_prefix_digits='+', + use_originators_calling_search_space=True) + self.assertRaises(TranslationError, lambda: tp.translate(digits='000491', css='in')) + + def test_pre_dot_with_mask(self): + tp = TranslationPattern(pattern='000.49!', + css=['translated'], + discard_digits='PreDot', + called_party_mask='XX', + use_originators_calling_search_space=False) + translated, css = tp.translate(digits='000491', css='in') + self.assertEqual('91', translated) + self.assertEqual(css, 'translated') + + def test_transform_wildcard_set(self): + tp = TranslationPattern(pattern='1.234XX', + css=['translated'], + discard_digits='PreDot', + called_party_mask='XXXXX') + translated, css = tp.translate(digits='123456', css='in') + self.assertEqual(css, 'translated') + self.assertEqual(translated, '23456') + + def test_transform_wildcard_set_1(self): + tp = TranslationPattern(pattern='1.234XX', + css=['translated'], + discard_digits='PreDot', + called_party_mask='XXX') + translated, css = tp.translate(digits='123456', css='in') + self.assertEqual(css, 'translated') + self.assertEqual(translated, '456') + + def test_transform_wildcard_digit_string(self): + tp = TranslationPattern(pattern='1.234XX', + css=['translated'], + discard_digits='PreDot', + called_party_mask='XXX') + translated, css = tp.translate(digits='123X[14-69]X', css='in') + self.assertEqual(css, 'translated') + self.assertEqual(translated, 'X[14569]X') diff --git a/playbooks/ucm-config-analyzer/src/transform_tar.py b/playbooks/ucm-config-analyzer/src/transform_tar.py new file mode 100755 index 0000000..6aef6a3 --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/transform_tar.py @@ -0,0 +1,275 @@ +#!/usr/bin/env python +""" +Transform a UCM config export TAR file by applying transformations to the CSV files within it +""" +import csv +import io +import os +import re +import tarfile +import time +from argparse import ArgumentParser +from collections.abc import Callable +from contextlib import contextmanager +from functools import partial +from io import TextIOWrapper, TextIOBase +from tarfile import TarFile +from typing import Optional + +# columns to remove from phones.csv +PHONE_CSV_EXCLUDED_FIELDS = ['Services Provisioning', 'CSS', 'AAR CSS', 'Network Locale', 'Media Resource Group List', + 'User Hold MOH Audio Source', 'Network Hold MOH Audio Source', 'Device User Locale', + 'Packet Capture Mode', 'Packet Capture Duration', 'Built in Bridge', 'Privacy', + 'Retry Video Call as Audio', 'Ignore Presentation Indicators', 'Module', + 'Phone Load Name', 'Module # Load Name', 'Information', 'Directory', 'Messages', + 'Services', 'Authentication Server', 'Proxy Server', 'Idle', 'Idle Timer', + 'MLPP Indication', 'MLPP Preemption', 'MLPP Domain', 'MTP Required', 'Digest User', + 'Allow CTI Control Flag', 'Device Presence Group', 'Device Security Profile', + 'Device Subscribe CSS', 'Unattended Port', 'Require DTMF Reception', 'RFC2833 Disabled', + 'Certificate Operation', 'Authentication String', 'Operation Completes By', + 'Device Protocol', 'Secure Shell User', 'Secure Shell Password', 'XML', 'Dial Rules', + 'CSS Reroute', 'Rerouting Calling Search Space', 'DTMF Signalling', + 'Default DTMF Capability', + 'MTP Preferred Originating Codec', 'Logout Profile', 'Signaling Port', + 'Outgoing Caller ID Pattern', + 'Calling Party Selection', 'Calling Party Presentation', 'Display IE Delivery', + 'Redirecting Number IE Delivery Outbound', 'Redirecting Number IE Delivery Inbound', + 'Gatekeeper Name', 'Technology Prefix', 'Zone', 'Motorola WSM Connection', + 'Subscriber Cellular Number', 'Follow me only when caller has dialed cellular num', + 'Disable Application Dial Rules', 'AAR Group', 'Logged Into Hunt Group', 'Remote Device', + 'Device Mobility Mode', 'DND Option', 'DND Incoming Call Alert', + 'BLF Audible Alert Setting (Phone Busy)', + 'BLF Audible Alert Setting (Phone Idle)', 'Protected Device', 'Join Across Lines', + 'Single Button Barge', + 'Application User', 'Always Use Prime Line', 'Always Use Prime Line for Voice Message', + 'Use Trusted Relay Point', 'Outbound Call Rollover', 'Phone Personalization', + 'Primary Phone', + 'Hotline Device', 'Secure Directory URL', 'Secure Idle URL', + 'Secure Information URL', 'Secure Messages URL', 'Secure Services URL', 'SRTP Allowed', + 'Feature Control Policy', 'Device Trust Mode', 'Allow presentation sharing using BFCP', + 'Early Offer support for voice and video calls (insert MTP if needed)', + 'Caller ID Calling Party Transformation CSS', + 'Caller ID Use Device Pool Calling Party Transformation CSS', + 'Remote Number Calling party Transformation CSS', + 'Remote Number Use Device Pool Calling Party Transformation CSS', + 'Allow iX Applicable Media', 'Require off-premise location', 'Confidential Access Mode', + 'Confidential Access Level', + 'Route calls to all remote destinations when client is not connected', + 'Emergency Location (ELIN) Group', 'Third-party Registration Required', + 'Block Incoming Calls while Roaming', + 'Home Network ID', 'Mobility Identity Name', 'Mobility Identity Destination Number', + 'Mobility Identity Answer Too Soon Timer', 'Mobility Identity Answer Too Late Timer', + 'Mobility Identity Delay Before Ringing Cell', 'Mobility Identity Time Zone', + 'Mobility Identity Is Mobile Phone', + 'Mobility Identity Enable Mobile Connect', 'Mobility Identity Mobility Profile', + 'Line CSS', + 'AAR Group(Line)', 'Line User Hold MOH Audio Source', 'Line Network Hold MOH Audio Source', + 'Auto Answer', 'Forward All CSS', 'Forward Busy Internal CSS', 'Forward Busy External CSS', + 'Forward No Answer Internal CSS', 'Forward No Answer External CSS', + 'Forward No Coverage Internal CSS', + 'Forward No Coverage External CSS', 'MLPP Target', 'MLPP CSS', + 'MLPP No Answer Ring Duration', + 'Busy Trigger', 'Visual Message Waiting Indicator Policy', 'Ring setting (Phone Idle)', + 'Ring Setting (Phone Active)', 'Caller Name', 'Caller Number', 'Redirected Number', + 'Dialed Number', 'Line Description', 'Line Presence Group', + 'Secondary CSS for Forward All', + 'Forward on CTI Failure Voice Mail', 'Forward on CTI Failure Destination', + 'Forward on CTI Failure CSS', 'AAR Destination Mask', 'AAR Voice Mail', + 'Forward Unregistered Internal CSS', + 'Forward Unregistered External CSS', 'Hold Reversion Ring Duration', + 'Hold Reversion Notification Interval', + 'Recording Profile', 'Monitoring Calling Search Space', + 'Calling Search Space Activation Policy', + 'CPG Audio Alert Setting(Phone Idle)', 'CPG Audio Alert Setting(Phone Active)', + 'Park Monitor Forward No Retrieve Ext Destination', + 'Park Monitor Forward No Retrieve Int Destination', + 'Park Monitor Forward No Retrieve Int Voice Mail', + 'Park Monitor Forward No Retrieve Ext Voice Mail', + 'Park Monitor Forward No Retrieve Ext CSS', 'Park Monitoring Reversion Timer', + 'Park Monitor Forward No Retrieve Int CSS', + 'Party Entrance Tone', 'Log Missed Calls', 'Allow Control of Device from CTI', + 'URI # on Directory Number', + 'URI # Route Partition on Directory Number', 'URI # Is Primary on Directory Number', + 'Reject Anonymous Calls', 'Urgent Priority', 'Recording Media Source', + 'Enterprise Is Urgent', + 'Enterprise Advertise via globally', 'Enterprise Add to Local Route Partition', + 'Enterprise Route Partition', + 'E.164 Is Urgent', 'E.164 Advertise via globally', 'E.164 Add to Local Route Partition', + 'E.164 Route Partition', 'Line Confidential Access Mode', 'Line Confidential Access Level', + 'External Call Control Profile', + 'Call Control Agent Profile', 'IsEnterprise Advertised Failover Number', + 'IsE.164 Advertised Failover Number', + 'URI # Advertise Globally via ILS', 'Calling Line ID Presentation When Diverted', + 'Intercom Maximum Number of Calls', 'Intercom Directory Number', + 'Intercom Route Partition', + 'Intercom Description', 'Intercom Alerting Name', 'Intercom ASCII Alerting Name', + 'Intercom CSS', 'Intercom Presence Group', + 'Intercom Display', 'Intercom ASCII Display', 'Intercom Line Text Label', + 'Intercom Speed Dial', + 'Intercom External Phone Number Mask', 'Intercom Caller Name', 'Intercom Caller Number', + 'Intercom Auto Answer', + 'Intercom Default Activated Device'] + +# columns to remove from enduser.csv +ENDUSER_CSV_EXCLUDED_FIELDS = ['ASSOCIATED PC', 'MIDDLE NAME', 'PAGER', 'HOME PHONE', 'BUILDING', 'SITE', + 'UNIQUE IDENTIFIER', + 'NICKNAME', + 'DELETED TIME STAMP', + 'DIGEST CREDENTIALS', 'PRESENCE GROUP', + 'SUBSCRIBE CSS', + 'ALLOW CONTROL OF DEVICE FROM CTI', 'MAX. DESK PICKUP WAIT TIME', + 'REMOTE DESTINATION LIMIT', + 'ENABLE USER FOR UNIFIED CM IM AND PRESENCE', 'ENABLE EMCC', + 'INCLUDE MEETING INFORMATION IN PRESENCE', 'ASSIGNED PRESENCE SERVER', + 'ENABLE END USER TO HOST CONFERENCE NOW', 'MEETING NUMBER', 'ATTENDEES ACCESS CODE', + 'EM MAX LOGIN TIME', 'SELF-SERVICE USER ID', 'PASSWORD LOCKED BY ADMIN', + 'PASSWORD CANT CHANGE', 'PASSWORD MUST CHANGE AT NEXT LOGIN', + 'PASSWORD DOES NOT EXPIRE', + 'PASSWORD AUTHENTICATION RULE', 'PASSWORD', 'PIN LOCKED BY ADMIN', + 'PIN CANT CHANGE', + 'PIN MUST CHANGE AT NEXT LOGIN', 'PIN DOES NOT EXPIRE', 'PIN AUTHENTICATION RULE', + 'PIN', 'APPLICATION SERVER NAME', 'CONTENT', 'ACCESS CONTROL GROUP', + 'DEFAULT PROFILE', 'DEVICE NAME', 'DESCRIPTION', + 'TYPE USER ASSOCIATION', + 'TYPE PATTERN USAGE', + 'NAME DIALING', 'MLPP PRECEDENCE AUTHORIZATION LEVEL', + 'MLPP USER IDENTIFICATION NUMBER', + 'MLPP PASSWORD', 'HEADSET SERIAL NUMBER'] + + +def progress(items): + """ + Primitive progress indicator; print a "." every 200 records + """ + i = 0 + for i, e in enumerate(items): + if i % 200 == 0: + print('.', end='', flush=True) + yield e + print(f', got {i + 1} records', end='', flush=True) + + +def remove_fields(in_file: TextIOBase, fields_to_remove: list[str], max_devices: int=None) -> TextIOBase: + """ + Remove fields from a CSV file + """ + + def single_column(column: str) -> str: + """ + Regular expression to match for a single column + """ + r = f'(?:{column})' + return r + + # buid regex to match columns to remove + col_re = f'^({"|".join(single_column(col) for col in fields_to_remove)})( \d+)?$' + col_re = re.compile(col_re) + + reader = csv.reader(in_file, delimiter=',', doublequote=True, escapechar=None, quotechar='"', + skipinitialspace=True, strict=True) + fieldnames = next(reader) + + # remove matching columns but keep "DEVICE NAME 1" and "TYPE USER ASSOCIATION 1" + keep_1st_few = {'DEVICE NAME', 'TYPE USER ASSOCIATION'} + col_idx_to_remove = {i + for i, field in enumerate(fieldnames) + if ((m := col_re.match(field)) and + (m.group(1) not in keep_1st_few or int(m.group(2)) > max_devices))} + + out_file = io.StringIO() + csv_writer = csv.writer(out_file, delimiter=',', doublequote=True, + escapechar=None, quotechar='"', quoting=csv.QUOTE_MINIMAL, skipinitialspace=True, + strict=True) + col_idx_to_keep = [i for i in range(len(fieldnames)) if i not in col_idx_to_remove] + csv_writer.writerow([fieldnames[i] for i in col_idx_to_keep]) + + print(f'removing {len(col_idx_to_remove)} columns: ', end='', flush=True) + for row in progress(reader): + csv_writer.writerow([row[i] for i in col_idx_to_keep]) + print() + out_file.seek(0) + return out_file + + + + +@contextmanager +def open_output_tar(tar_file: str, read_only: bool) -> Optional[TarFile]: + """ + Conditionally open a TAR file for writing + """ + if read_only: + yield None + return + with TarFile(name=tar_file, mode='w') as out: + yield out + return + + +def transform(): + """ + Main: transform given TAR file + """ + parser = ArgumentParser(description='Transform a UCM config export TAR file by applying transformations to the CSV ' + 'files within it') + parser.add_argument('tar_file', help='the TAR file to transform') + parser.add_argument('--readonly', action='store_true', help='do not write the transformed TAR file') + parser.add_argument('--maxdevices', type=int, help='maximum number of DEVICE NAME columns to keep. Default: 5', default=5) + + args = parser.parse_args() + input_tar = args.tar_file + read_only = args.readonly + max_devices = args.maxdevices + + # definition of transforms to apply to CSV files + csv_transforms: dict[str, list[Callable[[TextIOBase], TextIOBase]]] = { + 'enduser.csv': [partial(remove_fields, fields_to_remove=ENDUSER_CSV_EXCLUDED_FIELDS, max_devices=max_devices)], + 'phone.csv': [partial(remove_fields, fields_to_remove=PHONE_CSV_EXCLUDED_FIELDS)]} + + if not os.path.isfile(input_tar): + raise ValueError(f'{input_tar} is not a file') + with TarFile(name=input_tar, mode='r') as tar: + # filename for the transformed TAR file + out_tar = f'{os.path.splitext(input_tar)[0]}_transformed.tar' + + # conditionally open the output TAR file + with open_output_tar(out_tar, read_only) as out: + # iterate over the members of the TAR file + for member in tar.getmembers(): + member: tarfile.TarInfo + # check if transformations are defined for this file + if transformers := csv_transforms.get(member.name): + print(f'transforming {member.name}', flush=True) + if read_only: + continue + file = TextIOWrapper(tar.extractfile(member=member.name), encoding='utf-8') + for transformer in transformers: + transformed = transformer(file) + file.close() + file = transformed + bytes_io = io.BytesIO(file.read().encode('utf-8')) + file.close() + ti = tarfile.TarInfo(name=member.name) + ti.size = len(bytes_io.getvalue()) + # set mtime to current time + ti.mtime = int(time.time()) + bytes_io.seek(0) + out.addfile(ti, bytes_io) + bytes_io.close() + else: + # no transformers -> copy as is + print(f'copying {member.name} as is', flush=True) + if read_only: + continue + file = tar.extractfile(member) + out.addfile(member, file) + file.close() + # if + # for + # with + # with + return + + +if __name__ == '__main__': + transform() diff --git a/playbooks/ucm-config-analyzer/src/type_user_association.py b/playbooks/ucm-config-analyzer/src/type_user_association.py new file mode 100644 index 0000000..4ab5a6c --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/type_user_association.py @@ -0,0 +1,65 @@ +#!/usr/bin/env python +""" +Read enduser.csv from tar file and get TYPE_USER_ASSOCIATION values +""" +import csv +import glob +import os +from io import TextIOWrapper +from itertools import chain +from tarfile import TarFile + +from dotenv import load_dotenv + +from transform_tar import progress + + +def type_user_association_values(tar_file: str, csv_file: str) -> set[str]: + with TarFile(name=tar_file, mode='r') as tar: + try: + file = TextIOWrapper(tar.extractfile(member=csv_file), encoding='utf-8') + except KeyError: + print(f'No "{csv_file}" in "{tar_file}"') + return set() + reader = csv.reader(file, delimiter=',', doublequote=True, escapechar=None, quotechar='"', + skipinitialspace=True, strict=True) + first_line = next(reader) + type_user_association_columns = [i + for i, column in enumerate(first_line, start=0) + if column.startswith('TYPE USER ASSOCIATION')] + return set(chain.from_iterable((v + for c in type_user_association_columns + if (v := row[c])) + for row in progress(reader))) + + +def main(): + load_dotenv() + tar_file = os.getenv('TAR_FILE') + if not tar_file or not os.path.isfile(tar_file): + raise ValueError(f'{tar_file} is not a file') + csv_file = 'enduser.csv' + values = type_user_association_values(tar_file=tar_file, csv_file=csv_file) + for v in sorted(values): + print(f'* {v}') + + +def print_all(): + all_values = set() + for tar in sorted(glob.glob('*.tar')): + print(f'## {tar}') + values = type_user_association_values(tar_file=tar, csv_file='enduser.csv') + print() + for v in sorted(values): + print(f'* {v}') + print() + all_values.update(values) + + print('## All values') + for v in sorted(all_values): + print(f'* {v}') + + +if __name__ == '__main__': + print_all() + # main() diff --git a/playbooks/ucm-config-analyzer/src/ucmexport/__init__.py b/playbooks/ucm-config-analyzer/src/ucmexport/__init__.py new file mode 100644 index 0000000..374ea14 --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/ucmexport/__init__.py @@ -0,0 +1,2 @@ +from .proxy import Proxy +from .objects import * diff --git a/playbooks/ucm-config-analyzer/src/ucmexport/objects/__init__.py b/playbooks/ucm-config-analyzer/src/ucmexport/objects/__init__.py new file mode 100644 index 0000000..dade976 --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/ucmexport/objects/__init__.py @@ -0,0 +1,19 @@ +from .phone import * +from .devicepool import * +from .css import * +from .enduser import * +from .directorynumber import * +from .rdp import * +from .linegroup import * +from .huntlist import * +from .huntpilot import * +from .remotedestination import * +from .phonebuttontemplate import * +from .directedcallpark import * +from .callpickupgroup import * +from .callpark import * +from .routepattern import * +from .translationpattern import * +from .deviceprofile import * +from .location import * +from .locationedge import * diff --git a/playbooks/ucm-config-analyzer/src/ucmexport/objects/base.py b/playbooks/ucm-config-analyzer/src/ucmexport/objects/base.py new file mode 100644 index 0000000..f032d02 --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/ucmexport/objects/base.py @@ -0,0 +1,201 @@ +from tarfile import TarFile +from io import TextIOWrapper +from csv import DictReader +from re import compile +from itertools import chain + +import logging + +from collections import defaultdict +from typing import List, Dict, Set + +__all__ = ['RE_TO_SNAIL', 'to_snail', 'ObjBase', 'CsvBase', 'DNAandPartitionRelated', 'REMOVE_ATTR_FROM_PARENT'] + +log = logging.getLogger(__name__) + +RE_TO_SNAIL = compile(r"[ ./\-()']") + +# remove attributes from dict of parent: has significant performance impact +# .. and does not really save memory +REMOVE_ATTR_FROM_PARENT = False + + +def to_snail(key: str) -> str: + """ + Convert a header ro snail case. For example: "JIM DOE" -> "jim_doe" + :param key: string to convert + :return: snail case version + """ + return RE_TO_SNAIL.sub('_', key).lower().strip('_') + + +CHECK_FOR_NONE = False # Raise an Exception if some CSV has a "None" column +POP_NONE = True # Remove "None" column when importing CSV +CSV_TO_UPPER = True # Convert all CSV Headers to uppercase +WARN_LOWERCASE_HEADER = True # log a warning for CSV files that have lowercase headers + +DNAandPartitionRelated = Dict[str, Set[str]] + + +class ObjMeta(type): + + def __new__(mcs, class_name, *args, **kwargs): + c = super(ObjMeta, mcs).__new__(mcs, class_name, *args, **kwargs) + # class specific mapping from snail case identifiers to attributes + c._snail_to_attribute = dict() + return c + + +class ObjBase(metaclass=ObjMeta): + __slots__ = ['_obj'] + + def __init__(self, o: Dict): + if CHECK_FOR_NONE: + assert next((k for k in o if k is None), '') == '', \ + f'ObjBase.__init__ None key found {", ".join(f"{k}:{v}" for k, v in o.items())}' + if POP_NONE: + # sometimes there seems to be a "None" column at the end which breaks stuff + o.pop(None, None) + self._obj = o + + @property + def dict(self): + return self._obj + + def __getattr__(self, item): + if item == '_obj': + raise AttributeError + attribute = self._snail_to_attribute.get(item) + if attribute is None: + # check if there is any attribute that matches the snail.. + attribute = next((a for a in self._obj if item == to_snail(a)), None) + if attribute is None: + raise AttributeError + self._snail_to_attribute[item] = attribute + try: + r = self._obj[attribute] + except KeyError: + raise AttributeError + + if r == 't': + return True + elif r == 'f': + return False + return r + + def __repr__(self): + return f'{self.__class__.__name__}({self})' + + def __str__(self): + return super(ObjBase, self).__repr__() + + +class CsvBase: + __slots__ = ['_tar', '_objects', '_by_attribute'] + + def __init__(self, tar: str): + self._tar = tar + self._objects = None + self._by_attribute: Dict[str, Dict[str, List[...]]] = dict() + + @property + def list(self) -> List[ObjBase]: + def progress(items): + print(f'reading {self.factory.__name__}s from {csv_file}', end='', flush=True) + i = 0 + for i, e in enumerate(items): + if i % 200 == 0: + print('.', end='', flush=True) + yield e + print(f', got {i + 1} {self.factory.__name__}s') + + if self._objects is None: + # read from CSV. Determine CSV file name from class name + # class names are assumed to be Container + csv_file = self.__class__.__name__.lower() + assert csv_file.endswith('container') + # strip 'container' + csv_file = f'{csv_file[:-9]}.csv' + log.debug(f'{self.__class__.__name__}.list: reading {csv_file} from {self._tar}') + + with TarFile(name=self._tar, mode='r') as tar: + try: + file = TextIOWrapper(tar.extractfile(member=csv_file), encoding='utf-8') + except KeyError: + # file not found + self._objects = [] + else: + if CSV_TO_UPPER: + def upper_first_line(it): + first_line = next(it) + first_line_upper = first_line.upper() + if WARN_LOWERCASE_HEADER and first_line != first_line_upper: + logging.warning(f'found lowercase header in {csv_file}') + return chain([first_line_upper], it) + file = upper_first_line(file) + csv_reader = DictReader(file, delimiter=',', doublequote=True, escapechar=None, quotechar='"', + skipinitialspace=True, strict=True) + self._objects = [self.__class__.factory(o) for o in progress(csv_reader)] + log.debug(f'done reading {csv_file} from {self._tar}: {len(self._objects)} objects read') + return self._objects + + def by_attribute(self, attribute: str) -> Dict[str, List[ObjBase]]: + """ + get list of objects by attribute key + :param attribute: attribute or property name to create the grouping from + :return: dictionary with attribute values as key and list of objects as values + """ + + def attr_key(o: ObjBase) -> str: + """ + get key value by accessing an attribute of an object; but respect snail mapping + :param o: object + :return: key value + """ + return o.__getattr__(attribute) + + def property_key(o: ObjBase) -> str: + """ + get key value by evaluating a property + :param o: object + :return: key value + """ + return prop.fget(o) + + # _by_attribute is a cache of groupings + if (d := self._by_attribute.get(attribute)) is None: + # 1st time this grouping is requested + # attribute can be a name of a property or an attribute + if prop := self.factory.__dict__.get(attribute): + # get keys using fget() of the property object + key = property_key + else: + # get keys by directly accessing the attribute: look at the dict of an object + key = attr_key + d = defaultdict(list) + for o in self.list: + d[key(o)].append(o) + d = dict(d) + self._by_attribute[attribute] = d + return d + + def __getattr__(self, item: str): + """ + Implement properties in the form of by_. + Each function returns a dict of list of objects where the key are the attribute values + :param item: + :return: + """ + if not item.startswith('by_'): + raise AttributeError + attribute = item[3:] + try: + return self.by_attribute(attribute) + except KeyError: + raise AttributeError + + def get(self, item, default=None): + try: + return self[item] + except KeyError: + return default diff --git a/playbooks/ucm-config-analyzer/src/ucmexport/objects/callpark.py b/playbooks/ucm-config-analyzer/src/ucmexport/objects/callpark.py new file mode 100644 index 0000000..3a351af --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/ucmexport/objects/callpark.py @@ -0,0 +1,38 @@ +from .base import * + +from typing import List + +__all__ = ['CallPark', 'CallParkContainer'] + + +class CallPark(ObjBase): + @property + def number(self) -> str: + return self.__getattr__('number') + + @property + def partition(self) -> str: + return self.route_partition + + @property + def number_and_partition(self) -> str: + return f'{self.number}:{self.partition}' + + @property + def description(self) -> str: + return self.__getattr__('description') + + @property + def ucm(self) -> str: + return self.unified_callmanager + + def __str__(self): + return self.number_and_partition + + +class CallParkContainer(CsvBase): + factory = CallPark + + @property + def list(self) -> List[CallPark]: + return super(CallParkContainer, self).list diff --git a/playbooks/ucm-config-analyzer/src/ucmexport/objects/callpickupgroup.py b/playbooks/ucm-config-analyzer/src/ucmexport/objects/callpickupgroup.py new file mode 100644 index 0000000..d0de7db --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/ucmexport/objects/callpickupgroup.py @@ -0,0 +1,43 @@ +from .base import * + +from re import compile +from typing import List + +__all__ = ['CallPickupGroupContainer', 'CallPickupGroup'] + + +class CallPickupGroup(ObjBase): + @property + def number(self) -> str: + return self.cpg_number + + @property + def name(self) -> str: + return self.cpg_name + + @property + def description(self) -> str: + return self.__getattr__('description') + + @property + def partition(self) -> str: + return self.route_partition + + @property + def number_and_partition(self): + return f'{self.number}:{self.partition}' + + def __str__(self): + return self.name + + @property + def associated_cpgs(self) -> List[str]: + return [v for k, v in self.dict.items() if k.startswith('ASSOC') and v] + + +class CallPickupGroupContainer(CsvBase): + factory = CallPickupGroup + + @property + def list(self) -> List[CallPickupGroup]: + return super(CallPickupGroupContainer, self).list diff --git a/playbooks/ucm-config-analyzer/src/ucmexport/objects/css.py b/playbooks/ucm-config-analyzer/src/ucmexport/objects/css.py new file mode 100644 index 0000000..b40e56d --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/ucmexport/objects/css.py @@ -0,0 +1,57 @@ +from .base import * + +from typing import Dict, List + +__all__ = ['Css', 'CssContainer'] + + +class Css(ObjBase): + def __init__(self, o: Dict): + super(Css, self).__init__(o) + # get a list of partitions + d = self.dict + keys_to_delete = [k for k in d if k.startswith('ROUTE PARTITION ')] + partitions = [p for p in (d[k] for k in keys_to_delete) if p] + self._partitions = partitions + for k in keys_to_delete: + d.pop(k) + + def __str__(self): + return self.name + + @property + def name(self) -> str: + return self.dict['NAME'] + + @property + def description(self) -> str: + return self.dict['DESCRIPTION'] + + @property + def partition_usage(self) -> str: + return self.type_of_partition_usage + + @property + def partitions(self) -> List[str]: + return self._partitions + + @property + def partitions_string(self) -> str: + return ':'.join(self.partitions) + + +class CssContainer(CsvBase): + factory = Css + + @property + def list(self) -> List[Css]: + return super(CssContainer, self).list + + def __getitem__(self, item) -> Css: + return self.by_name[item][0] + + def partition_names(self, css_name: str) -> List[str]: + if css_name: + return self[css_name].partitions + else: + return [] diff --git a/playbooks/ucm-config-analyzer/src/ucmexport/objects/devicepool.py b/playbooks/ucm-config-analyzer/src/ucmexport/objects/devicepool.py new file mode 100644 index 0000000..442c78b --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/ucmexport/objects/devicepool.py @@ -0,0 +1,50 @@ +from .base import * + +from typing import Dict, List + +__all__ = ['DevicePool', 'DevicePoolContainer'] + + +class DevicePool(ObjBase): + + def __str__(self): + return self.name + + @property + def name(self) -> str: + return self.device_pool_name + + @property + def region(self) -> str: + return self.dict['Region'] + + @property + def aar_css(self) -> str: + return self.aar_calling_search_space + + @property + def srst_reference(self) -> str: + return self.dict['SRST REFÈRENCE'] + + @property + def geo_location(self) -> str: + return self.dict['GEO LOCATION'] + + @property + def physical_location(self) -> str: + return self.dict['PHYSICAL LOCATION'] + + +class DevicePoolContainer(CsvBase): + factory = DevicePool + + @property + def list(self) -> List[DevicePool]: + return super(DevicePoolContainer, self).list + + @property + def by_name(self) -> Dict[str, List[DevicePool]]: + return self.by_attribute('device_name') + + def __getitem__(self, item)->DevicePool: + return self.by_name[item][0] diff --git a/playbooks/ucm-config-analyzer/src/ucmexport/objects/deviceprofile.py b/playbooks/ucm-config-analyzer/src/ucmexport/objects/deviceprofile.py new file mode 100644 index 0000000..ec7deb8 --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/ucmexport/objects/deviceprofile.py @@ -0,0 +1,72 @@ +from .base import * + +from re import compile +import itertools +from .phone import CommonPhoneAndDeviceProfile, CommonPhoneAndDeviceProfileContainer +from typing import List, Dict, Iterable + +__all__ = ['DeviceProfile', 'DeviceProfileContainer'] + +ATTRIBUTE_PATTERN = compile(r'(.+) (\d+)') + + +class DeviceProfile(CommonPhoneAndDeviceProfile): + + def __str__(self): + return self.device_profile_name + + @property + def device_profile_name(self): + return self.dict['DEVICE PROFILE NAME'] + + +DPDict = Dict[str, List[DeviceProfile]] + + +class DeviceProfileContainer(CommonPhoneAndDeviceProfileContainer): + factory = DeviceProfile + + @property + def by_dp_name(self) -> DPDict: + return self.by_attribute('device_profile_name') + + def __getitem__(self, item) -> DeviceProfile: + return self.by_dp_name[item][0] + + @property + def list(self) -> List[DeviceProfile]: + return super().list + + @property + def by_device_type(self) -> DPDict: + return super().by_device_type + + @property + def by_user_id(self) -> DPDict: + return super().by_user_id + + @property + def by_dn_and_partition(self) -> DPDict: + return super().by_dn_and_partition + + @property + def by_call_pickup_group(self) -> DPDict: + return super().by_call_pickup_group + + @property + def by_login_user_id(self) -> DPDict: + return self.by_attribute('login_user_id') + + def with_uri(self) -> Iterable[DeviceProfile]: + return (p for p in self.list if p.has_uri) + + def related_lines(self) -> DNAandPartitionRelated: + """ + Determine which DN:partitions are related b/c they exist on the same device + :return: dictionary, key is DN:partition, values are set of DN:partition. + Only contains DN:partition keys where there are actually related DN:partition values + """ + return {dnp: others for dnp, phone_set in self.by_dn_and_partition.items() + if (others := set(itertools.chain.from_iterable((line.dn_and_partition + for line in p.lines.values() + ) for p in phone_set)) - {dnp})} diff --git a/playbooks/ucm-config-analyzer/src/ucmexport/objects/directedcallpark.py b/playbooks/ucm-config-analyzer/src/ucmexport/objects/directedcallpark.py new file mode 100644 index 0000000..1518f60 --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/ucmexport/objects/directedcallpark.py @@ -0,0 +1,34 @@ +from .base import * + +from typing import List + +__all__ = ['DirectedCallParkContainer', 'DirectedCallPark'] + + +class DirectedCallPark(ObjBase): + @property + def number(self) -> str: + return self.__getattr__('number') + + @property + def partition(self)->str: + return self.route_partition + + @property + def number_and_partition(self)->str: + return f'{self.number}:{self.partition}' + + @property + def description(self) -> str: + return self.__getattr__('description') + + def __str__(self): + return self.number_and_partition + + +class DirectedCallParkContainer(CsvBase): + factory = DirectedCallPark + + @property + def list(self) -> List[DirectedCallPark]: + return super(DirectedCallParkContainer, self).list diff --git a/playbooks/ucm-config-analyzer/src/ucmexport/objects/directorynumber.py b/playbooks/ucm-config-analyzer/src/ucmexport/objects/directorynumber.py new file mode 100644 index 0000000..3801698 --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/ucmexport/objects/directorynumber.py @@ -0,0 +1,45 @@ +from .base import * + +from typing import Dict, List + +__all__ = ['DirectoryNumber', 'DirectoryNumberContainer'] + + +class DirectoryNumber(ObjBase): + @property + def number(self) -> str: + return self.directory_number + + @property + def partition(self) -> str: + return self.route_partition + + @property + def number_and_partition(self) -> str: + return f'{self.number}:{self.partition}' + + @property + def call_pickup_group(self) -> str: + return self.__getattr__('call_pickup_group') + + def __str__(self): + return f'{self.number_and_partition}' + + +class DirectoryNumberContainer(CsvBase): + factory = DirectoryNumber + + @property + def by_number_partition(self) -> Dict[str, List[DirectoryNumber]]: + return self.by_attribute('number_and_partition') + + @property + def by_call_pickup_group(self) -> Dict[str, List[DirectoryNumber]]: + return self.by_attribute('call_pickup_group') + + @property + def list(self) -> List[DirectoryNumber]: + return super(DirectoryNumberContainer, self).list + + def __getitem__(self, item) -> DirectoryNumber: + return self.by_number_partition[item][0] diff --git a/playbooks/ucm-config-analyzer/src/ucmexport/objects/enduser.py b/playbooks/ucm-config-analyzer/src/ucmexport/objects/enduser.py new file mode 100644 index 0000000..25f4fde --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/ucmexport/objects/enduser.py @@ -0,0 +1,145 @@ +from .base import * +from typing import List, Dict, Optional +import re + +__all__ = ['EndUser', 'EndUserContainer'] + + +class PrimaryExtension: + def __init__(self, primary_extension): + m = re.match(r'(\S+) in (.+)', primary_extension) + if m is None: + self.dn = primary_extension + self.partition = '' + else: + self.dn = m.group(1) + self.partition = m.group(2) + + def __str__(self): + return f'{self.dn}:{self.partition}' + + def __repr__(self): + return f'PrimaryExtension({self})' + + +class DeviceAssociation: + def __init__(self, device_name, default_profile, description, type_association): + self.device_name = device_name + self.default_profile = default_profile + self.description = description + self.type_association = type_association + + def __str__(self): + return self.device_name + + def __repr__(self): + return f'DeviceAssociation({self})' + + +class EndUser(ObjBase): + def __init__(self, o: Dict): + # remove empty columns + o = {k: v for k, v in o.items() if v} + super(EndUser, self).__init__(o) + self._primary_extensions = None + self._device_associations = None + + def __str__(self): + return self.user_id + + @property + def first_name(self) -> str: + return self.dict['FIRST NAME'] + + @property + def last_name(self) -> str: + return self.__getattr__('last_name') + + @property + def user_id(self) -> str: + return self.__getattr__('user_id') + + @property + def phone(self) -> str: + return self.telephone_number + + @property + def building(self) -> str: + return self.__getattr__('building') + + @property + def site(self) -> str: + return self.__getattr__('site') + + @property + def em_profile_name(self) -> Optional[str]: + """ + extension mobility profile if available + """ + return next((da.device_name + for da in self.device_associations + if da.type_association == 'Profile Available'), None) + + @property + def primary_extensions(self) -> Dict[str, PrimaryExtension]: + if self._primary_extensions is None: + i = 0 + primary_extensions = dict() + while True: + i += 1 + try: + pe = self._obj.pop(f'PRIMARY EXTENSION {i}') + tpe = self._obj.pop(f'TYPE PATTERN USAGE {i}') + except KeyError: + break + if not pe: + break + assert primary_extensions.get(tpe) is None + primary_extensions[tpe] = PrimaryExtension(pe) + self._primary_extensions = primary_extensions + return self._primary_extensions + + @property + def primary_extension(self) -> Optional[str]: + return self.primary_extensions.get('Primary') + + @property + def device_associations(self) -> List[DeviceAssociation]: + if self._device_associations is None: + device_associations = [] + i = 0 + while True: + i += 1 + try: + dn = self._obj.pop(f'DEVICE NAME {i}') + dp = self._obj.pop(f'DEFAULT PROFILE {i}') + desc = self._obj.pop(f'DESCRIPTION {i}') + tua = self._obj.pop(f'TYPE USER ASSOCIATION {i}') + except KeyError: + break + if not dn: + break + device_associations.append(DeviceAssociation(dn, dp, desc, tua)) + self._device_associations = device_associations + return self._device_associations + + @property + def cti_controlled(self) -> List[DeviceAssociation]: + return [da + for da in self.device_associations + if da.type_association == 'Cti Control In'] + + +class EndUserContainer(CsvBase): + factory = EndUser + + @property + def list(self) -> List[EndUser]: + return super(EndUserContainer, self).list + + @property + def by_em_profile_name(self) -> Dict[str, List[EndUser]]: + return self.by_attribute('em_profile_name') + + def __getitem__(self, item) -> EndUser: + return self.by_user_id[item][0] diff --git a/playbooks/ucm-config-analyzer/src/ucmexport/objects/huntlist.py b/playbooks/ucm-config-analyzer/src/ucmexport/objects/huntlist.py new file mode 100644 index 0000000..8bad4c6 --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/ucmexport/objects/huntlist.py @@ -0,0 +1,112 @@ +from .base import * +from .linegroup import LineGroupContainer + +from re import compile +from typing import Dict, List, Set +from itertools import chain + +from .phone import CommonPhoneAndDeviceProfile, CommonPhoneAndDeviceProfileContainer + +__all__ = ['HuntList', 'HuntListContainer', 'HuntListMember'] + + +class HuntListMember(ObjBase): + + @property + def selection_order(self) -> int: + return int(self.dict['SELECTION ORDER']) + + @property + def line_group(self) -> str: + return self.dict['LINE GROUP'] + + def __str__(self): + return f'{self.selection_order}:{self.line_group}' + + +class HuntList(ObjBase): + attribute_pattern = compile(r'(.+) (\d+)') + + def __init__(self, o: Dict): + super(HuntList, self).__init__(o) + self._members = None + + @property + def name(self) -> str: + return self.dict['NAME'] + + def __str__(self): + return self.name + + @property + def description(self) -> str: + return self.dict['DESCRIPTION'] + + @property + def enabled(self) -> bool: + return self.route_list_enabled + + @property + def for_vm(self) -> bool: + return self.huntlist_for_vm + + @property + def members(self) -> List[HuntListMember]: + member_index = None + members = [] + keys_to_remove = [] + if self._members is None: + for k, v in self.dict.items(): + if m := self.attribute_pattern.match(k): + if REMOVE_ATTR_FROM_PARENT: + keys_to_remove.append(k) + attribute = m.group(1) + index = int(m.group(2)) + if member_index != index: + members.append({attribute: v}) + member_index = index + else: + members[-1][attribute] = v + self._members = [HuntListMember(m) for m in members if m['SELECTION ORDER']] + d = self.dict + for k in keys_to_remove: + d.pop(k) + return self._members + + def pattern_and_partition_set(self, hunt_list_container: 'HuntListContainer') -> Set[str]: + """ + All pattern:partition values in all line groups of the hunt list + :return: set of pattern:partition strings + """ + line_groups = set(m.line_group for m in self.members) + lg_container = hunt_list_container.line_group_container + r = set(chain.from_iterable( + lg_container[line_group].pattern_and_partition_set() + for line_group in line_groups)) + r1 = set() + for line_group in line_groups: + r1 |= lg_container[line_group].pattern_and_partition_set() + return r + + def phones_or_device_profiles(self, hunt_list_container: 'HuntListContainer', + container: CommonPhoneAndDeviceProfileContainer) -> Set[CommonPhoneAndDeviceProfile]: + """ + All phones on which one of the DNPs is present + """ + members = self.pattern_and_partition_set(hunt_list_container=hunt_list_container) + return set(chain.from_iterable(container.by_dn_and_partition.get(dnp, []) for dnp in members)) + + +class HuntListContainer(CsvBase): + factory = HuntList + + def __init__(self, tar: str, line_group_container: LineGroupContainer): + super(HuntListContainer, self).__init__(tar) + self.line_group_container = line_group_container + + def __getitem__(self, item) -> HuntList: + return self.by_name[item][0] + + @property + def list(self) -> List[HuntList]: + return super(HuntListContainer, self).list diff --git a/playbooks/ucm-config-analyzer/src/ucmexport/objects/huntpilot.py b/playbooks/ucm-config-analyzer/src/ucmexport/objects/huntpilot.py new file mode 100644 index 0000000..fe76c05 --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/ucmexport/objects/huntpilot.py @@ -0,0 +1,144 @@ +from collections import defaultdict +from itertools import chain +from re import compile +from typing import List, Dict, Set + +from .base import * +from .huntlist import HuntListContainer +from .phone import CommonPhoneAndDeviceProfileContainer, CommonPhoneAndDeviceProfile + +__all__ = ['HuntPilot', 'HuntPilotContainer', 'HuntPilotHuntList'] + + +class HuntPilotHuntList(ObjBase): + @property + def hunt_list(self) -> str: + return self.dict['HUNT LIST'] + + @property + def external_number_mask(self) -> str: + return self.dict['EXTERNAL NUMBER MASK'] + + @property + def max_callers(self) -> str: + return self.maximum_number_of_callers_in_queue + + @property + def destination_queue_full(self) -> str: + return self.destination_when_queue_is_full + + @property + def full_css(self) -> str: + return self.full_queue_calling_search_space + + def __str__(self): + return self.hunt_list + + +HUNT_LIST_PATTERN = compile(r'(.+) (\d+)') + + +class HuntPilot(ObjBase): + def __init__(self, o: Dict): + super(HuntPilot, self).__init__(o) + self._hunt_lists = None + + @property + def hunt_pilot(self) -> str: + return self.dict['HUNT PILOT'] + + @property + def partition(self) -> str: + return self.route_partition + + @property + def description(self) -> str: + return self.dict['DESCRIPTION'] + + @property + def pilot_and_partition(self) -> str: + return f'{self.hunt_pilot}:{self.partition}' + + @property + def routethis_pattern(self) -> bool: + return self.dict['ROUTETHIS PATTERN'] == 't' + + def __str__(self): + return self.pilot_and_partition + + @property + def hunt_lists(self) -> List[HuntPilotHuntList]: + if self._hunt_lists is None: + hl_attrs = ((HUNT_LIST_PATTERN.match(k), k, v) for k, v in self.dict.items()) + hl_attrs = ((m.group(1), int(m.group(2)), k, v) for m, k, v in hl_attrs if m) + hunt_lists = defaultdict(dict) + keys_to_delete = [] + for attribute, index, k, v in hl_attrs: + if REMOVE_ATTR_FROM_PARENT: + keys_to_delete.append(k) + hunt_lists[index][attribute] = v + self._hunt_lists = [HuntPilotHuntList(hl) for hl in hunt_lists.values()] + d = self.dict + for k in keys_to_delete: + d.pop(k) + return self._hunt_lists + + def pattern_and_partition_set(self, hunt_pilot_container: 'HuntPilotContainer') -> Set[str]: + """ + All pattern:partition values in all line groups of all hunt list of the pilot + :return: set of pattern:partition strings + """ + hunt_lists = set(m.hunt_list for m in self.hunt_lists) + hunt_list_container = hunt_pilot_container.hunt_list_container + + r = set(chain.from_iterable( + hunt_list_container[hunt_list].pattern_and_partition_set(hunt_list_container=hunt_list_container) + for hunt_list in hunt_lists + if hunt_list)) + return r + + def phones_or_device_profiles(self, hunt_pilot_container: 'HuntPilotContainer', + container: CommonPhoneAndDeviceProfileContainer) -> Set[CommonPhoneAndDeviceProfile]: + """ + All phones or device profiles on which one of the DNPs is present + """ + members = self.pattern_and_partition_set(hunt_pilot_container=hunt_pilot_container) + return set(chain.from_iterable(container.by_dn_and_partition.get(dnp, []) for dnp in members)) + + +class HuntPilotContainer(CsvBase): + factory = HuntPilot + + def __init__(self, tar: str, hunt_list_container: HuntListContainer): + super(HuntPilotContainer, self).__init__(tar) + self.hunt_list_container = hunt_list_container + + def __getitem__(self, item) -> HuntPilot: + """ + Get HuntPilot by hunt pilot name + :param item: hunt pilot name + :return: selected hunt Pilot + """ + return self.by_hunt_pilot[item][0] + + @property + def list(self) -> List[HuntPilot]: + return super(HuntPilotContainer, self).list + + def pattern_and_partition_sets(self) -> Dict[str, Set[str]]: + """ + pattern:partition sets for each huntpilot indexed by hunt pilot name + :return: + """ + r = {hp.hunt_pilot: hp.pattern_and_partition_set(hunt_pilot_container=self) for hp in self.list} + return r + + def pilot_related_patterns_and_partitions(self) -> DNAandPartitionRelated: + result: DNAandPartitionRelated = defaultdict(set) + for dn_and_p_set in self.pattern_and_partition_sets().values(): + for dnp in dn_and_p_set: + others = dn_and_p_set.copy() + others.remove(dnp) + if others: + result[dnp] |= others + return result diff --git a/playbooks/ucm-config-analyzer/src/ucmexport/objects/linegroup.py b/playbooks/ucm-config-analyzer/src/ucmexport/objects/linegroup.py new file mode 100644 index 0000000..7e1659d --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/ucmexport/objects/linegroup.py @@ -0,0 +1,123 @@ +from .base import * + +from re import compile +from collections import defaultdict +from itertools import chain + +from .phone import CommonPhoneAndDeviceProfileContainer, CommonPhoneAndDeviceProfile + +from typing import Dict, List, Set + +__all__ = ['LineGroup', 'LineGroupMember', 'LineGroupContainer'] + + +class LineGroupMember(ObjBase): + @property + def selection_order(self) -> int: + return int(self.line_selection_order) + + @property + def dn_or_pattern(self) -> str: + return self.dict['DN OR PATTERN'] + + @property + def partition(self) -> str: + return self.route_partition + + @property + def pattern_and_partition(self) -> str: + return f'{self.dn_or_pattern}:{self.partition}' + + def __str__(self): + return f'{self.pattern_and_partition}:{self.selection_order}' + + +ATTRIBUTE_PATTERN = compile(r'(.+) (\d+)') + + +class LineGroup(ObjBase): + + def __init__(self, o: Dict): + super(LineGroup, self).__init__(o) + self._members = None + + @property + def name(self) -> str: + return self.dict['NAME'] + + @property + def distribution_algorithm(self) -> str: + return self.type_distribution_algorithm + + def __str__(self): + return self.name + + @property + def members(self) -> List[LineGroupMember]: + if self._members is None: + member_index = None + keys_to_remove = [] + members = [] + for k, v in self.dict.items(): + if m := ATTRIBUTE_PATTERN.match(k): + if REMOVE_ATTR_FROM_PARENT: + keys_to_remove.append(k) + attribute = m.group(1) + index = int(m.group(2)) + if member_index != index: + members.append({attribute: v}) + member_index = index + else: + members[-1][attribute] = v + self._members = [LineGroupMember(m) for m in members if m['DN OR PATTERN']] + d = self.dict + for k in keys_to_remove: + d.pop(k) + return self._members + + def pattern_and_partition_set(self) -> Set[str]: + """ + All pattern:partition values in the line group + :return: set of pattern:partition strings + """ + return set([m.pattern_and_partition for m in self.members]) + + def phones_or_device_profiles(self, + container: CommonPhoneAndDeviceProfileContainer) -> Set[CommonPhoneAndDeviceProfile]: + """ + All phones on which one of the DNPs is present + """ + members = self.pattern_and_partition_set() + return set(chain.from_iterable(container.by_dn_and_partition.get(dnp, []) for dnp in members)) + + +class LineGroupContainer(CsvBase): + factory = LineGroup + + def __init__(self, tar: str): + super(LineGroupContainer, self).__init__(tar) + self._related_patterns_and_partitions = None + + @property + def list(self) -> List[LineGroup]: + return super(LineGroupContainer, self).list + + def __getitem__(self, item) -> LineGroup: + return self.by_name[item][0] + + def related_patterns_and_partitions(self) -> DNAandPartitionRelated: + """ + Identify DN/partiton relations based on being members in the same line group + :return: + """ + if self._related_patterns_and_partitions is None: + result = defaultdict(set) + for line_group in self.list: + pattern_and_partition_set = line_group.pattern_and_partition_set() + for item in pattern_and_partition_set: + others = pattern_and_partition_set.copy() + others.remove(item) + if others: + result[item] |= others + self._related_patterns_and_partitions = result + return self._related_patterns_and_partitions diff --git a/playbooks/ucm-config-analyzer/src/ucmexport/objects/location.py b/playbooks/ucm-config-analyzer/src/ucmexport/objects/location.py new file mode 100644 index 0000000..7cf0cb1 --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/ucmexport/objects/location.py @@ -0,0 +1,62 @@ +from .base import * + +from typing import Dict, List, Set +from dataclasses import dataclass + +__all__ = ['Location', 'LocationContainer'] + + +@dataclass +class AssociatedLocation: + location: str + rsvp_setting: str + + +class Location(ObjBase): + + def __init__(self, o: Dict): + super().__init__(o) + self._ass_loc: List[AssociatedLocation] = None + + def __str__(self): + return self.name + + @property + def name(self) -> str: + return self.dict['NAME'] + + @property + def audio_bandwidth(self) -> int: + return int(self.dict['AUDIO BANDWIDTH']) + + @property + def video_bandwidth(self) -> int: + return int(self.dict['VIDEO BANDWIDTH']) + + @property + def immersive_video_bandwidth(self) -> int: + return int(self.dict['IMMERSIVE VIDEO BANDWIDTH']) + + @property + def associated_locations(self) -> List[AssociatedLocation]: + if self._ass_loc is None: + ass_loc = [] + i = 1 + while True: + location = self.dict.get(f'ASSOCIATED LOCATION {i}') + if location is None: + break + if location: + rsvp_setting = self.dict.get(f'RSVP SETTING {i}') + ass_loc.append(AssociatedLocation(location=location, rsvp_setting=rsvp_setting)) + i += 1 + self._ass_loc = ass_loc + return self._ass_loc + + +class LocationContainer(CsvBase): + factory = Location + + @property + def list(self) -> List[Location]: + return super().list diff --git a/playbooks/ucm-config-analyzer/src/ucmexport/objects/locationedge.py b/playbooks/ucm-config-analyzer/src/ucmexport/objects/locationedge.py new file mode 100644 index 0000000..e0766eb --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/ucmexport/objects/locationedge.py @@ -0,0 +1,43 @@ +from .base import * + +from typing import List + +__all__ = ['LocationEdge', 'LocationEdgeContainer'] + + +class LocationEdge(ObjBase): + + def __str__(self): + return f'{self.location}->{self.neighboring_location}' + + @property + def location(self) -> str: + return self.dict['LOCATION'] + + @property + def neighboring_location(self) -> str: + return self.dict['NEIGHBORING LOCATION'] + + @property + def weight(self) -> int: + return int(self.dict['WEIGHT']) + + @property + def audio_bandwidth(self) -> int: + return int(self.dict['AUDIO BANDWIDTH']) + + @property + def video_bandwidth(self) -> int: + return int(self.dict['VIDEO BANDWIDTH']) + + @property + def immersive_video_bandwidth(self) -> int: + return int(self.dict['IMMERSIVE VIDEO BANDWIDTH']) + + +class LocationEdgeContainer(CsvBase): + factory = LocationEdge + + @property + def list(self) -> List[LocationEdge]: + return super().list diff --git a/playbooks/ucm-config-analyzer/src/ucmexport/objects/phone.py b/playbooks/ucm-config-analyzer/src/ucmexport/objects/phone.py new file mode 100644 index 0000000..cd8b011 --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/ucmexport/objects/phone.py @@ -0,0 +1,479 @@ +from .base import * + +from collections import defaultdict +from re import compile, match +import itertools + +from typing import List, Dict, Iterable, Set + +__all__ = ['Phone', 'PhoneContainer', 'BusyLampField', 'SpeedDial', 'Line', 'Uri', 'PhoneDict', + 'CommonPhoneAndDeviceProfileContainer', 'CommonPhoneAndDeviceProfile'] + + +class Uri(ObjBase): + """ + URI information on a line + """ + + @property + def uri(self) -> str: + return self.on_directory_number + + @property + def route_partition(self) -> str: + return self.route_partition_on_directory_number + + def __str__(self): + return f'{self.uri}:{self.route_partition}' + + +URI_PATTERN = compile(r'URI (\d) (.+)') + + +class Line(ObjBase): + """ + a line on a phone + """ + + def __init__(self, o: Dict): + super(Line, self).__init__(o) + self._uris = None + + def __str__(self): + return f'{self.directory_number}:{self.route_partition}' + + @property + def directory_number(self) -> str: + return self.__getattr__('directory_number') + + @property + def partition(self) -> str: + return self.route_partition + + @property + def dn_and_partition(self) -> str: + return f'{self.directory_number}:{self.partition}' + + @property + def external_phone_number_mask(self) -> str: + return self.__getattr__('external_phone_number_mask') + + @property + def css(self) -> str: + return self.line_css + + @property + def aar_group(self) -> str: + return self.aar_group_line + + @property + def call_pickup_group(self) -> str: + return self.__getattr__('call_pickup_group') + + @property + def uris(self) -> Dict[int, Uri]: + if self._uris is None: + # collect uri information + uris: Dict[int, Dict] = defaultdict(dict) + remove_keys = [] + for k, v in self.dict.items(): + if k.startswith('URI '): + if REMOVE_ATTR_FROM_PARENT: + remove_keys.append(k) + uri_index = k.split(' ')[1] + attribute = k[5 + len(uri_index):] + uris[uri_index][attribute] = v + d = self.dict + for k in remove_keys: + d.pop(k) + self._uris = {int(k): Uri(uri) for k, uri in uris.items() if uri['ON DIRECTORY NUMBER']} + return self._uris + + +SD_PATTERN = compile(r'SPEED DIAL (\w+) (\d+)') + + +class SpeedDial(ObjBase): + @property + def label(self) -> str: + return self.dict['LABEL'] + + @property + def number(self) -> str: + return self.dict['NUMBER'] + + def __str__(self): + return f'{self.label}:{self.number}' + + +BLF_PATTERN = compile(r'BUSY LAMP FIELD (.+) (\d+)') + + +class BusyLampField(ObjBase): + def __init__(self, o: Dict): + super(BusyLampField, self).__init__(o) + dn = self.dict['DIRECTORY NUMBER'] + if dn and (m := match(r'(\d+) in (\D+)', dn)): + # noinspection PyUnboundLocalVariable + self._dn = m.group(1) + self._partition = m.group(2) + else: + self._dn = '' + self._partition = '' + + @property + def label(self) -> str: + return self.__getattr__('label') + + @property + def destination(self) -> str: + return self.__getattr__('destination') + + @property + def directory_number(self) -> str: + return self._dn + + @property + def partition(self) -> str: + return self._partition + + @property + def dn_and_partition(self) -> str: + return f'{self.directory_number}:{self.partition}' + + @property + def call_pickup(self) -> bool: + return self.__getattr__('call_pickup') + + def __str__(self): + return f'{self.label}:{self.destination}:{self.directory_number}:{self.call_pickup}' + + +ATTRIBUTE_PATTERN = compile(r'(.+) (\d+)') + + +class CommonPhoneAndDeviceProfile(ObjBase): + """ + Commonalities of Phone and DeviceProfile + """ + def __init__(self, o: Dict): + super().__init__(o) + self._lines = None + self._speed_dials = None + self._blfs = None + self._user_ids = None + + @property + def lines(self) -> Dict[int, Line]: + if self._lines is None: + line = None + line_index = None + lines = dict() + remove_keys = [] + line_index_len = None + for k, v in self.dict.items(): + k: str + # look for 'Directory Number" and 'Speed Dial' + if (k[0] in 'DS') and \ + ((sd := k.startswith('DIRECTORY N')) or k.startswith('SPEE')): + # end collecting the current line + if line is not None: + # collect the line + lines[int(line_index)] = Line(line) + line = None + # noinspection PyUnboundLocalVariable + if sd and v: + # start a new line only if the directory number is not empty + line = dict() + line_index = k.split(' ')[-1] + # index len including preceeding space + line_index_len = len(line_index) + 1 + elif not sd: + # we are done as soon as we hit the 1st Speed Dial + break + else: + line = None + + if line is not None: + # remove this key from phone + if REMOVE_ATTR_FROM_PARENT: + remove_keys.append(k) + # add an attribute to the line + # attribute is all up to the line index + k = k[:-line_index_len] + line[k] = v + + if line is not None: + # store the line + lines[int(line_index)] = Line(line) + self._lines = lines + d = self.dict + for k in remove_keys: + d.pop(k) + return self._lines + + @property + def speed_dials(self) -> Dict[int, SpeedDial]: + if self._speed_dials is None: + keys_to_delete = [] + speed_dials = defaultdict(dict) + for k, v in self.dict.items(): + if not k.startswith('SPEED '): + continue + if m := SD_PATTERN.match(k): + if REMOVE_ATTR_FROM_PARENT: + keys_to_delete.append(k) + attribute = m.group(1) + index = int(m.group(2)) + speed_dials[index][attribute] = v + self._speed_dials = {k: SpeedDial(v) for k, v in speed_dials.items() if v['NUMBER']} + d = self.dict + for k in keys_to_delete: + d.pop(k) + return self._speed_dials + + @property + def busy_lamp_fields(self) -> Dict[int, BusyLampField]: + if self._blfs is None: + blfs = defaultdict(dict) + keys_to_delete = [] + for k, v in self.dict.items(): + if not k.startswith('BUSY'): + continue + if m := BLF_PATTERN.match(k): + if REMOVE_ATTR_FROM_PARENT: + keys_to_delete.append(k) + attribute = m.group(1) + index = int(m.group(2)) + blfs[index][attribute] = v + self._blfs = {k: BusyLampField(v) for k, v in blfs.items() if + v['DESTINATION'] or v['DIRECTORY NUMBER'] or v['CALL PICKUP']} + d = self.dict + for k in keys_to_delete: + d.pop(k) + return self._blfs + + def __lt__(self, other): + return str(self) < str(other) + + @property + def device_type(self): + return self.dict['DEVICE TYPE'] + + @property + def phone_button_template(self): + return self.__getattr__('phone_button_template') + + @property + def uris(self) -> Iterable[Uri]: + return list(itertools.chain.from_iterable(line.uris.values() for line in self.lines.values())) + + @property + def has_uri(self) -> bool: + if list(self.uris): + return True + else: + return False + + def user_id(self, index: int) -> str: + return self.__getattr__(f'user_id_{index}') + + @property + def user_ids(self) -> List[str]: + if self._user_ids is None: + self._user_ids = [] + i = 1 + + while True: + try: + uid = self.user_id(i) + if not uid: + break + self._user_ids.append(uid) + except AttributeError: + break + i += 1 + return self._user_ids + + @property + def user_set(self) -> Set[str]: + """ + all user ids referenced on the phone or profile: USER ID X + :return: set of user IDs + """ + return set(self.user_ids) + + +class Phone(CommonPhoneAndDeviceProfile): + def __init__(self, o: Dict): + super().__init__(o) + + def __str__(self): + return self.device_name + + @property + def device_name(self): + return self.dict['DEVICE NAME'] + + @property + def device_pool(self): + return self.dict['DEVICE POOL'] + + @property + def css(self): + return self.dict.get('CSS') + + @property + def aar_css(self): + return self.dict.get('AAR CSS') + + @property + def location(self): + return self.dict.get('LOCATION') + + @property + def owner(self) -> str: + return self.owner_user_id + + @property + def user_set(self) -> Set[str]: + """ + all user ids referenced on the phone: owner and USER ID X + :return: set of user IDs + """ + us = super().user_set + if self.owner: + us.add(self.owner) + return us + + +PhoneAndDevicePoolDict = Dict[str, List[CommonPhoneAndDeviceProfile]] + + +class CommonPhoneAndDeviceProfileContainer(CsvBase): + """ + Commonalities of Phone and Device Profile Container + """ + def __init__(self, tar: str): + super().__init__(tar) + self._line_related_patterns_and_partitions = None + self._by_user_id = None + self._by_dn_and_partition = None + self._by_call_pickup_group = None + + @property + def by_device_type(self) -> PhoneAndDevicePoolDict: + return self.by_attribute('device_type') + + @property + def by_user_id(self) -> PhoneAndDevicePoolDict: + """ + indexed by user ids + :return: + """ + if self._by_user_id is None: + d: PhoneAndDevicePoolDict = defaultdict(list) + for phone in self.list: + for user_id in phone.user_set: + d[user_id].append(phone) + self._by_user_id = d + return self._by_user_id + + @property + def by_dn_and_partition(self) -> PhoneAndDevicePoolDict: + """ + Get sets of phones indexed by dn:partition provisioned on these phones + :return: dict of sets of phones indexed by dn:partition provisioned on these phones + """ + if self._by_dn_and_partition is None: + r = defaultdict(set) + for phone in self.list: + for line in phone.lines.values(): + r[line.dn_and_partition].add(phone) + self._by_dn_and_partition = dict(r) + return self._by_dn_and_partition + + @property + def by_call_pickup_group(self) -> PhoneAndDevicePoolDict: + """ + Get sets of phones indexed by call pickup groups provisioned on lines of these phones + :return: dict of sets of phones indexed by call pickup group name + """ + if self._by_call_pickup_group is None: + result = defaultdict(set) + for phone in self.list: + # DPGs on all lines; we skip lines w/ empty DPG + cpgs = set(cpg + for line in phone.lines.values() + if (cpg := line.call_pickup_group)) + for cpg in cpgs: + result[cpg].add(phone) + self._by_call_pickup_group = dict(result) + return self._by_call_pickup_group + + +PhoneDict = Dict[str, List[Phone]] + + +class PhoneContainer(CommonPhoneAndDeviceProfileContainer): + factory = Phone + + def __init__(self, tar: str): + super(PhoneContainer, self).__init__(tar) + + @property + def by_device_name(self) -> PhoneDict: + return self.by_attribute('device_name') + + def __getitem__(self, item) -> Phone: + return self.by_device_name[item][0] + + @property + def list(self) -> List[Phone]: + return super(PhoneContainer, self).list + + @property + def by_device_pool(self) -> PhoneDict: + return self.by_attribute('device_pool') + + @property + def by_device_type(self) -> PhoneDict: + return super().by_device_type + + @property + def by_owner(self) -> PhoneDict: + return self.by_attribute('owner') + + @property + def by_user_id(self) -> PhoneDict: + return super().by_user_id + + def with_owner(self) -> Iterable[Phone]: + p: Phone + return (p for p in self.list if p.owner) + + def without_owner(self) -> Iterable[Phone]: + p: Phone + return (p for p in self.list if not p.owner) + + def with_uri(self) -> Iterable[Phone]: + return (p for p in self.list if p.has_uri) + + @property + def by_dn_and_partition(self) -> PhoneDict: + return super().by_dn_and_partition + + @property + def by_call_pickup_group(self) -> PhoneDict: + return super().by_call_pickup_group + + def related_lines(self) -> DNAandPartitionRelated: + """ + Determine which DN:partitions are related b/c they exist on the same device + :return: dictionary, key is DN:partition, values are set of DN:partition. + Only contains DN:partition keys where there are actually related DN:partition values + """ + return {dnp: others for dnp, phone_set in self.by_dn_and_partition.items() + if (others := set(itertools.chain.from_iterable((line.dn_and_partition + for line in p.lines.values() + ) for p in phone_set)) - {dnp})} diff --git a/playbooks/ucm-config-analyzer/src/ucmexport/objects/phonebuttontemplate.py b/playbooks/ucm-config-analyzer/src/ucmexport/objects/phonebuttontemplate.py new file mode 100644 index 0000000..1de54c6 --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/ucmexport/objects/phonebuttontemplate.py @@ -0,0 +1,89 @@ +from .base import * + +from re import compile + +from typing import List, Dict + +__all__ = ['PhoneButtonTemplate', 'PhoneButtonTemplateContainer', 'PhoneButton', 'BLF_FEATURE_TYPES'] + +BLF_FEATURE_TYPES = ['Speed Dial BLF', 'Call Park BLF'] + + +class PhoneButton(ObjBase): + @property + def feature_type(self) -> str: + return self.type_of_feature + + @property + def label(self) -> str: + return self.dict['LABEL'] + + @property + def parameter(self) -> str: + return self.dict['PARAMETER'] + + @property + def fixed_feature(self) -> bool: + return self.isfixedfeature + + def __str__(self): + return self.label + + +BUTTON_ATTRIBUTES = compile(r'(.+) (\d+)') + + +class PhoneButtonTemplate(ObjBase): + def __init__(self, o: Dict): + super(PhoneButtonTemplate, self).__init__(o) + self._buttons = None + + @property + def name(self) -> str: + return self.dict['NAME'] + + @property + def number_of_buttons(self) -> int: + return int(self.dict['NUMBER OF BUTTONS']) + + @property + def model_type(self) -> str: + return self.type_of_model + + @property + def protocol_type(self) -> str: + return self.type_of_protocol + + def __str__(self): + return self.name + + @property + def buttons(self) -> List[PhoneButton]: + if self._buttons is None: + items = iter(self.dict.items()) + buttons = [] + for k, v in items: + k: str + if BUTTON_ATTRIBUTES.match(k) is None: + continue + if not v or v == 'None': + break + button = dict() + button[k[:k.rfind(' ')]] = v + for _ in range(3): + k, v = next(items) + button[k[:k.rfind(' ')]] = v + buttons.append(PhoneButton(button)) + self._buttons = buttons + return self._buttons + + +class PhoneButtonTemplateContainer(CsvBase): + factory = PhoneButtonTemplate + + @property + def list(self) -> List[PhoneButtonTemplate]: + return super(PhoneButtonTemplateContainer, self).list + + def __getitem__(self, item) -> PhoneButtonTemplate: + return self.by_name[item][0] diff --git a/playbooks/ucm-config-analyzer/src/ucmexport/objects/rdp.py b/playbooks/ucm-config-analyzer/src/ucmexport/objects/rdp.py new file mode 100644 index 0000000..3deeb58 --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/ucmexport/objects/rdp.py @@ -0,0 +1,37 @@ +from .base import * + +from typing import List + +__all__ = ['Rdp', 'RdpContainer'] + + +class Rdp(ObjBase): + @property + def name(self) -> str: + return self.remote_destination_profile_name + + @property + def description(self) -> str: + return self.dict['DESCRIPTION'] + + @property + def mobility_user(self) -> str: + return self.mobility_user_id + + @property + def device_pool(self) -> str: + return self.dict['DEVICE POOL'] + + @property + def css(self) -> str: + return self.dict['CSS'] + + pass + + +class RdpContainer(CsvBase): + factory = Rdp + + @property + def list(self) -> List[Rdp]: + return super(RdpContainer, self).list diff --git a/playbooks/ucm-config-analyzer/src/ucmexport/objects/remotedestination.py b/playbooks/ucm-config-analyzer/src/ucmexport/objects/remotedestination.py new file mode 100644 index 0000000..3d8b1c4 --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/ucmexport/objects/remotedestination.py @@ -0,0 +1,172 @@ +from .base import * + +from re import compile +from collections import defaultdict + +from typing import Dict, List, Tuple + +__all__ = ['RemoteDestination', 'RemoteDestinationContainer', 'Destination', 'Schedule'] + + +class Schedule(ObjBase): + @property + def day_of_week(self) -> str: + return self.dict['DAY OF WEEK'] + + @property + def start_time(self) -> str: + return self.dict['START TIME'] + + @property + def end_time(self) -> str: + return self.dict['END TIME'] + + def __str__(self): + return f'{self.day_of_week}:{self.start_time}:{self.end_time}' + + +ATTRIBUTE_PATTERN = compile(r'(.+) (\d+)') + + +class Destination(ObjBase): + def __init__(self, o: dict): + # sometimes route_partition seems to be a list + if isinstance(rp := o['ROUTE PARTITION'], list): + o['ROUTE PARTITION'] = rp[0] + + super(Destination, self).__init__(o) + self._schedules = None + + @property + def destination(self) -> str: + return self.dict['DESTINATION'] + + @property + def associated_line_number(self) -> str: + return self.dict['ASSOCIATED LINE NUMBER'] + + @property + def route_partition(self) -> str: + return self.dict['ROUTE PARTITION'] + + @property + def line_number_and_partition(self) -> str: + return f'{self.associated_line_number}:{self.route_partition}' + + def __str__(self): + return f'{self.destination}:{self.line_number_and_partition}' + + @property + def schedules(self) -> List[Schedule]: + if self._schedules is None: + schedules = [] + keys_do_delete = [] + schedule_index = None + schedule = None + for k, v in self.dict.items(): + if m := ATTRIBUTE_PATTERN.match(k): + if REMOVE_ATTR_FROM_PARENT: + keys_do_delete.append(k) + index = m.group(2) + attribute = m.group(1) + if index != schedule_index: + schedule_index = index + if schedule != None and schedule['DAY OF WEEK']: + schedules.append(Schedule(schedule)) + schedule = dict() + schedule[attribute] = v + if schedule and schedule['DAY OF WEEK']: + schedules.append(Schedule(schedule)) + d = self.dict + for k in keys_do_delete: + d.pop(k) + self._schedules = schedules + return self._schedules + + +SCHEDULE_ATTRIBUTES = set(['DAY OF WEEK', 'START TIME', 'END TIME']) + + +class RemoteDestination(ObjBase): + def __init__(self, o: Dict): + # remotedestination.csv seems to have entries with issues. + # Looks like some rows habe NULL in the "TIME ZONE" + # column. In these rows all following columns need to be shifted left + if o['TIME ZONE'] == 'NULL': + # generator for all keys and values of o + i = ((k, v) for k, v in o.items()) + # consume all keys until TIME_ZONE + next(k for k, _ in i if k == 'TIME ZONE') + prev_key = 'TIME ZONE' + # iterate through remaining keys and values + for k, v in i: + o[prev_key] = v + prev_key = k + # delete last key (None) + o.pop(prev_key) + super(RemoteDestination, self).__init__(o) + self._destinations = None + + @property + def name(self) -> str: + return self.dict['NAME'] + + @property + def remote_destinaton_profile(self) -> str: + return self.dict['REMOTE DESTINATION PROFILE'] + + def __str__(self): + return self.name + + @property + def destinations(self) -> List[Destination]: + if self._destinations is None: + destinations = [] + destination = None + keys_to_remove = [] + for k, v in self.dict.items(): + if k.startswith('DESTINATION '): + # first column of a destination + if destination is not None: + destinations.append(Destination(destination)) + destination = dict() + if destination is not None: + # this is an attribute of a destination + m = ATTRIBUTE_PATTERN.match(k) + attribute = m.group(1) + if attribute in SCHEDULE_ATTRIBUTES: + # keep the indices; will be consumed later + attribute = k + destination[attribute] = v + if REMOVE_ATTR_FROM_PARENT: + keys_to_remove.append(k) + if destination is not None: + destinations.append(Destination(destination)) + d = self.dict + for k in keys_to_remove: + d.pop(k) + self._destinations = destinations + return self._destinations + + +class RemoteDestinationContainer(CsvBase): + factory = RemoteDestination + + def __init__(self, tar: str): + super(RemoteDestinationContainer, self).__init__(tar) + self._by_line_number_and_partition = None + + @property + def list(self) -> List[RemoteDestination]: + return super(RemoteDestinationContainer, self).list + + @property + def by_line_number_and_partition(self) -> Dict[str, List[Tuple[RemoteDestination, Destination]]]: + if self._by_line_number_and_partition is None: + result = defaultdict(list) + for remote_destination in self.list: + for destination in remote_destination.destinations: + result[destination.line_number_and_partition].append((remote_destination, destination)) + self._by_line_number_and_partition = result + + return self._by_line_number_and_partition diff --git a/playbooks/ucm-config-analyzer/src/ucmexport/objects/routepattern.py b/playbooks/ucm-config-analyzer/src/ucmexport/objects/routepattern.py new file mode 100644 index 0000000..054c6d3 --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/ucmexport/objects/routepattern.py @@ -0,0 +1,41 @@ +from .base import * + +__all__ = ['RoutePattern', 'RoutePatternContainer'] + +from typing import List + + +class RoutePattern(ObjBase): + @property + def pattern(self) -> str: + return self.route_pattern + + @property + def partition(self) -> str: + return self.route_partition + + @property + def pattern_and_partition(self) -> str: + return f'{self.pattern}:{self.partition}' + + def __str__(self): + return self.pattern_and_partition + + +class RoutePatternContainer(CsvBase): + factory = RoutePattern + + def __init__(self, tar: str): + super(RoutePatternContainer, self).__init__(tar) + self._list = None + + @property + def list(self) -> List[RoutePattern]: + if self._list is None: + r = super(RoutePatternContainer, self).list + r.sort(key=lambda v: v.pattern_and_partition) + self._list = r + return self._list + + def __getitem__(self, item) -> RoutePattern: + return self.by_pattern_and_partition[item] diff --git a/playbooks/ucm-config-analyzer/src/ucmexport/objects/translationpattern.py b/playbooks/ucm-config-analyzer/src/ucmexport/objects/translationpattern.py new file mode 100644 index 0000000..7d979ec --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/ucmexport/objects/translationpattern.py @@ -0,0 +1,115 @@ +import re +from .base import * + +from typing import List + +__all__ = ['TranslationPattern', 'TranslationPatternContainer', 'TP_UNLIMITED'] + +TP_UNLIMITED = 100 + + +class TranslationPattern(ObjBase): + + @property + def pattern(self) -> str: + return self.translation_pattern + + @property + def partition(self) -> str: + return self.route_partition + + @property + def pattern_and_partition(self) -> str: + return f'{self.pattern}:{self.partition}' + + @property + def urgent(self) -> bool: + return self.urgent_priority + + @property + def css(self) -> str: + return self.calling_search_space + + @property + def use_originators_calling_search_space(self) -> bool: + return self.use_originator_s_calling_search_space + + @property + def route_option(self) -> str: + return self.__getattr__('route_option') + + @property + def block(self): + """ + Blocking Pattern + :return: + """ + return self.route_option + + @property + def block_pattern_option(self) -> bool: + """ + Block option for blocking pattern + :return: + """ + return self.block_this_pattern_option + + @property + def discard_digits(self) -> str: + """ + Digit Discard Instruction + :return: + """ + return self.__getattr__('discard_digits') + + @property + def called_party_mask(self) -> str: + return self.called_party_transform_mask + + @property + def called_party_prefix_digits(self) -> str: + return self.called_party_prefix_digits__outgoing_calls + + @property + def route_next_hop_by_calling_party_number(self) -> bool: + return self.__getattr__('route_next_hop_by_calling_party_number') + + def __str__(self): + return self.pattern_and_partition + + @property + def length(self) -> int: + """ + Determine length of digit string matched by TP + :return: + """ + p = self.pattern + if p.endswith('!'): + return TP_UNLIMITED + # remove separator + p = p.replace('.', '') + # replace [..] with a single X + p = re.sub(r'\[[0-9\-]+\]', 'X', p) + return len(p) + + @property + def translated_length(self) -> int: + """ + Length of pattern after translation + :return: + """ + if self.called_party_mask: + return len(self.called_party_mask) + if (pattern_length := self.length) == TP_UNLIMITED: + return TP_UNLIMITED + if self.discard_digits == 'PreDot': + pattern_length -= self.pattern.index('.') + return pattern_length + len(self.called_party_prefix_digits) + + +class TranslationPatternContainer(CsvBase): + factory = TranslationPattern + + @property + def list(self) -> List[TranslationPattern]: + return super(TranslationPatternContainer, self).list diff --git a/playbooks/ucm-config-analyzer/src/ucmexport/proxy/__init__.py b/playbooks/ucm-config-analyzer/src/ucmexport/proxy/__init__.py new file mode 100644 index 0000000..d321361 --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/ucmexport/proxy/__init__.py @@ -0,0 +1,59 @@ +from ucmexport.objects import * + +from itertools import chain +from typing import List, Dict, Set + + +class Proxy: + def __init__(self, tar: str): + self.css = CssContainer(tar) + self.device_pools = DevicePoolContainer(tar) + self.directed_call_park = DirectedCallParkContainer(tar) + self.directory_number = DirectoryNumberContainer(tar) + self.end_user = EndUserContainer(tar) + self.call_park = CallParkContainer(tar) + self.call_pickup_group = CallPickupGroupContainer(tar) + + self.line_group = LineGroupContainer(tar) + self.hunt_list = HuntListContainer(tar, self.line_group) + self.hunt_pilot = HuntPilotContainer(tar, self.hunt_list) + + self.phone_button_template = PhoneButtonTemplateContainer(tar) + self.phones = PhoneContainer(tar) + self.rdp = RdpContainer(tar) + self.remote_destination = RemoteDestinationContainer(tar) + self.route_pattern = RoutePatternContainer(tar) + self.translation_pattern = TranslationPatternContainer(tar) + self.device_profile = DeviceProfileContainer(tar) + self.location = LocationContainer(tar) + self.location_edge = LocationEdgeContainer(tar) + + self._dn_partition_by_enduser = None + + def dn_partition_by_enduser(self) -> Dict[EndUser, Set[str]]: + """ + get sets of dn:partitions by enduser by looking lines on phones owned by each user + :return: + """ + if self._dn_partition_by_enduser is None: + self._dn_partition_by_enduser = \ + {user: dnp_set for user in self.end_user.list + if (dnp_set := set(chain.from_iterable((l.dn_and_partition + for l in p.lines.values() + ) + for p in self.phones.by_owner.get(user.user_id, []))))} + return self._dn_partition_by_enduser + + def phones_with_blf(self) -> Dict[str, List[Phone]]: + phone_button_templates = self.phone_button_template.list + + def is_blf(pb: PhoneButton): + return pb.feature_type in BLF_FEATURE_TYPES + + pbt_with_blf = [pbt for pbt in phone_button_templates if any(map(is_blf, pbt.buttons))] + phones_by_button_template = self.phones.by_phone_button_template + + phones_w_blf = {pbt: phones for pbt in pbt_with_blf if + (phones := phones_by_button_template.get(pbt.name))} + phones_w_blf: Dict[PhoneButtonTemplate, List[Phone]] + return phones_w_blf diff --git a/playbooks/ucm-config-analyzer/src/user_dependency_graph/__init__.py b/playbooks/ucm-config-analyzer/src/user_dependency_graph/__init__.py new file mode 100644 index 0000000..c9f890c --- /dev/null +++ b/playbooks/ucm-config-analyzer/src/user_dependency_graph/__init__.py @@ -0,0 +1,612 @@ +import os +import webbrowser +from itertools import chain +from collections import defaultdict +import logging +import networkx as nx +import plotly.graph_objects as go +from pydantic import BaseModel, model_validator + +from ucmexport import * + +from typing import Union, Set, Generator, Dict, List, Optional + +log = logging.getLogger(__name__) + +JS_TEMPLATE_3D = """ + + + + + + + + +
+ + + +""" + + +class GDataNode(BaseModel): + id: str + name: str + group: Optional[str] = None + + + + +class GDataLink(BaseModel): + source: str + target: str + + @model_validator(mode='after') + def val_node(self): + source_type = self.source.split()[0] + target_type = self.target.split()[0] + # user is always a target + if source_type == 'user': + self.source, self.target = self.target, self.source + # dn is always a source + if target_type == 'dn': + self.source, self.target = self.target, self.source + return self + + +class GData(BaseModel): + nodes: list[GDataNode] + links: list[GDataLink] + + +class UserGraph(nx.Graph): + + @staticmethod + def user_node(user: Union[str, EndUser]): + if isinstance(user, str): + return f'user {user}' + return f'user {user.user_id}' + + def user_node_by_id(self, uid: str, proxy: Proxy): + user = proxy.end_user.get(uid) + if user is None: + return user + return self.user_node(user) + + @staticmethod + def phone_node(phone: Phone): + return f'phone {phone.device_name}' + + @staticmethod + def hunt_pilot_node(hp: HuntPilot): + return f'huntpilot {hp.pilot_and_partition}' + + @staticmethod + def cpg_node(cpg: Union[str, CallPickupGroup]): + if isinstance(cpg, str): + return f'CPG {cpg}' + return f'CPG {cpg.name}' + + @staticmethod + def ignore_uid(proxy: Proxy) -> set[str]: + # we want to ignore users that own an excessive number of phones + phones_per_user: dict[str, int] = defaultdict(int) + for phone in proxy.phones.list: + for user in phone.user_set: + phones_per_user[user] += 1 + ignore_uid = {uid for uid, count in phones_per_user.items() if count > 20} + print(f'ignoring {len(ignore_uid)} users with more than 20 phones: {", ".join(sorted(ignore_uid))}') + return ignore_uid + + def connected_user_nodes(self, node: str) -> Set[str]: + """ + Determine all user nodes connected to given node + :param node: node to search connected user nodes for + :return: set of connected user nodes + """ + try: + connected = nx.algorithms.node_connected_component(self, node) + except KeyError: + return set() + return set(c for c in connected if c.startswith('user')) + + def connected_user_ids(self, user_id: str) -> Set[str]: + """ + Determine user ids of users already connected to a user with a given id + :param user_id: user id + :return: set of ids of connected users + """ + connected = self.connected_user_nodes(f'user {user_id}') + return set(c[5:] for c in connected) + + def add_path(self, *nodes: str): + p_node = None + for node in nodes: + self.add_node(node) + if p_node: + self.add_edge(p_node, node) + p_node = node + + def related_users_hunt_pilot(self, proxy: Proxy) -> int: + """ + Add user relations to graph: users related via hunt pilot + :param proxy: + :return: + """ + print('Related users based on hunt pilots...') + users_added = 0 + ignore_uid = self.ignore_uid(proxy) + for hunt_pilot in proxy.hunt_pilot.list: + hp_node = self.hunt_pilot_node(hunt_pilot) + # all patterns:partitions on all line groups + dnps = hunt_pilot.pattern_and_partition_set(proxy.hunt_pilot) + # now we want to get the set of users on all phones with these dns + # start with all phones that have any of these dns + phones = set(chain.from_iterable(proxy.phones.by_dn_and_partition.get(dnp, []) for dnp in dnps)) + # finally get all users associated with these phones + user_ids = set(chain.from_iterable(phone.user_set for phone in phones)) + user_ids -= ignore_uid + # .. and we only want to look at users which actually exist as end users + users = [user for user_id in user_ids + if (user := proxy.end_user.get(user_id))] + if len(users) < 2: + continue + # add node for hunt pilot to graph + self.add_node(hp_node) + # now for each user add a node and create a link between hunt pilot and user + for user in users: + un = self.user_node(user) + self.add_node(un) + self.add_edge(hp_node, un) + users_added += (len(users) - 1) + # for + # for + return users_added + + def related_users_shared_phones(self, proxy: Proxy, only_new_relations=False) -> int: + """ + Find users that are related based on shared phones + :param proxy: + :param only_new_relations: + :return: number of related users + """ + users_added = 0 + print('Related users based on shared phones...') + ignore_uid = self.ignore_uid(proxy) + for phone in proxy.phones.list: + # only consider actually existing end users + user_node_set = {self.user_node(user) + for uid in phone.user_set + if (uid not in ignore_uid) and (user := proxy.end_user.get(uid))} + if len(user_node_set) < 2: + continue + # no need to add if all users are already connected + if only_new_relations: + not_connected_per_user = [user_node_set - {user_node} - self.connected_user_nodes(user_node) + for user_node in user_node_set] + if not any(not_connected_per_user): + continue + + # add a node for the phone + pn = self.phone_node(phone) + # self.add_node(pn) + # for each user add a node and an edge between phone and user + for user_node in user_node_set: + users_added += 1 + self.add_node(user_node) + self.add_edge(user_node, pn) + return users_added + + def related_users_shared_lines(self, proxy: Proxy, + only_first_phone=True, + only_first_dnp=True, + only_new_relations=False) -> int: + print('Related users based on shared lines...') + ignore_uid = self.ignore_uid(proxy) + + # get all users and the user's DNs from the users phones + phones_by_user_id = proxy.phones.by_user_id + dnps_by_user_id = {user_id: dnps + for user_id, phones in phones_by_user_id.items() + if ((user_id not in ignore_uid) and + (dnps := chain.from_iterable((line.dn_and_partition + for line in phone.lines.values()) + for phone in phones)))} + # for each user id and DN get phones with different users + phones_by_dn_and_partition = proxy.phones.by_dn_and_partition + phones_other_users_by_user_and_dnp = {user_id: phones_other_users_by_dn + for user_id, dnps in dnps_by_user_id.items() + if ((user_id not in ignore_uid) and + (phones_other_users_by_dn := + {dnp: phones + for dnp in dnps + if (phones := [phone + for phone in phones_by_dn_and_partition[dnp] + if (phone.user_set - {user_id} - ignore_uid)])}))} + + users_added = 0 + for user_id, phones_other_user_by_dnp in phones_other_users_by_user_and_dnp.items(): + other_users = set(chain.from_iterable(chain.from_iterable(phone.user_set + for phone in phones) + for phones in phones_other_user_by_dnp.values())) + other_users -= {user_id} + other_users -= ignore_uid + # we don't need to look at users that are already connected + if only_new_relations: + connected_users = self.connected_user_ids(user_id) + if connected_users: + other_users -= connected_users + other_users = sorted(other_users) + # for each other user (greater than the current user) and dnp get a list of DNs with list of phones + # this user is on + # we ignore phones the current user is one b/c that is the simple case of a shared phone which is + # handled separately + phones_by_other_user_and_dnp = {other_user_id: {dnp: phones_other_user + for dnp, phones in phones_other_user_by_dnp.items() + if (phones_other_user := + [phone + for phone in phones + if other_user_id in phone.user_set + and user_id not in phone.user_set])} + for other_user_id in other_users + if other_user_id > user_id} + for other_user_id in sorted(phones_by_other_user_and_dnp): + phones_by_dnp = phones_by_other_user_and_dnp[other_user_id] + for dnp in sorted(phones_by_dnp): + phones = phones_by_dnp[dnp] + own_phone_with_dn = next(phone + for phone in sorted(inner_phone + for inner_phone in + proxy.phones.by_dn_and_partition[dnp] + if user_id in inner_phone.user_set)) + for phone in phones: + # create path + # user_id-first_owned_phone_with_dnp-dnp-phone-other_user + self.add_path(self.user_node_by_id(user_id, proxy), + self.phone_node(own_phone_with_dn), + f'dn {dnp}', + self.phone_node(phone), + self.user_node_by_id(other_user_id, proxy)) + users_added += 1 + if only_first_phone: + break + # for phone .. + if only_first_dnp: + break + # for dnp ... + # for other_user_id ... + # for user_id .. + return users_added + + def not_used_related_users_shared_lines(self, proxy: Proxy) -> int: + users_added = 0 + for phone in proxy.phones.list: + users = phone.user_set + dnps = [line.dn_and_partition for line in phone.lines.values()] + # for each DN get the list of phones this DN is provisioned on and look whether any phone has a different + # user + for dnp in dnps: + other_phones = [p + for p in proxy.phones.by_dn_and_partition[dnp] + if p != phone] + if not other_phones: + # no need to look further if this is the only phone + continue + # this is the node for the dn (in case we need it) + dn_node = f'dn {dnp}' + for other_phone in other_phones: + # only add if the other phone has at least one different users + other_users = other_phone.user_set - users + if not other_users: + continue + for user in users: + un = self.user_node_by_id(uid=user, proxy=proxy) + if not un: + # this user does not exist? + continue + for other_user in other_users: + oun = self.user_node_by_id(uid=other_user, proxy=proxy) + if not oun: + continue + users_added += 1 + # nodes for users + self.add_node(un) + self.add_node(oun) + + # node for DN + self.add_node(dn_node) + + # phone and edges for phone + # user-phone-dn + pn = self.phone_node(phone) + self.add_node(pn) + self.add_edge(un, pn) + self.add_edge(pn, dn_node) + + # phone and edges for other phone + # user-phone-dn + opn = self.phone_node(other_phone) + self.add_node(opn) + self.add_edge(oun, opn) + self.add_edge(opn, dn_node) + # for + # for + # for + # for + # for + return users_added + + def related_users_blf(self, proxy: Proxy, only_new_relations=False) -> int: + """ + Identify users related via BLF monitoring (one user monitors DNs on another user's phone) + :param proxy: CSV proxy + :param only_new_relations: only a add user relations for users not already in the graph as related + :return: number of users added to the graph + """ + print('related users based on BLF...') + users_added = 0 + phones_by_user_id = proxy.phones.by_user_id + ignore_uuid = self.ignore_uid(proxy) + for user_id in phones_by_user_id: + if user_id in ignore_uuid: + continue + if proxy.end_user.get(user_id) is None: + log.warning(f'user {user_id} does not exist') + continue + for phone in phones_by_user_id[user_id]: + for blf_dnp in (blf.dn_and_partition + for blf in phone.busy_lamp_fields.values()): + # on which phones does this dnp exist... + monitored_phones = proxy.phones.by_dn_and_partition.get(blf_dnp, []) + for monitored_phone in monitored_phones: + monitored_user_ids = monitored_phone.user_set + for monitored_user_id in monitored_user_ids: + if monitored_user_id in ignore_uuid: + continue + if monitored_user_id == user_id: + continue + if (monitored_user := proxy.end_user.get(monitored_user_id)) is None: + continue + # create nodes and edges + # user-phone-blfdnp-dn-monitored phone-monitored user + un = self.user_node_by_id(uid=user_id, proxy=proxy) + mun = self.user_node(monitored_user) + if only_new_relations: + connected = self.connected_user_nodes(un) + if mun in connected: + continue + pn = self.phone_node(phone) + self.add_node(un) + self.add_node(pn) + self.add_edge(un, pn) + blfn = f'blf {blf_dnp}' + self.add_node(blfn) + self.add_edge(pn, blfn) + dn_node = f'dn {blf_dnp}' + self.add_node(dn_node) + self.add_edge(blfn, dn_node) + mpn = self.phone_node(monitored_phone) + self.add_node(mpn) + self.add_edge(dn_node, mpn) + + self.add_node(mun) + self.add_edge(mpn, mun) + users_added += 1 + # for + # for + # for + # for + # for + return users_added + + def related_users_cpg(self, proxy: Proxy, only_new_relations=False) -> int: + """ + Identify users related via call pickup groups on lines of any of the users' phones + :param proxy: CSV proxy + :param only_new_relations: only a add user relations for users not already in the graph as related + :return: number of users added to the graph + """ + print('related users based on call pickup groups...') + by_cpg = proxy.phones.by_call_pickup_group + users_added = set() + related_users_count = 0 + ignore_uid = self.ignore_uid(proxy) + for cpg_name, phones in by_cpg.items(): + # users related via this cpg is the union of all users of all phones related to this cpg + user_set = sorted(set(chain.from_iterable(phone.user_set + for phone in phones))) + if len(user_set) < 2: + # nothing to do if not at least two users are involved + continue + # for each user create edge: user-phone-cpg + cpg_node = self.cpg_node(cpg_name) + for phone in phones: + phone_node = self.phone_node(phone) + for user in phone.user_set: + if user in ignore_uid: + continue + if user in users_added: + continue + self.add_path(self.user_node(user), phone_node, cpg_node) + related_users_count += 1 + # for + # for + # for + return related_users_count + + def related_user_sets(self) -> Generator[Set[str], None, None]: + """ + Generator yielding sets of related user nodes + :return: + """ + for connected in nx.connected_components(self): + yield set(node for node in connected if node.startswith('user ')) + + def related_users_by_len(self) -> Dict[int, List[Set[str]]]: + """ + Cluster sets of related user nodes by size of the sets + :return: + """ + r = defaultdict(list) + for related in self.related_user_sets(): + r[len(related)].append(related) + return r + + def simplify(self): + """ + Try to simplify the graph + :return: None + """ + log.debug(f'{len(self.nodes)} nodes before cleanup') + nodes = list(self.nodes()) + removed_nodes = set() + for node in nodes: + if node in removed_nodes: + # already gone + continue + connected = nx.node_connected_component(self, node) + connected_users = [c for c in connected if c.startswith('user ')] + if len(connected_users) < 2: + # everything can go + log.debug(f'removing nodes: {", ".join(connected)}') + for c in connected: + self.remove_node(c) + removed_nodes.add(c) + # for + # if + # for + # remove all non-user nodes at the edges + while True: + nodes = [n for n in self.nodes if not n.startswith('user ')] + deleted_nodes = 0 + for node in nodes: + if len(list(nx.neighbors(self, node))) == 1: + self.remove_node(node) + deleted_nodes += 1 + if deleted_nodes: + log.debug(f'{deleted_nodes} edge nodes deleted') + continue + break + log.debug(f'{len(self.nodes)} nodes after cleanup') + + def draw(self): + """ + Draw the graph + :return: + """ + + pos = nx.spring_layout(self, iterations=60) + # pos = nx.kamada_kawai_layout(self) + # pos = nx.spiral_layout(self) + # pos = nx.circular_layout(self) + # -pos = nx.bipartite_layout(self) + # -pos = nx.shell_layout(self) + edge_x = [] + edge_y = [] + for edge in self.edges(): + x0, y0 = pos[edge[0]] + x1, y1 = pos[edge[1]] + edge_x.append(x0) + edge_x.append(x1) + edge_x.append(None) + edge_y.append(y0) + edge_y.append(y1) + edge_y.append(None) + + edge_trace = go.Scatter( + x=edge_x, y=edge_y, + line=dict(width=0.5, color='#888'), + hoverinfo='none', + mode='lines') + + node_x = [] + node_y = [] + hovertext = [] + + node_types = sorted(set(n.split()[0] for n in self.nodes())) + node_colors = {t: c for c, t in enumerate(node_types)} + + for node in self.nodes(): + # x, y = g.nodes[node]['pos'] + x, y = pos[node] + node_x.append(x) + node_y.append(y) + hovertext.append(node) + + node_trace = go.Scatter( + x=node_x, y=node_y, + mode='markers', + hoverinfo='text', + marker=dict( + showscale=True, + # colorscale options + # 'Greys' | 'YlGnBu' | 'Greens' | 'YlOrRd' | 'Bluered' | 'RdBu' | + # 'Reds' | 'Blues' | 'Picnic' | 'Rainbow' | 'Portland' | 'Jet' | + # 'Hot' | 'Blackbody' | 'Earth' | 'Electric' | 'Viridis' | + colorscale='YlGnBu', + reversescale=True, + color=[], + size=10, + line_width=2)) + node_trace.text = list(self.nodes()) + node_trace.marker.color = [node_colors[n.split()[0]] for n in self.nodes()] + + # noinspection PyTypeChecker + fig = go.Figure(data=[edge_trace, node_trace], + layout=go.Layout( + title='User groups', + titlefont_size=16, + showlegend=False, + hovermode='closest', + margin=dict(b=20, l=5, r=5, t=40), + xaxis=dict(showgrid=False, zeroline=False, showticklabels=False), + yaxis=dict(showgrid=False, zeroline=False, showticklabels=False)) + ) + fig.write_html('user dependencies.html') + fig.show() + + def draw3d(self): + """ + Draw the graph in 3d + :return: + """ + nodes = [] + node_groups = {} + for node in self.nodes: + node_group = node.split()[0] + if (node_group_id:=node_groups.get(node_group)) is None: + node_group_id = len(node_group) + node_groups[node_group_id] = node_group + nodes.append(GDataNode(id=node, name=node, group=str(node_group_id))) + links = [GDataLink(source=source, target=target) for source, target in self.edges] + g_data = GData(nodes=nodes, links=links) + html_file = 'user dependencies 3d.html' + with open(html_file, mode='w') as f: + f.write(JS_TEMPLATE_3D.replace('{}', g_data.model_dump_json(indent=2))) + webbrowser.open(f'file://{os.path.abspath(html_file)}', new=2) # nosec:'html_file, new=2)