schema-processor.py

import sys
import os
import json
import logging
from typing import Dict, List, Optional, Union, Set, Tuple
import requests # type: ignore
import xmltodict # type: ignore
from collections import defaultdict
import pandas as pd
from openpyxl.styles import PatternFill, Font
import re

# Configure logging
logging.basicConfig(
    level=logging.INFO,
    format='%(asctime)s - %(levelname)s - %(message)s'
)
logger = logging.getLogger(__name__)

def get_repo_path() -> str:
    """Get repository path from command line arguments."""
    if len(sys.argv) < 2:
        logger.error('Error: Repository path argument is required')
        print('Usage: python script.py <repository-path>')
        print('Example: python script.py schemas/2.6')
        sys.exit(1)
    
    return sys.argv[1]

def get_repository_contents(input_path: str) -> List[Dict]:
    """Fetch repository contents from GitHub API."""
    try:
        parts = input_path.split('/')
        repo = parts[0]
        path = '/'.join(parts[1:])
        
        base_url = f"https://api.github.com/repos/diggsml/{repo}/contents"
        url = f"{base_url}/{path}" if path else base_url
        
        logger.info(f"Fetching from URL: {url}")
        
        response = requests.get(url)
        response.raise_for_status()
        return response.json()
    
    except requests.exceptions.RequestException as error:
        logger.error(f'Error fetching repository contents: {str(error)}')
        if hasattr(error, 'response'):
            logger.error(f'API Response: {error.response.status_code} {error.response.reason}')
            logger.error(f'API Message: {error.response.text}')
        raise

def get_file_content(url: str) -> Optional[str]:
    """Fetch file content from URL."""
    try:
        response = requests.get(url)
        response.raise_for_status()
        return response.text
    except requests.exceptions.RequestException as error:
        logger.error(f'Error fetching file content: {str(error)}')
        if hasattr(error, 'response'):
            logger.error(f'API Response: {error.response.status_code} {error.response.reason}')
        return None

def should_process_file(filename: str) -> bool:
    """Check if file should be processed based on criteria."""
    if not filename:
        return False
    
    skip_files = ['complete.xsd', 'diggs.xsd', 'dictionary_diggs.xsd', 'xml.xsd']
    is_xsd_file = filename.lower().endswith('.xsd')
    is_skipped = filename.lower() in skip_files
    is_deprecated = 'deprecated' in filename.lower()
    
    logger.info(f"Checking file: {filename}")
    logger.info(f"- Is XSD: {is_xsd_file}")
    logger.info(f"- Is skipped: {is_skipped}")
    logger.info(f"- Is deprecated: {is_deprecated}")
    
    return is_xsd_file and not is_skipped

def get_namespace(schema: Dict) -> Optional[str]:
    """Extract and validate namespace from schema."""
    try:
        if not schema or '@targetNamespace' not in schema:
            logger.error(f'Invalid schema structure: {schema}')
            return None
        
        target_namespace = schema['@targetNamespace']
        logger.info(f"Found namespace: {target_namespace}")
        
        if target_namespace == 'http://diggsml.org/schemas/2.6' or target_namespace == 'http://diggsml.org/schema-dev':
            return 'diggs'
        elif target_namespace == 'http://diggsml.org/schemas/2.6/geotechnical' or target_namespace == 'http://diggsml.org/schema-dev/geotechnical':
            return 'diggs_geo'
        return None
    except Exception as e:
        logger.error(f"Error processing namespace: {str(e)}")
        return None

def extract_elements(schema: Dict) -> Dict[str, List]:
    """Extract elements from schema."""
    elements: Dict[str, List] = {
        'import': [],
        'element': [],
        'complexType': [],
        'simpleType': []
    }
    
    for type_name in elements.keys():
        if type_name in schema:
            # Handle both single items and lists
            items = schema[type_name]
            if not isinstance(items, list):
                items = [items]
            
            elements[type_name] = [{
                'name': item.get('@name', ''),
                'element': item
            } for item in items]
            
            logger.info(f"Extracted {len(elements[type_name])} {type_name} elements")
    
    return elements

def merge_schema_parts(schemas_by_namespace: Dict) -> Dict[str, Dict[str, Dict]]:
    """Merge schema parts by namespace."""
    result = {
        'diggs': defaultdict(dict),
        'diggs_geo': defaultdict(dict)
    }
    
    for namespace, schemas in schemas_by_namespace.items():
        logger.info(f"Merging schemas for namespace: {namespace}")
        for elements in schemas:
            for type_name in ['import', 'element', 'complexType', 'simpleType']:
                for item in elements[type_name]:
                    key = item['name'] or json.dumps(item['element'])
                    if key not in result[namespace][type_name]:
                        result[namespace][type_name][key] = item['element']
    
    return result

def sort_elements_by_name(elements_dict: dict) -> list:
    """Sort elements by their name attribute."""
    def get_element_name(element):
        # Check different possible locations for the name
        if '@name' in element:
            return element['@name'].lower()
        elif 'name' in element:
            return element['name'].lower()
        elif '@schemaLocation' in element:  # For import elements
            return element['@schemaLocation'].lower()
        return ''  # Default value for sorting if no name found
    
    # Convert dictionary values to list and sort
    elements_list = list(elements_dict.values())
    return sorted(elements_list, key=get_element_name)

def create_final_schema(namespace: str, elements: Dict) -> str:
    """Create final schema XML content."""
    logger.info(f"Creating final schema for namespace: {namespace}")
    
    # Define common namespaces
    common_namespaces = {
        'diggs': 'http://diggsml.org/schemas/2.6',
        'diggs_geo': 'http://diggsml.org/schemas/2.6/geotechnical',
        'xsi': 'http://www.w3.org/2001/XMLSchema-instance',
        'xmlns': 'http://www.w3.org/2001/XMLSchema',
        'xlink': 'http://www.w3.org/1999/xlink',
        'eml': 'http://www.energistics.org/energyml/data/commonv2',
        'witsml': 'http://www.energistics.org/energyml/data/witsmlv2',
        'gml': 'http://www.opengis.net/gml/3.2',
        'vc': 'http://www.w3.org/2007/XMLSchema-versioning'
    }

    # Base schema attributes
    schema_attrs = {
        '@targetNamespace': common_namespaces['diggs'] if namespace == 'diggs' else common_namespaces['diggs_geo'],
        '@elementFormDefault': 'qualified',
        '@version': '2.6',
        '@vc:minVersion': '1.0',
        '@vc:maxVersion': '1.1'
    }

    # Add namespace declarations based on schema type
    if namespace == 'diggs':
        schema_attrs.update({
            '@xmlns:diggs': common_namespaces['diggs'],
            '@xmlns:diggs_geo': common_namespaces['diggs_geo'],
            '@xmlns:g3.3': 'http://www.opengis.net/gml/3.3/ce',
            '@xmlns:glr': 'http://www.opengis.net/gml/3.3/lr',
            '@xmlns:glrov': 'http://www.opengis.net/gml/3.3/lrov',
            '@xmlns:xsi': common_namespaces['xsi'],
            '@xmlns': common_namespaces['xmlns'],
            '@xmlns:xlink': common_namespaces['xlink'],
            '@xmlns:eml': common_namespaces['eml'],
            '@xmlns:witsml': common_namespaces['witsml'],
            '@xmlns:gml': common_namespaces['gml'],
            '@xmlns:vc': common_namespaces['vc']
        })
    else:
        schema_attrs.update({
            '@xmlns:xlink': common_namespaces['xlink'],
            '@xmlns:diggs': common_namespaces['diggs'],
            '@xmlns:diggs_geo': common_namespaces['diggs_geo'],
            '@xmlns:eml': common_namespaces['eml'],
            '@xmlns:witsml': common_namespaces['witsml'],
            '@xmlns:xsi': common_namespaces['xsi'],
            '@xmlns': common_namespaces['xmlns'],
            '@xmlns:gml': common_namespaces['gml'],
            '@xmlns:vc': common_namespaces['vc']
        })

    # Create schema structure
    schema = {'schema': schema_attrs}
    
    # Add and sort elements
    for type_name in ['import', 'element', 'complexType', 'simpleType']:
        if elements[type_name]:
            # Sort elements alphabetically
            sorted_items = sort_elements_by_name(elements[type_name])
            if sorted_items:
                schema['schema'][type_name] = sorted_items
                logger.info(f"Added {len(sorted_items)} sorted {type_name} elements to final schema")

    # Convert to XML with single declaration
    xml_content = xmltodict.unparse(schema, pretty=True)
    
    # Ensure there's only one XML declaration line
    if not xml_content.startswith('<?xml'):
        xml_content = '<?xml version="1.0" encoding="utf-8"?>\n' + xml_content
    
    return xml_content

def get_element_by_ref(ref: str, schemas: Dict) -> Optional[Dict]:
    """Find referenced element in schemas."""
    if not isinstance(ref, str):
        logger.error(f"Invalid reference type: {type(ref)}")
        return None
    ns_prefix = ref.split(':')[0] if ':' in ref else ''
    element_name = ref.split(':')[1] if ':' in ref else ref
    
    for schema in schemas.values():
        elements = schema.get('schema', {}).get('element', [])
        if not isinstance(elements, list):
            elements = [elements]
        
        for element in elements:
            if element.get('@name') == element_name:
                return element
    
    return None

def is_numeric_type(type_str: str) -> bool:
    """Check if type is numeric."""
    numeric_types = {'double', 'decimal', 'float', 'integer', 'int', 'long', 'short', 'byte'}
    type_name = type_str.split(':')[-1].lower()
    return type_name in numeric_types

def is_target_type(type_str: str) -> bool:
    """Check if type meets target criteria."""
    if not type_str:
        return False
    
    numeric_types = {
        'decimal', 'integer', 'byte', 'int', 'long', 'negativeInteger',
        'nonPositiveInteger', 'positiveInteger', 'short', 'unsignedLong',
        'unsignedInt', 'unsignedByte', 'double', 'nonNegativeInteger',
        'unsignedShort', 'float', 'eml:PositiveLong', 'eml:NonNegativeLong'
    }
    
    type_lower = type_str.lower()
    base_type = type_str.split(':')[-1]
    
    # Check for measure but not measurement
    if 'measure' in type_lower and 'measurement' not in type_lower:
        return True
    
    # Check for ValueType (case sensitive)
    if 'ValueType' in type_str:
        return True
    
    # Check for code (case insensitive)
    if 'code' in type_lower:
        return True
    
    # Check if it's one of the numeric types
    return base_type in numeric_types

def find_parent_elements(complex_type_name: str, schemas: Dict) -> List[str]:
    """Find all elements that use this complex type."""
    parents = []
    
    for schema in schemas.values():
        elements = schema.get('schema', {}).get('element', [])
        if not isinstance(elements, list):
            elements = [elements]
        
        for element in elements:
            if element.get('@type') == complex_type_name:
                element_name = element.get('@name', '')
                if element_name:
                    # Add namespace prefix based on schema
                    ns_prefix = 'diggs_geo' if 'geotechnical' in schema['schema']['@targetNamespace'] else 'diggs'
                    parents.append(f"{ns_prefix}:{element_name}")
    
    return parents

def analyze_complex_types(schemas: Dict) -> pd.DataFrame:
    """Analyze complex types and their elements/attributes in schema files."""
    records = []
    
    logger.info("Starting detailed analysis of complex types...")
    
    for schema_file, schema_content in schemas.items():
        complex_types = schema_content.get('schema', {}).get('complexType', [])
        if not isinstance(complex_types, list):
            complex_types = [complex_types]
        
        for complex_type in complex_types:
            # Get complex type name with namespace prefix
            complex_type_name = complex_type.get('@name', '')
            ns_prefix = 'diggs_geo' if 'geotechnical' in schema_content['schema']['@targetNamespace'] else 'diggs'
            qualified_complex_type_name = f"{ns_prefix}:{complex_type_name}"
            
            # Find parent elements
            parent_elements = find_parent_elements(qualified_complex_type_name, schemas)
            
            # Always process attributes regardless of type
            attributes = get_all_attributes(complex_type, ns_prefix)
            for attribute in attributes:
                process_element_or_attribute(attribute, True, qualified_complex_type_name, 
                                          parent_elements, ns_prefix, schemas, records)
            
            # Only process elements if NOT a PropertyType
            if 'PropertyType' not in complex_type_name:
                elements = get_all_compositor_elements(complex_type, ns_prefix)
                for element in elements:
                    process_element_or_attribute(element, False, qualified_complex_type_name, 
                                              parent_elements, ns_prefix, schemas, records)
    
    # Create DataFrame
    df = pd.DataFrame(records)
    if not df.empty:
        # Ensure all required columns exist
        required_columns = ['Parent Element', 'Property', 'Property Xpath', 'Documentation', 
                          'Type', 'Min Occurs', 'Max Occurs', 'Child Elements', 'Parent Type', 
                          'Codelist Dictionary']
        for col in required_columns:
            if col not in df.columns:
                df[col] = ''
        
        # Sort by specified columns
        df = df.sort_values(['Parent Element', 'Type', 'Property'])
    
    return df

def get_compositor_elements(compositor: Dict, ns_prefix: str) -> List[Dict]:
    """Recursively extract elements from sequence and choice compositors."""
    elements = []
    
    if not isinstance(compositor, dict):
        return elements
        
    # Handle direct elements in this compositor
    if 'element' in compositor:
        if isinstance(compositor['element'], list):
            elements.extend(compositor['element'])
        else:
            elements.append(compositor['element'])
    
    # Handle nested sequence
    if 'sequence' in compositor:
        nested_seq = compositor['sequence']
        if nested_seq:
            # Process direct elements in sequence
            if 'element' in nested_seq:
                if isinstance(nested_seq['element'], list):
                    elements.extend(nested_seq['element'])
                else:
                    elements.append(nested_seq['element'])
            
            # Process nested choice in sequence
            if 'choice' in nested_seq:
                elements.extend(get_compositor_elements(nested_seq['choice'], ns_prefix))
    
    # Handle nested choice
    if 'choice' in compositor:
        nested_choice = compositor['choice']
        if nested_choice:
            # Process direct elements in choice
            if 'element' in nested_choice:
                if isinstance(nested_choice['element'], list):
                    elements.extend(nested_choice['element'])
                else:
                    elements.append(nested_choice['element'])
            
            # Process nested sequence in choice
            if 'sequence' in nested_choice:
                elements.extend(get_compositor_elements(nested_choice['sequence'], ns_prefix))
    
    return elements

def get_all_compositor_elements(complex_type: Dict, ns_prefix: str) -> List[Dict]:
    """Get all elements from a complex type's compositors."""
    elements = []
    
    # Handle complexContent
    complex_content = complex_type.get('complexContent', {})
    if complex_content:
        extension = complex_content.get('extension', {})
        if extension:
            # Get elements from sequence
            if 'sequence' in extension:
                elements.extend(get_compositor_elements(extension['sequence'], ns_prefix))
            
            # Get elements from choice
            if 'choice' in extension:
                elements.extend(get_compositor_elements(extension['choice'], ns_prefix))
    
    # Handle direct sequence and choice
    if 'sequence' in complex_type:
        elements.extend(get_compositor_elements(complex_type['sequence'], ns_prefix))
    
    if 'choice' in complex_type:
        elements.extend(get_compositor_elements(complex_type['choice'], ns_prefix))
    
    return elements

def process_element_or_attribute(item: Dict, is_attribute: bool, parent_type: str, 
                               parent_elements: List[str], ns_prefix: str, 
                               schemas: Dict, records: List[Dict]) -> None:
    """Process individual element or attribute and add to records."""
    # Handle both direct and referenced items
    item_type = item.get('@type', '')
    item_ref = item.get('@ref', '')
    
    if item_ref:
        # Handle references, including those with namespace prefixes
        ref_parts = item_ref.split(':')
        ref_ns = ref_parts[0] if len(ref_parts) > 1 else ns_prefix
        ref_name = ref_parts[-1]
        
        # Find referenced item in schemas
        referenced_item = None
        for schema in schemas.values():
            if is_attribute:
                container = schema.get('schema', {}).get('attribute', [])
            else:
                container = schema.get('schema', {}).get('element', [])
                
            if not isinstance(container, list):
                container = [container]
                
            for item_def in container:
                if item_def.get('@name') == ref_name:
                    referenced_item = item_def
                    break
                    
            if referenced_item:
                break
                
        if referenced_item:
            item_type = referenced_item.get('@type', '')
            item_name = item_ref
        else:
            logger.info(f"Referenced item {item_ref} not found in schemas")
            item_name = item_ref
    else:
        item_name = item.get('@name', '')
        if ':' not in item_name:
            item_name = f"{ns_prefix}:{item_name}"
    
    if is_attribute:
        item_name = f"@{item_name}"
    
    # Get occurrence constraints
    if is_attribute:
        min_occurs = '0' if item.get('@use') in [None, 'optional'] else '1'
        max_occurs = '1'
    else:
        min_occurs = item.get('@minOccurs', '1')
        max_occurs = item.get('@maxOccurs', '1')
    
    # Get documentation
    documentation = get_documentation(item)
    
    # Get child elements for PropertyType
    child_elements = []
    if item_type and 'PropertyType' in item_type:
        logger.info(f"Found PropertyType: {item_type}")
        child_elements = get_property_type_children(item_type, schemas)
        logger.info(f"Found child elements: {child_elements}")
    
    # Create record for each parent element
    for parent in parent_elements or ['']:
        record = {
            'Parent Element': parent,
            'Property': item_name,
            'Property Xpath': f"{parent}/{item_name}" if parent else item_name,
            'Documentation': documentation,
            'Type': item_type,
            'Min Occurs': min_occurs,
            'Max Occurs': max_occurs,
            'Child Elements': '\n'.join(child_elements) if child_elements else '',
            'Parent Type': parent_type,
            'Codelist Dictionary': ''
        }
        records.append(record)
        

def get_property_type_children(type_name: str, schemas: Dict) -> List[str]:
    """Get child elements of a PropertyType."""
    children = []
    logger.info(f"Searching for children of PropertyType: {type_name}")
    
    # Remove namespace prefix if present
    base_type_name = type_name.split(':')[-1] if ':' in type_name else type_name
    
    # Special handling for specific known PropertyTypes
    special_types = {
        'PointLocationPropertyType': ['diggs:PointLocation'],
        'LinearExtentPropertyType': ['diggs:LinearExtent'],
        'LocalityPropertyType': ['diggs:Locality'],
        'LocationPropertyType': ['diggs:Location'],
        'TimeIntervalOrInstantPropertyType': ['diggs:TimeInstant', 'diggs:TimeInterval'],
        'GP_SensorLocationsPropertyType': ['diggs:GP_SensorLocations'],
        'MeasurePropertyType': ['diggs:Measure'],
        'ValuePropertyType': ['diggs:Value'],
        'QuantityPropertyType': ['diggs:Quantity']
    }
    
    # Check for special types first
    if base_type_name in special_types:
        return special_types[base_type_name]
    
    # If type name doesn't end in 'PropertyType', return empty list
    if not base_type_name.endswith('PropertyType'):
        return children
    
    # Remove 'PropertyType' suffix to get the base name
    base_name = base_type_name[:-12]  # Remove 'PropertyType'
    
    def process_elements(container, ns_prefix):
        """Helper function to process elements from a container."""
        result = []
        if not container:
            return result
            
        # Handle element references
        elements = container.get('element', [])
        if elements:
            if not isinstance(elements, list):
                elements = [elements]
            for element in elements:
                ref = element.get('@ref')
                name = element.get('@name')
                if ref:
                    if ':' not in ref:
                        ref = f"{ns_prefix}:{ref}"
                    result.append(ref)
                    logger.info(f"Found child element ref: {ref}")
                elif name:
                    qualified_name = f"{ns_prefix}:{name}"
                    result.append(qualified_name)
                    logger.info(f"Found child element name: {qualified_name}")
        
        # Handle attribute references
        attributes = container.get('attribute', [])
        if attributes:
            if not isinstance(attributes, list):
                attributes = [attributes]
            for attr in attributes:
                ref = attr.get('@ref')
                name = attr.get('@name')
                if ref:
                    if ':' not in ref:
                        ref = f"@{ns_prefix}:{ref}"
                    else:
                        ref = f"@{ref}"
                    result.append(ref)
                    logger.info(f"Found child attribute ref: {ref}")
                elif name:
                    qualified_name = f"@{ns_prefix}:{name}"
                    result.append(qualified_name)
                    logger.info(f"Found child attribute name: {qualified_name}")
        
        return result

    # Search through all schemas
    for schema in schemas.values():
        schema_content = schema.get('schema', {})
        
        # Determine namespace prefix based on schema target namespace
        ns_prefix = 'diggs_geo' if 'geotechnical' in schema_content.get('@targetNamespace', '') else 'diggs'
        
        # Check complex types
        complex_types = schema_content.get('complexType', [])
        if not isinstance(complex_types, list):
            complex_types = [complex_types]
        
        for complex_type in complex_types:
            if complex_type.get('@name') == base_type_name:
                # Process complex content
                complex_content = complex_type.get('complexContent', {})
                if complex_content:
                    extension = complex_content.get('extension', {})
                    if extension:
                        # Process direct elements/attributes in extension
                        children.extend(process_elements(extension, ns_prefix))
                        
                        # Process sequence
                        sequence = extension.get('sequence', {})
                        if sequence:
                            children.extend(process_elements(sequence, ns_prefix))
                            
                            # Check for choice within sequence
                            choice_in_seq = sequence.get('choice', {})
                            if choice_in_seq:
                                children.extend(process_elements(choice_in_seq, ns_prefix))
                        
                        # Process choice at extension level
                        choice = extension.get('choice', {})
                        if choice:
                            children.extend(process_elements(choice, ns_prefix))
                
                # Process simple content
                simple_content = complex_type.get('simpleContent', {})
                if simple_content:
                    extension = simple_content.get('extension', {})
                    if extension:
                        children.extend(process_elements(extension, ns_prefix))
    
    if not children:
        # If no children found but it's a PropertyType, add the base element as a child
        base_element = f"diggs:{base_name}"
        children.append(base_element)
        logger.info(f"No children found, adding base element: {base_element}")
    
    # Remove duplicates while preserving order
    return list(dict.fromkeys(children))   
            
def get_documentation(element: Dict) -> str:
    """Extract and clean documentation from element."""
    doc_text = ''  # Initialize doc_text at function scope
    
    try:
        annotation = element.get('annotation', {})
        if annotation and isinstance(annotation, dict):
            documentation = annotation.get('documentation', '')
            if documentation:
                if isinstance(documentation, str):
                    doc_text = documentation
                elif isinstance(documentation, dict):
                    # Handle case where documentation might be in #text
                    doc_text = documentation.get('#text', '')
                
                # Clean up the documentation text:
                # 1. Replace newlines and carriage returns with spaces
                # 2. Strip leading/trailing spaces
                # 3. Convert multiple spaces to single space
                doc_text = re.sub(r'[\r\n]+', ' ', doc_text)
                doc_text = doc_text.strip()
                doc_text = re.sub(r'\s+', ' ', doc_text)
    except Exception as e:
        logger.error(f"Error extracting documentation: {str(e)}")
    
    return doc_text

    def process_compositor(compositor: Dict) -> List[Dict]:
        """Recursively process compositor elements."""
        comp_elements = []
        
        if not isinstance(compositor, dict):
            return comp_elements
            
        # Handle direct elements
        if 'element' in compositor:
            if isinstance(compositor['element'], list):
                comp_elements.extend(compositor['element'])
            else:
                comp_elements.append(compositor['element'])
        
        # Handle nested sequence
        if 'sequence' in compositor:
            nested_seq = compositor['sequence']
            if nested_seq:
                comp_elements.extend(process_compositor(nested_seq))
        
        # Handle nested choice
        if 'choice' in compositor:
            nested_choice = compositor['choice']
            if nested_choice:
                comp_elements.extend(process_compositor(nested_choice))
        
        return comp_elements
    
    # Handle complexContent
    complex_content = complex_type.get('complexContent', {})
    if complex_content:
        extension = complex_content.get('extension', {})
        if extension:
            if 'sequence' in extension:
                elements.extend(process_compositor(extension['sequence']))
            if 'choice' in extension:
                elements.extend(process_compositor(extension['choice']))
    
    # Handle direct sequence and choice
    if 'sequence' in complex_type:
        elements.extend(process_compositor(complex_type['sequence']))
    if 'choice' in complex_type:
        elements.extend(process_compositor(complex_type['choice']))
    
    return elements

def get_all_attributes(complex_type: Dict, ns_prefix: str) -> List[Dict]:
    """Get all attributes from a complex type, including referenced and direct attributes."""
    attributes = []
    
    def process_attributes_container(container):
        """Helper function to process attributes from a container."""
        if not container:
            return []
            
        attr_list = []
        # Handle attribute
        if 'attribute' in container:
            attrs = container['attribute']
            if not isinstance(attrs, list):
                attrs = [attrs]
            # Only include DIGGS namespace attributes
            for attr in attrs:
                ref = attr.get('@ref', '')
                name = attr.get('@name', '')
                if ref:
                    # Check if reference is in DIGGS namespace
                    is_diggs = ref.startswith(('diggs:', 'diggs_geo:')) or ':' not in ref
                    if is_diggs:
                        attr_list.append(attr)
                else:
                    # Direct attributes inherit namespace from complex type
                    attr_list.append(attr)
            
        # Handle attributeGroup references
        if 'attributeGroup' in container:
            attr_groups = container['attributeGroup']
            if not isinstance(attr_groups, list):
                attr_groups = [attr_groups]
            # Only include DIGGS namespace attribute groups
            for group in attr_groups:
                ref = group.get('@ref', '')
                if ref.startswith(('diggs:', 'diggs_geo:')) or ':' not in ref:
                    attr_list.append(group)
            
        return attr_list
    
    # Get direct attributes from complex type
    attributes.extend(process_attributes_container(complex_type))
    
    # Get attributes from complexContent/extension
    complex_content = complex_type.get('complexContent', {})
    if complex_content:
        extension = complex_content.get('extension', {})
        if extension:
            attributes.extend(process_attributes_container(extension))
            
        # Check for attributes in restriction
        restriction = complex_content.get('restriction', {})
        if restriction:
            attributes.extend(process_attributes_container(restriction))
    
    # Get attributes from simpleContent/extension
    simple_content = complex_type.get('simpleContent', {})
    if simple_content:
        extension = simple_content.get('extension', {})
        if extension:
            attributes.extend(process_attributes_container(extension))
            
        # Check for attributes in restriction
        restriction = simple_content.get('restriction', {})
        if restriction:
            attributes.extend(process_attributes_container(restriction))
    
    logger.info(f"Found {len(attributes)} total attributes")
    return attributes

def get_all_compositor_elements(complex_type: Dict, ns_prefix: str) -> List[Dict]:
    """Get all elements from a complex type's compositors."""
    elements = []
    
    def process_compositor(compositor: Dict) -> List[Dict]:
        """Recursively process compositor elements."""
        comp_elements = []
        
        if not isinstance(compositor, dict):
            return comp_elements
            
        # Handle direct elements
        if 'element' in compositor:
            if isinstance(compositor['element'], list):
                elements_list = compositor['element']
            else:
                elements_list = [compositor['element']]
                
            # Only include DIGGS namespace elements
            for elem in elements_list:
                ref = elem.get('@ref', '')
                name = elem.get('@name', '')
                if ref:
                    # Check if reference is in DIGGS namespace
                    is_diggs = ref.startswith(('diggs:', 'diggs_geo:')) or ':' not in ref
                    if is_diggs:
                        comp_elements.append(elem)
                else:
                    # Direct elements inherit namespace from complex type
                    comp_elements.append(elem)
        
        # Handle element references through groups
        if 'group' in compositor:
            groups = compositor['group']
            if not isinstance(groups, list):
                groups = [groups]
            # Only include DIGGS namespace groups
            for group in groups:
                ref = group.get('@ref', '')
                if ref.startswith(('diggs:', 'diggs_geo:')) or ':' not in ref:
                    comp_elements.append(group)
        
        # Handle nested compositors
        for nested_type in ['sequence', 'choice', 'all']:
            if nested_type in compositor:
                nested_comp = compositor[nested_type]
                if nested_comp:
                    if isinstance(nested_comp, list):
                        for comp in nested_comp:
                            comp_elements.extend(process_compositor(comp))
                    else:
                        comp_elements.extend(process_compositor(nested_comp))
        
        return comp_elements
    
    # Handle complexContent
    complex_content = complex_type.get('complexContent', {})
    if complex_content:
        # Handle extension
        extension = complex_content.get('extension', {})
        if extension:
            for compositor_type in ['sequence', 'choice', 'all', 'group']:
                if compositor_type in extension:
                    elements.extend(process_compositor({compositor_type: extension[compositor_type]}))
                
        # Handle restriction
        restriction = complex_content.get('restriction', {})
        if restriction:
            for compositor_type in ['sequence', 'choice', 'all', 'group']:
                if compositor_type in restriction:
                    elements.extend(process_compositor({compositor_type: restriction[compositor_type]}))
    
    # Handle direct compositors at root level
    for compositor_type in ['sequence', 'choice', 'all', 'group']:
        if compositor_type in complex_type:
            elements.extend(process_compositor({compositor_type: complex_type[compositor_type]}))
    
    return elements

def find_global_declaration(ref: str, schemas: Dict, is_attribute: bool) -> Optional[Dict]:
    """Find global element or attribute declaration in schemas."""
    if not ref:
        return None
        
    # Handle namespace prefix
    ref_parts = ref.split(':')
    ref_name = ref_parts[-1]
    ref_ns = ref_parts[0] if len(ref_parts) > 1 else None
    
    for schema in schemas.values():
        schema_content = schema.get('schema', {})
        target_ns = schema_content.get('@targetNamespace', '')
        
        # Check if we're in the right namespace
        if ref_ns:
            if (ref_ns == 'diggs' and 'geotechnical' in target_ns) or \
               (ref_ns == 'diggs_geo' and 'geotechnical' not in target_ns):
                continue
        
        # Look in appropriate container (element or attribute)
        container_type = 'attribute' if is_attribute else 'element'
        container = schema_content.get(container_type, [])
        if not isinstance(container, list):
            container = [container]
            
        # Search for matching declaration
        for item in container:
            if item.get('@name') == ref_name:
                return item
    
    return None

def process_element_or_attribute(item: Dict, is_attribute: bool, parent_type: str, 
                               parent_elements: List[str], ns_prefix: str, 
                               schemas: Dict, records: List[Dict]) -> None:
    """Process individual element or attribute and add to records."""
    # Handle both direct and referenced items
    item_type = item.get('@type', '')
    item_ref = item.get('@ref', '')
    
    if item_ref:
        # Find the referenced global declaration
        referenced_item = find_global_declaration(item_ref, schemas, is_attribute)
        if referenced_item:
            # Get type and documentation from referenced declaration
            item_type = referenced_item.get('@type', '')
            documentation = get_documentation(referenced_item)
        else:
            logger.info(f"Referenced item {item_ref} not found in schemas")
            documentation = get_documentation(item)
            
        item_name = item_ref
    else:
        item_name = item.get('@name', '')
        if ':' not in item_name:
            item_name = f"{ns_prefix}:{item_name}"
        documentation = get_documentation(item)
    
    if is_attribute:
        if not item_name.startswith('@'):
            item_name = f"@{item_name}"
    
    # Get occurrence constraints
    if is_attribute:
        min_occurs = '0' if item.get('@use') in [None, 'optional'] else '1'
        max_occurs = '1'
    else:
        min_occurs = item.get('@minOccurs', '1')
        max_occurs = item.get('@maxOccurs', '1')
    
    # Get child elements for PropertyType
    child_elements = []
    if item_type and 'PropertyType' in item_type:
        logger.info(f"Found PropertyType: {item_type}")
        child_elements = get_property_type_children(item_type, schemas)
        logger.info(f"Found child elements: {child_elements}")
    
    # Create record for each parent element
    for parent in parent_elements or ['']:
        record = {
            'Parent Element': parent,
            'Property': item_name,
            'Property Xpath': f"{parent}/{item_name}" if parent else item_name,
            'Documentation': documentation,
            'Type': item_type,
            'Min Occurs': min_occurs,
            'Max Occurs': max_occurs,
            'Child Elements': '\n'.join(child_elements) if child_elements else '',
            'Parent Type': parent_type,
            'Codelist Dictionary': ''
        }
        records.append(record)
        
def create_excel_report(df: pd.DataFrame, output_file: str = 'schema_analysis.xlsx'):
    """Create formatted Excel report with specified styling."""
    from openpyxl.styles import PatternFill, Font, Alignment, Border, Side
    
    try:
        # Define column order
        column_order = [
            'Parent Element', 'Property', 'Property Xpath', 'Documentation',
            'Type', 'Min Occurs', 'Max Occurs', 'Child Elements',
            'Parent Type', 'Codelist Dictionary'
        ]
        
        # Reorder columns
        df = df[column_order]
        
        with pd.ExcelWriter(output_file, engine='openpyxl') as writer:
            df.to_excel(writer, sheet_name='Schema Analysis', index=False)
            
            workbook = writer.book
            worksheet = writer.sheets['Schema Analysis']
            
            # Format header row
            header_font = Font(name='Arial', size=12, bold=True, color='000000')
            header_fill = PatternFill(start_color='FFFF00', end_color='FFFF00', fill_type='solid')
            
            # Define border style
            thin_border = Border(
                left=Side(style=None),
                right=Side(style=None),
                top=Side(style=None),
                bottom=Side(style='thin', color='000000')
            )
            
            # Apply header formatting
            for cell in worksheet[1]:
                cell.font = header_font
                cell.fill = header_fill
                cell.border = thin_border
            
            # Define styles
            normal_font = Font(name='Arial', size=10, color='000000')
            bold_red_font = Font(name='Arial', size=10, bold=True, color='FF0000')
            bold_blue_font = Font(name='Arial', size=10, bold=True, color='0000FF')
            
            # Apply alternating row colors based on Parent Element changes
            white_fill = PatternFill(start_color='FFFFFF', end_color='FFFFFF', fill_type='solid')
            gray_fill = PatternFill(start_color='D3D3D3', end_color='D3D3D3', fill_type='solid')
            
            current_parent = None
            current_fill = white_fill
            
            # Numeric types that should be highlighted in red
            numeric_types = {
                'decimal', 'integer', 'byte', 'int', 'long', 'negativeInteger',
                'nonPositiveInteger', 'positiveInteger', 'short', 'unsignedLong',
                'unsignedInt', 'unsignedByte', 'double', 'nonNegativeInteger',
                'unsignedShort', 'float', 'eml:PositiveLong', 'eml:NonNegativeLong'
            }
            
            # Format data rows
            for row_idx in range(2, worksheet.max_row + 1):
                # Get the parent element value
                parent = worksheet.cell(row=row_idx, column=1).value
                
                # Change fill color when parent element changes
                if parent != current_parent:
                    current_parent = parent
                    current_fill = white_fill if current_fill == gray_fill else gray_fill
                
                # Get Type value to determine font color for entire row
                type_value = worksheet.cell(row=row_idx, column=column_order.index('Type') + 1).value
                row_font = normal_font
                
                if type_value:
                    type_str = str(type_value)
                    type_lower = type_str.lower()
                    base_type = type_str.split(':')[-1]  # Get part after namespace prefix
                    
                    # Check for measure but not measurement, and not in PropertyType
                    is_measure = ('measure' in type_lower and 
                                'measurement' not in type_lower and 
                                'propertytype' not in type_lower)
                    
                    # Check for ValueType (case sensitive)
                    is_value_type = 'ValueType' in base_type and 'PropertyType' not in base_type
                    
                    # Check for numeric types
                    is_numeric = base_type in numeric_types
                    
                    # Check for code (case insensitive)
                    is_code = 'code' in type_lower and 'propertytype' not in type_lower
                    
                    if is_measure or is_value_type or is_numeric:
                        row_font = bold_red_font
                    elif is_code:
                        row_font = bold_blue_font
                
                # Apply formatting to each cell in the row
                for cell in worksheet[row_idx]:
                    cell.fill = current_fill
                    cell.font = row_font
                    cell.border = thin_border
                    # Set middle alignment for all cells that don't have special formatting
                    if not cell.alignment or not cell.alignment.wrap_text:
                        cell.alignment = Alignment(vertical='center')
            
            # Adjust column widths and wrap text
            for col_idx, column in enumerate(worksheet.columns, 1):
                col_letter = column[0].column_letter
                column_name = column_order[col_idx-1]
                
                # Set specific widths and wrapping for certain columns
                if column_name in ['Documentation', 'Child Elements']:
                    worksheet.column_dimensions[col_letter].width = 50
                    for cell in column:
                        cell.alignment = Alignment(wrap_text=True, vertical='center')
                else:
                    # Calculate width based on content
                    max_length = 0
                    for cell in column:
                        try:
                            if cell.value:
                                max_length = max(max_length, len(str(cell.value)))
                        except:
                            pass
                    # Set width with a minimum of 10 and maximum of 30
                    worksheet.column_dimensions[col_letter].width = max(min(max_length + 2, 30), 10)
            
        logger.info(f"Created Excel workbook: {output_file}")
        
    except Exception as error:
        logger.error(f"Error creating Excel report: {str(error)}")
        raise

def  analyze_schemas(schema_files: List[str]) -> None:
    if not schema_files:
        logger.error("No schema files provided")
        return
    
    if not all(isinstance(f, str) for f in schema_files):
        logger.error("Invalid schema file names")
        return    """
    Analyze the generated XSD schema files and create detailed Excel report.
    
    Args:
        schema_files: List of schema file paths to analyze
    """
    schemas = {}
    
    # Load all schemas
    for schema_file in schema_files:
        logger.info(f"Loading schema file: {schema_file}")
        try:
            with open(schema_file, 'r', encoding='utf-8') as f:
                schemas[schema_file] = xmltodict.parse(f.read())
        except Exception as error:
            logger.error(f"Error loading {schema_file}: {str(error)}")
            continue
    
    # Perform analysis
    df = analyze_complex_types(schemas)
    
    # Create Excel report
    create_excel_report(df)

def main() -> None:
    """Main function to process schemas."""
    try:
        logger.info('DIGGS Schema Processor Starting...')
        
        # Get repository path from command line
        repo_path = get_repo_path()
        logger.info(f"Using repository path: {repo_path}")

        # Get repository contents
        logger.info('Fetching repository contents...')
        contents = get_repository_contents(repo_path)
        
        if not isinstance(contents, list):
            raise ValueError('Invalid response from GitHub API')
        
        logger.info(f"Found {len(contents)} items in repository")
        
        # Process each file
        schemas_by_namespace: Dict[str, List] = {
            'diggs': [],
            'diggs_geo': []
        }
        
        processed_files = 0
        
        for item in contents:
            if item['type'] == 'file' and should_process_file(item['name']):
                logger.info(f"Processing {item['name']}...")
                content = get_file_content(item['download_url'])
                if content:
                    try:
                        # Parse XML to dict
                        result = xmltodict.parse(content)
                        namespace = get_namespace(result['schema'])
                        
                        if namespace:
                            elements = extract_elements(result['schema'])
                            schemas_by_namespace[namespace].append(elements)
                            processed_files += 1
                            logger.info(f"Successfully processed {item['name']}")
                        else:
                            logger.info(f"Skipping {item['name']} - not in target namespace")
                    except Exception as error:
                        logger.error(f"Error processing {item['name']}: {str(error)}")
            else:
                logger.info(f"Skipping {item['name']} - does not meet processing criteria")
        
        logger.info(f"Successfully processed {processed_files} files")
        
        # Merge and sort schemas
        merged_schemas = merge_schema_parts(schemas_by_namespace)
        
        # Create and write final schema files
        for namespace in ['diggs', 'diggs_geo']:
            schema_xml = create_final_schema(namespace, merged_schemas[namespace])
            
            filename = f"{namespace}_ns.xsd"
            with open(filename, 'w', encoding='utf-8') as f:
                f.write(schema_xml)
            logger.info(f"Created {filename}")
        
        logger.info('Processing complete!')
        try:
            logger.info("Analyzing generated schema files...")
            schema_files = ['diggs_ns.xsd', 'diggs_geo_ns.xsd']
            analyze_schemas(schema_files)
            logger.info("Schema analysis complete!")
            sys.exit(0)
        except Exception as error:
            logger.error(f"Error during schema analysis: {str(error)}")
            raise
       
    except Exception as error:
        logger.error(f'Fatal error: {str(error)}')
        sys.exit(1)

    sys.exit(0)

if __name__ == '__main__':
    main()