backend_handler.py

import sys
import os
import logging
import argparse
from concurrent.futures import ThreadPoolExecutor, as_completed


# Add code/ and code/classes/ to sys.path so all modules are importable.
_CODE_DIR    = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'code')
_CLASSES_DIR = os.path.join(_CODE_DIR, 'classes')
for _d in (_CODE_DIR, _CLASSES_DIR):
    if _d not in sys.path:
        sys.path.insert(0, _d)

from classes._common import *              # LEVELS, TREE_TYPES, TypeDetector, BeautifulSoup, re, json, os, …
from classes.NodeNavigator import NodeNavigator
from classes.ContentExtractor import ContentExtractor
from classes.DocumentParser import DocumentParser
from classes.Downloader import Downloader
from classes.DownloadTree import DocumentTree
from classes.FullDocumentTree import FullDocumentTree

logger = logging.getLogger(__name__)


# ---------------------------------------------------------------------------
# Public API  (used by parsers/, tower.py, LinkingLawTree.py)
# ---------------------------------------------------------------------------

def _normalize_type(text: str) -> str:
    return TypeDetector.normalize(text)


def strip_script_and_style(html: str) -> str:
    return DocumentParser.strip_scripts(html)


def extract_document_metadata(html: str) -> dict:
    return DocumentParser(html).metadata()


def download_resource(url: str, output_dir: str = 'output'):
    Downloader(output_dir).download(url)


def extract_parts(html: str) -> list:
    soup = BeautifulSoup(html, 'html.parser')
    parts = []
    for span in soup.find_all('span', class_='S_PRT'):
        part_id = span.get('id', '')
        ttl = span.find('span', class_='S_PRT_TTL')
        den = span.find('span', class_='S_PRT_DEN')
        parts.append({
            'id':           part_id,
            'anchor':       f'{part_id}_ttl',
            'title':        ttl.get_text(strip=True) if ttl else '',
            'denomination': den.get_text(strip=True) if den else '',
        })
    return parts


def extract_articles_from_chapter(chapter_element) -> list:
    articles = []
    for span in chapter_element.find_all('span', class_='S_ART'):
        art_id = span.get('id', '')
        ttl = span.find('span', class_='S_ART_TTL')
        den = span.find('span', class_='S_ART_DEN')
        articles.append({
            'id':           art_id,
            'anchor':       f'{art_id}_ttl',
            'title':        ttl.get_text(strip=True) if ttl else '',
            'denomination': den.get_text(strip=True) if den else '',
            'element':      span,
        })
    return articles


def extract_content_from_article(article_element) -> dict:
    return ContentExtractor.article_content(article_element)


def build_database(html: str, ref_filter=None) -> list:
    return DocumentParser(html).build_database(ref_filter)


def save_database(html: str, path: str = 'output/database.json') -> list:
    return DocumentParser(html).save_database(path)


def build_tree(html: str, file=None):
    DocumentTree(html, file=file).print()


def build_full_tree(html: str, file=None, text_limit: int = 80):
    FullDocumentTree(html, file=file, text_limit=text_limit).print()


def build_filtered_tree_and_database(html: str, tree_file=None, db_path=None,
                                     text_limit: int = 80) -> list:
    """Generate a full tree and database keeping only LEGE/OUG/OU references.

    Parses the HTML once and reuses the result for both outputs.
    """
    def _is_tree_ref(text):
        return TypeDetector.normalize(text) in TREE_TYPES

    parser = DocumentParser(html)
    FullDocumentTree.from_parser(parser, file=tree_file, text_limit=text_limit,
                                 ref_filter=_is_tree_ref).print()
    data = parser.build_database(ref_filter=_is_tree_ref)
    if db_path:
        os.makedirs(os.path.dirname(db_path) or '.', exist_ok=True)
        with open(db_path, 'w', encoding='utf-8') as f:
            json.dump(data, f, ensure_ascii=False, indent=2)
        logger.info(f"Filtered database saved to {db_path} ({len(data)} parts)")
    return data


def get_spans_by_class(html: str, class_name: str) -> list:
    pattern = re.compile(
        rf'<span[^>]*\bclass="[^"]*\b{re.escape(class_name)}\b[^"]*"[^>]*>(.*?)</span>',
        re.DOTALL | re.IGNORECASE,
    )
    return pattern.findall(html)


# ---------------------------------------------------------------------------
# Entry point
# ---------------------------------------------------------------------------

def initialize_strucute(id):
    # Create necessary directories for output
    os.makedirs('output', exist_ok=True)
    os.makedirs('input', exist_ok=True)
    os.makedirs(f'input/{id}', exist_ok=True)
    
   



def create_view_tree(html_content, resource_id):
    parser   = DocumentParser(html_content)
    meta     = parser.metadata()
    doc_type = meta['type']
    out_dir  = f'input/{resource_id}'
    logger.info(f"Document type: {doc_type}")

    parser.save_database(path=f'{out_dir}/database.json')

    if doc_type in TREE_TYPES:
        tree_path = f'{out_dir}/tree.txt'
        with open(tree_path, 'w', encoding='utf-8') as f:
            DocumentTree.from_parser(parser, file=f).print()
        logger.info(f"Tree saved to {tree_path}")

        full_tree_path = f'{out_dir}/full_tree.txt'
        with open(full_tree_path, 'w', encoding='utf-8') as f:
            FullDocumentTree.from_parser(parser, file=f).print()
        logger.info(f"Full tree saved to {full_tree_path}")
    else:
        logger.info(f"Skipping tree generation for type '{doc_type}' (only for {TREE_TYPES})")

    build_filtered_tree_and_database(
        html_content,
        tree_file=open(f'{out_dir}/filtered_tree.txt', 'w', encoding='utf-8'),
        db_path=f'{out_dir}/filtered_database.json',
    )


def run_complete_law(doc_id, keep_annotations=False):
    from CompleteLawGenerator import generate
    out_dir  = os.path.join('output', doc_id)
    out_path = os.path.join(out_dir, 'complete_law.txt')
    os.makedirs(out_dir, exist_ok=True)
    text = generate(doc_id, strip_annot=not keep_annotations)
    with open(out_path, 'w', encoding='utf-8') as f:
        f.write(text)
    logger.info(f"Complete law written to {out_path}")


def run_link_graph(doc_id, depth=1, allow_download=False, use_filtered=True,
                   max_nodes=300, include_citations=False):
    from classes.GraphBuilder import GraphBuilder
    from classes.GraphWriter import GraphWriter
    nodes, edges = GraphBuilder(
        root_id           = doc_id,
        max_depth         = depth,
        use_filtered      = use_filtered,
        allow_download    = allow_download,
        max_nodes         = max_nodes,
        include_citations = include_citations,
    ).build()
    out_dir = os.path.join('output', doc_id)
    os.makedirs(out_dir, exist_ok=True)
    json_path = os.path.join(out_dir, 'link_graph.json')
    with open(json_path, 'w', encoding='utf-8') as f:
        json.dump({'nodes': list(nodes.values()), 'edges': edges},
                  f, ensure_ascii=False, indent=2)
    try:
        from CompleteLawGenerator import generate as _gen_law
        law_text = _gen_law(doc_id)
    except Exception:
        law_text = ''

    writer = GraphWriter(nodes, edges, doc_id)
    writer.write_dot(os.path.join(out_dir, 'link_graph.dot'))
    writer.write_html(os.path.join(out_dir, 'link_graph.html'), law_text=law_text)
    writer.write_text(os.path.join(out_dir, 'modifications.txt'))
    mod_count = sum(1 for e in edges if e['action'] != 'citation')
    cit_count = len(edges) - mod_count
    logger.info(f"{len(nodes)} nodes | {mod_count} modifications | {cit_count} citations → {out_dir}/")

def _process_doc(doc_id: str, args) -> None:
    url = f"https://legislatie.just.ro/Public/DetaliiDocument/{doc_id}"
    try:
        Downloader(output_dir='input').download(url)
    except Exception as e:
        logger.error(f"Failed to download {doc_id}: {e}")
        return

    html_path = f'input/{doc_id}/index.html'
    if not os.path.exists(html_path):
        logger.error(f"HTML missing for {doc_id}, skipping")
        return

    db_path = f'input/{doc_id}/database.json'
    if not os.path.exists(db_path):
        with open(html_path, encoding='utf-8') as f:
            html_content = f.read()
        create_view_tree(html_content, doc_id)
    else:
        logger.info(f"Already processed {doc_id}, skipping parse")

    if args.complete_law:
        run_complete_law(doc_id, keep_annotations=args.keep_annotations)

    if args.link_graph:
        run_link_graph(
            doc_id,
            depth             = args.depth,
            allow_download    = args.download,
            use_filtered      = not args.all_refs,
            max_nodes         = args.max_nodes,
            include_citations = args.citations,
        )


def main():
    os.makedirs('input', exist_ok=True)

    ap = argparse.ArgumentParser(description='Download and process Romanian legislation documents.')
    ap.add_argument('--ids', required=True,
                    help='Document ID(s) from Portal Legislativ (comma-separated)')

    # complete-law options
    ap.add_argument('--complete-law', action='store_true',
                    help='Generate a clean full-text of the law (output/<id>/complete_law.txt)')
    ap.add_argument('--keep-annotations', action='store_true',
                    help='Keep (la DD-MM-YYYY, ...) modification notes in complete-law output')

    # link-graph options
    ap.add_argument('--link-graph', action='store_true',
                    help='Build a modification graph (output/<id>/link_graph.*)')
    ap.add_argument('--depth', type=int, default=1,
                    help='Hops to follow beyond the root document (default: 1)')
    ap.add_argument('--download', action='store_true',
                    help='Fetch and parse uncached referenced documents')
    ap.add_argument('--all-refs', action='store_true',
                    help='Include HG / DECIZIE refs, not just LEGE/OUG/OU')
    ap.add_argument('--max-nodes', type=int, default=300,
                    help='Hard cap on collected graph nodes (default: 300)')
    ap.add_argument('--citations', action='store_true',
                    help='Include citation-only edges in the graph')

    args = ap.parse_args()

    doc_ids = [d.strip() for d in args.ids.split(',') if d.strip()]
    with ThreadPoolExecutor(max_workers=min(len(doc_ids), 4)) as pool:
        futures = {pool.submit(_process_doc, doc_id, args): doc_id for doc_id in doc_ids}
        for fut in as_completed(futures):
            doc_id = futures[fut]
            try:
                fut.result()
            except Exception as e:
                logger.error(f"Unhandled error for {doc_id}: {e}")


if __name__ == "__main__":
    main()