From 716c7064e9b06f2dae810c35d03fcad5fcaac20d Mon Sep 17 00:00:00 2001
From: Mirochill <200482516+Mirochill@users.noreply.github.com>
Date: Sun, 24 May 2026 12:46:01 +0200
Subject: [PATCH] Replace cgi content type parsing

---
 dirhunt/crawler_url.py            | 14 +++++++++++---
 dirhunt/tests/test_crawler_url.py |  6 +++++-
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/dirhunt/crawler_url.py b/dirhunt/crawler_url.py
index 246cce9..68e608c 100644
--- a/dirhunt/crawler_url.py
+++ b/dirhunt/crawler_url.py
@@ -1,6 +1,6 @@
 # -*- coding: utf-8 -*-
-import cgi
 import socket
+from email.message import Message
 
 from bs4 import BeautifulSoup
 from requests import RequestException
@@ -17,6 +17,14 @@
 }
 
 
+def parse_content_type(content_type):
+    if not content_type:
+        return ''
+    message = Message()
+    message['content-type'] = content_type
+    return message.get_content_type()
+
+
 class CrawlerUrl(object):
     def __init__(self, crawler, url, depth=3, source=None, exists=None, type=None, timeout=10):
         """
@@ -71,7 +79,7 @@ def start(self):
                         self.crawler.results.put(Error(self, e))
                         self.close()
                         return self
-                    content_type = cgi.parse_header(resp.headers.get('Content-Type', ''))[0]
+                    content_type = parse_content_type(resp.headers.get('Content-Type', ''))
                     soup = BeautifulSoup(text, 'html.parser') if content_type == 'text/html' else None
         except RequestException as e:
             self.crawler.current_processed_count += 1
@@ -112,7 +120,7 @@ def maybe_rewrite(self):
     def must_be_downloaded(self, response):
         """The file must be downloaded to obtain information.
         """
-        return self.maybe_directory() or (cgi.parse_header(response.headers.get('Content-Type', ''))[0] in [
+        return self.maybe_directory() or (parse_content_type(response.headers.get('Content-Type', '')) in [
             'text/css', 'application/javascript'
         ])
 
diff --git a/dirhunt/tests/test_crawler_url.py b/dirhunt/tests/test_crawler_url.py
index 0a64d0b..c6ce8e2 100644
--- a/dirhunt/tests/test_crawler_url.py
+++ b/dirhunt/tests/test_crawler_url.py
@@ -3,13 +3,17 @@
 import requests
 import requests_mock
 
-from dirhunt.crawler_url import CrawlerUrl
+from dirhunt.crawler_url import CrawlerUrl, parse_content_type
 from dirhunt.tests.base import CrawlerTestBase
 
 from dirhunt.tests._compat import patch, Mock
 
 
 class TestCrawlerUrl(CrawlerTestBase, unittest.TestCase):
+    def test_parse_content_type(self):
+        self.assertEqual(parse_content_type('text/html; charset=utf-8'), 'text/html')
+        self.assertEqual(parse_content_type(''), '')
+
     def test_start(self):
         crawler = self.get_crawler()
         crawler.closing = False