From cac2acb161e325b2f9398a958841a8b27d2f412d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 May 2023 21:34:21 +0000 Subject: [PATCH 1/5] Bump requests from 2.20.0 to 2.31.0 Bumps [requests](https://github.com/psf/requests) from 2.20.0 to 2.31.0. - [Release notes](https://github.com/psf/requests/releases) - [Changelog](https://github.com/psf/requests/blob/main/HISTORY.md) - [Commits](https://github.com/psf/requests/compare/v2.20.0...v2.31.0) --- updated-dependencies: - dependency-name: requests dependency-type: direct:production ... Signed-off-by: dependabot[bot] --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 53dc129b7..b76a01b84 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -requests==2.20.0 +requests==2.31.0 gunicorn==19.9.0 lxml==4.9.2 redis==3.5.3 From 68eea0c1124e6b2db39685a70a14c08e67d62e7a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=9B=8B=E9=BB=84edu?= Date: Mon, 18 May 2026 19:15:11 +0800 Subject: [PATCH 2/5] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E4=BB=A3=E7=90=86?= =?UTF-8?q?=E6=9D=A5=E6=BA=90=E5=B9=B6=E6=94=AF=E6=8C=81=E6=8C=89=E5=9C=B0?= =?UTF-8?q?=E5=8C=BA=E8=8E=B7=E5=8F=96=E4=BB=A3=E7=90=86?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 2 + api/proxyApi.py | 70 ++++++++++++++--- fetcher/proxyFetcher.py | 166 ++++++++++++++++++++++++++++++++++++++-- helper/check.py | 4 +- setting.py | 3 + util/six.py | 5 +- util/webRequest.py | 31 +++++++- 7 files changed, 258 insertions(+), 23 deletions(-) diff --git a/.gitignore b/.gitignore index e31375c1e..20c8c3627 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ .idea/ docs/_build *.pyc +*.pyc.* +__pycache__/ *.log .tox /SPEC.md diff --git a/api/proxyApi.py b/api/proxyApi.py index bd2de57e2..b6947a1cb 100644 --- a/api/proxyApi.py +++ b/api/proxyApi.py @@ -17,6 +17,7 @@ __author__ = 'JHao' import platform +from random import shuffle from werkzeug.wrappers import Response from flask import Flask, jsonify, request @@ -42,15 +43,41 @@ def force_type(cls, response, environ=None): app.response_class = JsonResponse api_list = [ - {"url": "/get", "params": "type: ''https'|''", "desc": "get a proxy"}, - {"url": "/pop", "params": "", "desc": "get and delete a proxy"}, + {"url": "/get", "params": "type: ''https'|'', count: '1', region: 'e.g. CN'", "desc": "get proxy"}, + {"url": "/pop", "params": "type: ''https'|'', region: 'e.g. CN'", "desc": "get and delete a proxy"}, {"url": "/delete", "params": "proxy: 'e.g. 127.0.0.1:8080'", "desc": "delete an unable proxy"}, - {"url": "/all", "params": "type: ''https'|''", "desc": "get all proxy from proxy pool"}, - {"url": "/count", "params": "", "desc": "return proxy count"} + {"url": "/all", "params": "type: ''https'|'', region: 'e.g. CN'", "desc": "get all proxy from proxy pool"}, + {"url": "/count", "params": "region: 'e.g. CN'", "desc": "return proxy count"} # 'refresh': 'refresh proxy pool', ] +def _get_https_arg(): + return request.args.get("type", "").lower() == 'https' + + +def _get_region_arg(): + return request.args.get("region", "").strip() + + +def _get_count_arg(default=1): + try: + return max(int(request.args.get("count", default)), 1) + except (TypeError, ValueError): + return default + + +def _filter_region(proxies, region): + if not region: + return proxies + region = region.lower() + return [proxy for proxy in proxies if str(proxy.region).lower() == region] + + +def _get_filtered_proxies(https=False, region=""): + return _filter_region(proxy_handler.getAll(https), region) + + @app.route('/') def index(): return {'url': api_list} @@ -58,15 +85,33 @@ def index(): @app.route('/get/') def get(): - https = request.args.get("type", "").lower() == 'https' - proxy = proxy_handler.get(https) - return proxy.to_dict if proxy else {"code": 0, "src": "no proxy"} + https = _get_https_arg() + region = _get_region_arg() + count = _get_count_arg() + if count == 1 and not region: + proxy = proxy_handler.get(https) + return proxy.to_dict if proxy else {"code": 0, "src": "no proxy"} + + proxies = _get_filtered_proxies(https, region) + if not proxies: + return {"code": 0, "src": "no proxy"} + shuffle(proxies) + result = [proxy.to_dict for proxy in proxies[:count]] + return result[0] if count == 1 else jsonify(result) @app.route('/pop/') def pop(): - https = request.args.get("type", "").lower() == 'https' - proxy = proxy_handler.pop(https) + https = _get_https_arg() + region = _get_region_arg() + if region: + proxies = _get_filtered_proxies(https, region) + shuffle(proxies) + proxy = proxies[0] if proxies else None + if proxy: + proxy_handler.delete(proxy) + else: + proxy = proxy_handler.pop(https) return proxy.to_dict if proxy else {"code": 0, "src": "no proxy"} @@ -78,8 +123,9 @@ def refresh(): @app.route('/all/') def getAll(): - https = request.args.get("type", "").lower() == 'https' - proxies = proxy_handler.getAll(https) + https = _get_https_arg() + region = _get_region_arg() + proxies = _get_filtered_proxies(https, region) return jsonify([_.to_dict for _ in proxies]) @@ -92,7 +138,7 @@ def delete(): @app.route('/count/') def getCount(): - proxies = proxy_handler.getAll() + proxies = _get_filtered_proxies(region=_get_region_arg()) http_type_dict = {} source_dict = {} for proxy in proxies: diff --git a/fetcher/proxyFetcher.py b/fetcher/proxyFetcher.py index 17a65bc2a..a46222101 100644 --- a/fetcher/proxyFetcher.py +++ b/fetcher/proxyFetcher.py @@ -16,6 +16,8 @@ import json from time import sleep +from lxml import etree + from util.webRequest import WebRequest @@ -24,12 +26,75 @@ class ProxyFetcher(object): proxy getter """ + @staticmethod + def _parse_proxies_from_text(text): + if not text: + return [] + proxy_pattern = re.compile(r'(?\s*?(\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3})\s*?(\d+)', r.text) - for proxy in proxies: - yield ":".join(proxy) + request = WebRequest() + ti_url = "https://ip.ihuan.me/ti.html" + tqdl_url = "https://ip.ihuan.me/tqdl.html" + ti_resp = request.get(ti_url, timeout=10, verify=False) + form_data = {} + if ti_resp.tree is not None: + for input_tag in ti_resp.tree.xpath("//form//input[@name]"): + name = "".join(input_tag.xpath("./@name")).strip() + value = "".join(input_tag.xpath("./@value")).strip() + if name: + form_data[name] = value + + key = form_data.get("key") + if not key: + key_match = re.search(r'name=["\']key["\'][^>]*value=["\']([^"\']+)', ti_resp.text) + if not key_match: + key_match = re.search(r'key["\']?\s*[:=]\s*["\']([0-9a-f]{16,})', ti_resp.text) + key = key_match.group(1) if key_match else "" + + if not key: + return + + header = { + "Origin": "https://ip.ihuan.me", + "Referer": ti_url, + } + data = form_data.copy() + data.update({ + "num": "2000", + "port": "", + "kill_port": "", + "address": "", + "kill_address": "", + "anonymity": "", + "type": "", + "post": "", + "sort": "1", + "key": key, + }) + r = request.post(tqdl_url, header=header, data=data, timeout=10, verify=False) + proxies = ProxyFetcher._parse_proxies_from_tree(r.tree) + proxies.extend(ProxyFetcher._parse_proxies_from_text(r.text)) + for proxy in ProxyFetcher._yield_unique_proxies(proxies): + yield proxy @staticmethod def freeProxy09(page_count=1): @@ -181,6 +284,55 @@ def freeProxy12(): if ip and port: yield "%s:%s" % (ip, port) + @staticmethod + def freeProxy13(): + """ FreeVPNNode 中国代理 https://cn.freevpnnode.com/free-proxy-for-china/ """ + # url = "https://cn.freevpnnode.com/free-proxy-for-china/" + url = "https://cn.freevpnnode.com/free-proxy/" + r = WebRequest().get(url, timeout=5, retry_time=1, verify=False) + proxies = ProxyFetcher._parse_proxies_from_tree(r.tree) + proxies.extend(ProxyFetcher._parse_proxies_from_text(r.text)) + for proxy in ProxyFetcher._yield_unique_proxies(proxies): + yield proxy + + @staticmethod + def freeProxy14(): + """ SCDN 代理接口 """ + # url = "https://proxy.scdn.io/get_proxies.php?protocol=&country=%E4%B8%AD%E5%9B%BD&per_page=100&page=1" + url = "https://proxy.scdn.io/get_proxies.php?protocol=&country=&per_page=100&page=1" + r = WebRequest().get(url, timeout=5, retry_time=1, verify=False) + try: + data = r.json + proxies = [] + table_html = data.get("table_html") if isinstance(data, dict) else "" + if table_html: + tree = etree.HTML("%s
" % table_html) + proxies.extend(ProxyFetcher._parse_proxies_from_tree(tree)) + + if not proxies: + proxies = ProxyFetcher._parse_proxies_from_json(data) + if not proxies: + proxies = ProxyFetcher._parse_proxies_from_text(r.text) + for proxy in ProxyFetcher._yield_unique_proxies(proxies): + yield proxy + except Exception as e: + print(e) + + @staticmethod + def freeProxy15(): + """ Geonode Free Proxy 中国代理 https://geonode.com/free-proxy-list/ """ + # url = "https://proxylist.geonode.com/api/proxy-list?limit=500&page=1&sort_by=lastChecked&sort_type=desc&country=CN" + url = "https://proxylist.geonode.com/api/proxy-list?limit=500&page=1&sort_by=lastChecked&sort_type=desc" + r = WebRequest().get(url, timeout=5, retry_time=1, verify=False) + try: + proxies = ProxyFetcher._parse_proxies_from_json(r.json) + if not proxies: + proxies = ProxyFetcher._parse_proxies_from_text(r.text) + for proxy in ProxyFetcher._yield_unique_proxies(proxies): + yield proxy + except Exception as e: + print(e) + # @staticmethod # def wallProxy01(): # """ diff --git a/helper/check.py b/helper/check.py index 937645c0f..0b732b84d 100644 --- a/helper/check.py +++ b/helper/check.py @@ -79,9 +79,9 @@ def preValidator(cls, proxy): @classmethod def regionGetter(cls, proxy): try: - url = 'https://searchplugin.csdn.net/api/v1/ip/get?ip=%s' % proxy.proxy.split(':')[0] + url = 'https://api.ip.sb/geoip/%s' % proxy.proxy.split(':')[0] r = WebRequest().get(url=url, retry_time=1, timeout=2).json - return r['data']['address'] + return r.get('country_code') except: return 'error' diff --git a/setting.py b/setting.py index f6ed89c8a..e8a669f58 100644 --- a/setting.py +++ b/setting.py @@ -57,6 +57,9 @@ "freeProxy10", "freeProxy11", "freeProxy12", + "freeProxy13", + "freeProxy14", + "freeProxy15", ] # ############# proxy validator ################# diff --git a/util/six.py b/util/six.py index 14ee059ba..d31e12138 100644 --- a/util/six.py +++ b/util/six.py @@ -30,7 +30,10 @@ def iteritems(d, **kw): from urlparse import urlparse if PY3: - from imp import reload as reload_six + try: + from importlib import reload as reload_six + except ImportError: + from imp import reload as reload_six else: reload_six = reload diff --git a/util/webRequest.py b/util/webRequest.py index bf0555216..97164773a 100644 --- a/util/webRequest.py +++ b/util/webRequest.py @@ -86,8 +86,38 @@ def get(self, url, header=None, retry_time=3, retry_interval=5, timeout=5, *args self.log.info("retry %s second after" % retry_interval) time.sleep(retry_interval) + def post(self, url, header=None, retry_time=3, retry_interval=5, timeout=5, *args, **kwargs): + """ + post method + :param url: target url + :param header: headers + :param retry_time: retry time + :param retry_interval: retry interval + :param timeout: network timeout + :return: + """ + headers = self.header + if header and isinstance(header, dict): + headers.update(header) + while True: + try: + self.response = requests.post(url, headers=headers, timeout=timeout, *args, **kwargs) + return self + except Exception as e: + self.log.error("requests: %s error: %s" % (url, str(e))) + retry_time -= 1 + if retry_time <= 0: + resp = Response() + resp.status_code = 200 + self.response = resp + return self + self.log.info("retry %s second after" % retry_interval) + time.sleep(retry_interval) + @property def tree(self): + if not self.response.content: + return None return etree.HTML(self.response.content) @property @@ -101,4 +131,3 @@ def json(self): except Exception as e: self.log.error(str(e)) return {} - From 13e43eb235ff8f0bd81e05022db580b8c25ed5f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=9B=8B=E9=BB=84edu?= Date: Mon, 18 May 2026 19:44:30 +0800 Subject: [PATCH 3/5] =?UTF-8?q?=E4=BF=AE=E5=A4=8D=E5=BD=92=E5=B1=9E?= =?UTF-8?q?=E5=9C=B0=E6=9F=A5=E8=AF=A2=E5=A4=B1=E6=95=88?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- helper/check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/helper/check.py b/helper/check.py index 0b732b84d..f382ece72 100644 --- a/helper/check.py +++ b/helper/check.py @@ -79,9 +79,9 @@ def preValidator(cls, proxy): @classmethod def regionGetter(cls, proxy): try: - url = 'https://api.ip.sb/geoip/%s' % proxy.proxy.split(':')[0] + url = 'https://opendata.baidu.com/api.php?query=%s&co=&resource_id=6006&oe=utf8' % proxy.proxy.split(':')[0] r = WebRequest().get(url=url, retry_time=1, timeout=2).json - return r.get('country_code') + return r.data[0].location except: return 'error' From 7715057fbc5724b62c1a646776008b32b54972a2 Mon Sep 17 00:00:00 2001 From: jhao104 Date: Wed, 20 May 2026 21:54:08 +0800 Subject: [PATCH 4/5] =?UTF-8?q?[update]=20=E6=9B=B4=E6=96=B0changelog?= =?UTF-8?q?=EF=BC=8C=E6=B7=BB=E5=8A=A0Next=20(unreleased)=E7=89=88?= =?UTF-8?q?=E6=9C=AC=E5=8F=98=E6=9B=B4=E8=AE=B0=E5=BD=95?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- docs/changelog.rst | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/changelog.rst b/docs/changelog.rst index e3889882c..558b9680a 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -3,6 +3,12 @@ ChangeLog ========== +Next (unreleased) +------------------ + +1. 新增代理源 **谷德代理**; (2026-05-14) +2. 引入tox自动化测试, 放弃Python 3.7以下版本支持; (2026-05-14) + 2.4.2 (2024-01-18) ------------------ From b141e638851a02c3d636934f792ceb90d437ecd7 Mon Sep 17 00:00:00 2001 From: jhao104 Date: Mon, 25 May 2026 21:40:25 +0800 Subject: [PATCH 5/5] =?UTF-8?q?refactor:=20=E5=9B=9E=E9=80=80region/count?= =?UTF-8?q?=E5=8F=82=E6=95=B0=EF=BC=8CIP=E6=9F=A5=E8=AF=A2=E6=94=B9?= =?UTF-8?q?=E7=94=A8ip.sb=E6=8E=A5=E5=8F=A3?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- api/proxyApi.py | 70 +++++++++---------------------------------------- helper/check.py | 4 +-- 2 files changed, 14 insertions(+), 60 deletions(-) diff --git a/api/proxyApi.py b/api/proxyApi.py index b6947a1cb..bd2de57e2 100644 --- a/api/proxyApi.py +++ b/api/proxyApi.py @@ -17,7 +17,6 @@ __author__ = 'JHao' import platform -from random import shuffle from werkzeug.wrappers import Response from flask import Flask, jsonify, request @@ -43,41 +42,15 @@ def force_type(cls, response, environ=None): app.response_class = JsonResponse api_list = [ - {"url": "/get", "params": "type: ''https'|'', count: '1', region: 'e.g. CN'", "desc": "get proxy"}, - {"url": "/pop", "params": "type: ''https'|'', region: 'e.g. CN'", "desc": "get and delete a proxy"}, + {"url": "/get", "params": "type: ''https'|''", "desc": "get a proxy"}, + {"url": "/pop", "params": "", "desc": "get and delete a proxy"}, {"url": "/delete", "params": "proxy: 'e.g. 127.0.0.1:8080'", "desc": "delete an unable proxy"}, - {"url": "/all", "params": "type: ''https'|'', region: 'e.g. CN'", "desc": "get all proxy from proxy pool"}, - {"url": "/count", "params": "region: 'e.g. CN'", "desc": "return proxy count"} + {"url": "/all", "params": "type: ''https'|''", "desc": "get all proxy from proxy pool"}, + {"url": "/count", "params": "", "desc": "return proxy count"} # 'refresh': 'refresh proxy pool', ] -def _get_https_arg(): - return request.args.get("type", "").lower() == 'https' - - -def _get_region_arg(): - return request.args.get("region", "").strip() - - -def _get_count_arg(default=1): - try: - return max(int(request.args.get("count", default)), 1) - except (TypeError, ValueError): - return default - - -def _filter_region(proxies, region): - if not region: - return proxies - region = region.lower() - return [proxy for proxy in proxies if str(proxy.region).lower() == region] - - -def _get_filtered_proxies(https=False, region=""): - return _filter_region(proxy_handler.getAll(https), region) - - @app.route('/') def index(): return {'url': api_list} @@ -85,33 +58,15 @@ def index(): @app.route('/get/') def get(): - https = _get_https_arg() - region = _get_region_arg() - count = _get_count_arg() - if count == 1 and not region: - proxy = proxy_handler.get(https) - return proxy.to_dict if proxy else {"code": 0, "src": "no proxy"} - - proxies = _get_filtered_proxies(https, region) - if not proxies: - return {"code": 0, "src": "no proxy"} - shuffle(proxies) - result = [proxy.to_dict for proxy in proxies[:count]] - return result[0] if count == 1 else jsonify(result) + https = request.args.get("type", "").lower() == 'https' + proxy = proxy_handler.get(https) + return proxy.to_dict if proxy else {"code": 0, "src": "no proxy"} @app.route('/pop/') def pop(): - https = _get_https_arg() - region = _get_region_arg() - if region: - proxies = _get_filtered_proxies(https, region) - shuffle(proxies) - proxy = proxies[0] if proxies else None - if proxy: - proxy_handler.delete(proxy) - else: - proxy = proxy_handler.pop(https) + https = request.args.get("type", "").lower() == 'https' + proxy = proxy_handler.pop(https) return proxy.to_dict if proxy else {"code": 0, "src": "no proxy"} @@ -123,9 +78,8 @@ def refresh(): @app.route('/all/') def getAll(): - https = _get_https_arg() - region = _get_region_arg() - proxies = _get_filtered_proxies(https, region) + https = request.args.get("type", "").lower() == 'https' + proxies = proxy_handler.getAll(https) return jsonify([_.to_dict for _ in proxies]) @@ -138,7 +92,7 @@ def delete(): @app.route('/count/') def getCount(): - proxies = _get_filtered_proxies(region=_get_region_arg()) + proxies = proxy_handler.getAll() http_type_dict = {} source_dict = {} for proxy in proxies: diff --git a/helper/check.py b/helper/check.py index f382ece72..0b732b84d 100644 --- a/helper/check.py +++ b/helper/check.py @@ -79,9 +79,9 @@ def preValidator(cls, proxy): @classmethod def regionGetter(cls, proxy): try: - url = 'https://opendata.baidu.com/api.php?query=%s&co=&resource_id=6006&oe=utf8' % proxy.proxy.split(':')[0] + url = 'https://api.ip.sb/geoip/%s' % proxy.proxy.split(':')[0] r = WebRequest().get(url=url, retry_time=1, timeout=2).json - return r.data[0].location + return r.get('country_code') except: return 'error'