diff --git a/.travis.yml b/.travis.yml index 1a35ab9..6845da9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,5 @@ language: python sudo: required -before_script: - - sudo add-apt-repository ppa:chris-lea/libsodium -y - - sudo apt-get -qq update - - sudo apt-get install libsodium13 -y python: - "2.7" services: @@ -13,13 +9,13 @@ env: global: - TZ=Europe/Kiev before_install: - - pip install python-coveralls + - pip install python-coveralls pytest==3.2.3 - python2 bootstrap.py - mv openprocurement/auction/worker/tests/data/auction_worker_travis.yaml openprocurement/auction/worker/tests/data/auction_worker_defaults.yaml install: - bin/buildout -N - curl -X PUT 0.0.0.0:5984/auctions script: - - bin/pytest + - bin/pytest openprocurement/auction/worker/tests/unit/ after_success: - coveralls diff --git a/README.md b/README.md index d48a53f..f52edc2 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ -[![Build Status](https://travis-ci.org/openprocurement/openprocurement.auction.worker.svg?branch=master)](https://travis-ci.org/openprocurement/openprocurement.auction.worker) -[![Coverage Status](https://coveralls.io/repos/github/openprocurement/openprocurement.auction.worker/badge.svg?branch=master)](https://coveralls.io/github/openprocurement/openprocurement.auction.worker?branch=master) +[![Build Status](https://travis-ci.org/ProzorroUKR/openprocurement.auction.worker.svg?branch=master)](https://travis-ci.org/ProzorroUKR/openprocurement.auction.worker) +[![Coverage Status](https://coveralls.io/repos/github/ProzorroUKR/openprocurement.auction.worker/badge.svg?branch=master)](https://coveralls.io/github/ProzorroUKR/openprocurement.auction.worker?branch=master) [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0) Introduction diff --git a/buildout.cfg b/buildout.cfg index 20134c1..3b3fdc8 100644 --- a/buildout.cfg +++ b/buildout.cfg @@ -17,7 +17,6 @@ eggs = WTForms WTForms-JSON - [versions] pbr = 1.8.0 oslo.middleware = 2.8.0 @@ -25,7 +24,12 @@ stevedore = 1.8.0 oslo.i18n = 2.6.0 oslo.context = 0.6.0 oslo.config = 2.3.0 +Flask = 0.12.2 +coverage = 4.4.1 +pytest = 3.2.3 +pytest-cov = 2.5.1 +requests-oauthlib = 0.8.0 [sources] chromedriver = git https://github.com/enkidulan/chromedriver.git -openprocurement.auction = git https://github.com/openprocurement/openprocurement.auction.git branch=esco +openprocurement.auction = git https://github.com/ProzorroUKR/openprocurement.auction.git branch=esco diff --git a/openprocurement/auction/worker/auction.py b/openprocurement/auction/worker/auction.py index 36fc82b..acffcc1 100644 --- a/openprocurement/auction/worker/auction.py +++ b/openprocurement/auction/worker/auction.py @@ -82,6 +82,7 @@ def __init__(self, tender_id, self._auction_data = auction_data else: self.debug = False + self._end_auction_event = Event() self.bids_actions = BoundedSemaphore() self.session = RequestsSession() diff --git a/openprocurement/auction/worker/deprecated_auction_config.json b/openprocurement/auction/worker/deprecated_auction_config.json new file mode 100644 index 0000000..5c58448 --- /dev/null +++ b/openprocurement/auction/worker/deprecated_auction_config.json @@ -0,0 +1,14 @@ +{ + "procurementMethodType": { + "closeFrameworkAgreementUA": null, + "closeFrameworkAgreementSelectionUA": null, + "aboveThresholdEU": null, + "aboveThresholdUA": null, + "aboveThresholdUA.defense": null, + "competitiveDialogueEU.stage2": null, + "competitiveDialogueUA.stage2": null, + "esco": null, + "simple.defense": null, + "belowThreshold": null + } +} \ No newline at end of file diff --git a/openprocurement/auction/worker/deprecated_auction_config_filter.py b/openprocurement/auction/worker/deprecated_auction_config_filter.py new file mode 100644 index 0000000..a6b8a69 --- /dev/null +++ b/openprocurement/auction/worker/deprecated_auction_config_filter.py @@ -0,0 +1,68 @@ +import os +import json +import operator +import logging +from dateutil.parser import parse + + +LOGGER = logging.getLogger("Auction Worker") + +VALID_AUCTION_TYPES = ("new", "deprecated") + + +def get_deprecated_auction_config_path(): + return os.path.join(os.path.dirname(os.path.abspath(__file__)), "deprecated_auction_config.json") + + +DEPRECATED_AUCTION_CONFIG_PATH = os.getenv( + "DEPRECATED_AUCTION_CONFIG_PATH", get_deprecated_auction_config_path() +) + +with open(DEPRECATED_AUCTION_CONFIG_PATH) as _file: + CONFIG_DATA = json.load(_file) + + +def is_tender_processed_by_auction(_tender, auction_type): + if auction_type not in VALID_AUCTION_TYPES: + raise ValueError("Auction type must be one of ".format(VALID_AUCTION_TYPES)) + + tender_period_start_date_str = _tender.get("tenderPeriod", {}).get("startDate", None) + if not tender_period_start_date_str: + LOGGER.error("There is no tenderPeriod startDate in tender {}".format(_tender["id"])) + tender_period_start_date_str = "2000-01-01T00:00:00+00:00" + + tender_period_start_date = parse(tender_period_start_date_str) + _filters_statuses = [] + + for filter_key, filter_data in CONFIG_DATA.items(): + _filters_statuses.append( + is_match_criteria(filter_data, _tender, filter_key, tender_period_start_date) + ) + + if auction_type == VALID_AUCTION_TYPES[0]: + return any(_filters_statuses) + else: + return all(map(operator.not_, _filters_statuses)) + + +def is_match_criteria(filter_data, _tender, _filter_key, tender_period_start_date): + try: + tender_field_value = _tender[_filter_key] + except KeyError: + LOGGER.error("There is no {} field in tender {}".format(_filter_key, _tender["id"])) + return False + else: + param_start_date = filter_data.get(tender_field_value, None) + + try: + config_start_date = parse(param_start_date) + except ValueError: + LOGGER.error("Invalid Date string {} for {} filter key".format(param_start_date, _filter_key)) + return False + except TypeError: + return False + else: + if tender_period_start_date >= config_start_date: + return True + else: + return False diff --git a/openprocurement/auction/worker/forms.py b/openprocurement/auction/worker/forms.py index ca936ff..d47091d 100644 --- a/openprocurement/auction/worker/forms.py +++ b/openprocurement/auction/worker/forms.py @@ -34,7 +34,9 @@ def validate_bid_change_on_bidding(form, field): raise ValidationError(u'Too high value') else: minimal_bid = form.document['stages'][stage_id]['amount'] - if field.data > (minimal_bid - form.document['minimalStep']['amount']): + max_allowed = minimal_bid - form.document['minimalStep']['amount'] + max_allowed = float(str(max_allowed)) # convert floats to more likely values, ex 0.19999999999999996 to 0.2 + if field.data > max_allowed: raise ValidationError(u'Too high value') diff --git a/openprocurement/auction/worker/includeme.py b/openprocurement/auction/worker/includeme.py index ff38d98..4bdaa90 100644 --- a/openprocurement/auction/worker/includeme.py +++ b/openprocurement/auction/worker/includeme.py @@ -27,3 +27,15 @@ def competitiveDialogueUA(components): def aboveThresholdUAdefense(components): _register(components, 'aboveThresholdUA.defense') + + +def simpledefense(components): + _register(components, 'simple.defense') + + +def closeFrameworkAgreementUA(components): + _register(components, 'closeFrameworkAgreementUA') + + +def closeFrameworkAgreementSelectionUA(components): + _register(components, 'closeFrameworkAgreementSelectionUA') diff --git a/openprocurement/auction/worker/mixins.py b/openprocurement/auction/worker/mixins.py index 2f000be..cc582fe 100644 --- a/openprocurement/auction/worker/mixins.py +++ b/openprocurement/auction/worker/mixins.py @@ -34,6 +34,7 @@ AUCTION_WORKER_SERVICE_START_STAGE, AUCTION_WORKER_SERVICE_START_NEXT_STAGE, ) +from openprocurement.auction.worker.deprecated_auction_config_filter import is_tender_processed_by_auction LOGGER = logging.getLogger("Auction Worker") @@ -133,6 +134,13 @@ def prepare_auction_document(self): self.auction_document['test_auction_data'] = deepcopy(self._auction_data) self.get_auction_info(prepare=True) + + if not is_tender_processed_by_auction(self._auction_data['data'], auction_type="deprecated"): + LOGGER.info('Skip tender {} as that tender work with new auctions'.format( + self._auction_data['data'].get('id'))) + + return + if self.worker_defaults.get('sandbox_mode', False): submissionMethodDetails = self._auction_data['data'].get('submissionMethodDetails', '') if submissionMethodDetails == 'quick(mode:no-auction)': diff --git a/openprocurement/auction/worker/server.py b/openprocurement/auction/worker/server.py index ffc45c0..2b6ad9b 100644 --- a/openprocurement/auction/worker/server.py +++ b/openprocurement/auction/worker/server.py @@ -1,4 +1,4 @@ -from flask_oauthlib.client import OAuth +from flask_oauthlib.client import OAuth, OAuthException from flask import Flask, request, jsonify, url_for, session, abort, redirect import os from urlparse import urljoin @@ -69,6 +69,11 @@ def log_request(self): log.write(self.format_request(), extra=extra) +def return_oauth_exception(e): + app.logger.warning("Failed auth response {}".format(e)) + return abort(503) + + @app.route('/login') def login(): if 'bidder_id' in request.args and 'hash' in request.args: @@ -82,11 +87,15 @@ def login(): ) else: callback_url = url_for('authorized', next=next_url, _external=True) - response = app.remote_oauth.authorize( - callback=callback_url, - bidder_id=request.args['bidder_id'], - hash=request.args['hash'] - ) + + try: + response = app.remote_oauth.authorize( + callback=callback_url, + bidder_id=request.args['bidder_id'], + hash=request.args['hash'] + ) + except OAuthException as e: + return return_oauth_exception(e) if 'return_url' in request.args: session['return_url'] = request.args['return_url'] session['login_bidder_id'] = request.args['bidder_id'] @@ -100,7 +109,10 @@ def login(): @app.route('/authorized') def authorized(): if not('error' in request.args and request.args['error'] == 'access_denied'): - resp = app.remote_oauth.authorized_response() + try: + resp = app.remote_oauth.authorized_response() + except OAuthException as e: + return return_oauth_exception(e) if resp is None or hasattr(resp, 'data'): app.logger.info("Error Response from Oauth: {}".format(resp)) return abort(403, 'Access denied') @@ -138,12 +150,16 @@ def relogin(): app.logger.info("Bidder {} with login_hash {} start re-login".format( session['login_bidder_id'], session['login_hash'], ), extra=prepare_extra_journal_fields(request.headers)) - return app.remote_oauth.authorize( - callback=session['login_callback'], - bidder_id=session['login_bidder_id'], - hash=session['login_hash'], - auto_allow='1' - ) + try: + resp = app.remote_oauth.authorize( + callback=session['login_callback'], + bidder_id=session['login_bidder_id'], + hash=session['login_hash'], + auto_allow='1' + ) + except OAuthException as e: + return return_oauth_exception(e) + return resp return redirect( urljoin(request.headers['X-Forwarded-Path'], '.').rstrip('/') ) diff --git a/openprocurement/auction/worker/tests/unit/test_forms.py b/openprocurement/auction/worker/tests/unit/test_forms.py index 05c15ab..e8a61d3 100644 --- a/openprocurement/auction/worker/tests/unit/test_forms.py +++ b/openprocurement/auction/worker/tests/unit/test_forms.py @@ -188,6 +188,32 @@ def test_bids_form(auction, features_auction): 'bids' +def test_bids_form_float(auction): + from copy import deepcopy + + # test values + bid = 1772091.11 + step = 23062.86 + prev_bid = 1795153.97 + + # the problem and the solution + assert prev_bid - step == 1772091.1099999999 # not 1772091.11 + assert str(prev_bid - step) == "1772091.11" + assert float(str(prev_bid - step)) == 1772091.11 + + # test that validation actually works as the example above + form_data = { + 'bid': bid, + 'bidder_id': 'f7c8cd1d56624477af8dc3aa9c4b3ea3', + } + form = BidsForm().from_json(form_data) + form.document = deepcopy(test_auction_document) + form.document["minimalStep"]["amount"] = step + form.document["stages"][-1]["amount"] = prev_bid + form.auction = auction + assert form.validate() is True + + def test_form_handler(app): app.application.form_handler = form_handler headers = {'Content-Type': 'application/json'} diff --git a/openprocurement/auction/worker/tests/unit/test_server.py b/openprocurement/auction/worker/tests/unit/test_server.py index 40ac226..92779ca 100644 --- a/openprocurement/auction/worker/tests/unit/test_server.py +++ b/openprocurement/auction/worker/tests/unit/test_server.py @@ -3,9 +3,8 @@ from datetime import datetime, timedelta from dateutil.tz import tzlocal from mock import MagicMock, patch -from openprocurement.auction.worker.server import ( - _LoggerStream -) +from openprocurement.auction.worker.server import _LoggerStream +from flask_oauthlib.client import OAuthException def test_logger_stream_write(): @@ -71,6 +70,12 @@ def test_server_login(app): session['login_hash'] = u'bd4a790aac32b73e853c26424b032e5a29143d1f' session['login_callback'] = 'http://localhost/authorized' + app.application.remote_oauth.authorize.side_effect = OAuthException("Invalid response") + res = app.get('/login?bidder_id=5675acc9232942e8940a034994ad883e&' + 'hash=bd4a790aac32b73e853c26424b032e5a29143d1f', + headers=headers) + assert res.status == "503 SERVICE UNAVAILABLE" + def test_server_authorized(app): headers = { @@ -110,6 +115,10 @@ def test_server_authorized(app): assert auctions_loggedin is True assert path is True + app.application.remote_oauth.authorized_response.side_effect = OAuthException("Invalid response") + res = app.get('/authorized', headers=headers) + assert res.status == "503 SERVICE UNAVAILABLE" + def test_server_relogin(app): headers = { @@ -139,6 +148,11 @@ def test_server_relogin(app): assert res.status == '302 FOUND' assert res.location == 'https://my.test.url' + app.application.remote_oauth.authorize.side_effect = OAuthException("Invalid response") + with patch('openprocurement.auction.worker.server.session', s): + res = app.get('/relogin', headers=headers) + assert res.status == "503 SERVICE UNAVAILABLE" + def test_server_check_authorization(app): diff --git a/setup.py b/setup.py index f61df3f..bfcd969 100644 --- a/setup.py +++ b/setup.py @@ -1,7 +1,7 @@ from setuptools import setup, find_packages import os -VERSION = '0.1.1' +VERSION = '0.1.7dp' INSTALL_REQUIRES = [ 'setuptools', @@ -27,6 +27,9 @@ 'competitiveDialogueEU.stage2 = openprocurement.auction.worker.includeme:competitiveDialogueEU', 'competitiveDialogueUA.stage2 = openprocurement.auction.worker.includeme:competitiveDialogueUA', 'aboveThresholdUA.defense = openprocurement.auction.worker.includeme:aboveThresholdUAdefense', + 'simple.defense = openprocurement.auction.worker.includeme:simpledefense', + 'closeFrameworkAgreementUA = openprocurement.auction.worker.includeme:closeFrameworkAgreementUA', + 'closeFrameworkAgreementSelectionUA = openprocurement.auction.worker.includeme:closeFrameworkAgreementSelectionUA', ], 'openprocurement.auction.robottests': [ 'auction_test = openprocurement.auction.worker.tests.functional.main:includeme' @@ -52,6 +55,7 @@ packages=find_packages(exclude=['ez_setup']), namespace_packages=['openprocurement', 'openprocurement.auction'], include_package_data=True, + package_data={'openprocurement.auction.worker': ['*.json']}, zip_safe=False, install_requires=INSTALL_REQUIRES, extras_require=EXTRAS_REQUIRE,