Filter config for new auction

.travis.yml

@@ -1,9 +1,5 @@
 language: python
 sudo: required
-before_script:
-  - sudo add-apt-repository ppa:chris-lea/libsodium -y
-  - sudo apt-get -qq update
-  - sudo apt-get install libsodium13 -y
 python:
   - "2.7"
 services:
@@ -13,13 +9,13 @@ env:
   global:
     - TZ=Europe/Kiev
 before_install:
-  - pip install python-coveralls
+  - pip install python-coveralls pytest==3.2.3
   - python2 bootstrap.py
   - mv openprocurement/auction/worker/tests/data/auction_worker_travis.yaml openprocurement/auction/worker/tests/data/auction_worker_defaults.yaml
 install:
   - bin/buildout -N
   - curl -X PUT 0.0.0.0:5984/auctions
 script:
-  - bin/pytest
+  - bin/pytest openprocurement/auction/worker/tests/unit/
 after_success:
   - coveralls

README.md

@@ -1,5 +1,5 @@
-[![Build Status](https://travis-ci.org/openprocurement/openprocurement.auction.worker.svg?branch=master)](https://travis-ci.org/openprocurement/openprocurement.auction.worker)
-[![Coverage Status](https://coveralls.io/repos/github/openprocurement/openprocurement.auction.worker/badge.svg?branch=master)](https://coveralls.io/github/openprocurement/openprocurement.auction.worker?branch=master)
+[![Build Status](https://travis-ci.org/ProzorroUKR/openprocurement.auction.worker.svg?branch=master)](https://travis-ci.org/ProzorroUKR/openprocurement.auction.worker)
+[![Coverage Status](https://coveralls.io/repos/github/ProzorroUKR/openprocurement.auction.worker/badge.svg?branch=master)](https://coveralls.io/github/ProzorroUKR/openprocurement.auction.worker?branch=master)
 [![License](https://img.shields.io/badge/License-Apache%202.0-blue.svg)](https://opensource.org/licenses/Apache-2.0)
 
 Introduction

buildout.cfg

@@ -17,15 +17,19 @@ eggs =
     WTForms
     WTForms-JSON
 
-
 [versions]
 pbr = 1.8.0
 oslo.middleware = 2.8.0
 stevedore = 1.8.0
 oslo.i18n = 2.6.0
 oslo.context = 0.6.0
 oslo.config = 2.3.0
+Flask = 0.12.2
+coverage = 4.4.1
+pytest = 3.2.3
+pytest-cov = 2.5.1
+requests-oauthlib = 0.8.0
 
 [sources]
 chromedriver = git https://github.com/enkidulan/chromedriver.git
-openprocurement.auction = git https://github.com/openprocurement/openprocurement.auction.git branch=esco
+openprocurement.auction = git https://github.com/ProzorroUKR/openprocurement.auction.git branch=esco

openprocurement/auction/worker/auction.py

@@ -82,6 +82,7 @@ def __init__(self, tender_id,
             self._auction_data = auction_data
         else:
             self.debug = False
+
         self._end_auction_event = Event()
         self.bids_actions = BoundedSemaphore()
         self.session = RequestsSession()

openprocurement/auction/worker/deprecated_auction_config.json

@@ -0,0 +1,14 @@
+{
+  "procurementMethodType": {
+    "closeFrameworkAgreementUA": null,
+    "closeFrameworkAgreementSelectionUA": null,
+    "aboveThresholdEU": null,
+    "aboveThresholdUA": null,
+    "aboveThresholdUA.defense": null,
+    "competitiveDialogueEU.stage2":  null,
+    "competitiveDialogueUA.stage2":  null,
+    "esco": null,
+    "simple.defense": null,
+    "belowThreshold": null
+  }
+}

openprocurement/auction/worker/deprecated_auction_config_filter.py

@@ -0,0 +1,68 @@
+import os
+import json
+import operator
+import logging
+from dateutil.parser import parse
+
+
+LOGGER = logging.getLogger("Auction Worker")
+
+VALID_AUCTION_TYPES = ("new", "deprecated")
+
+
+def get_deprecated_auction_config_path():
+    return os.path.join(os.path.dirname(os.path.abspath(__file__)), "deprecated_auction_config.json")
+
+
+DEPRECATED_AUCTION_CONFIG_PATH = os.getenv(
+    "DEPRECATED_AUCTION_CONFIG_PATH", get_deprecated_auction_config_path()
+)
+
+with open(DEPRECATED_AUCTION_CONFIG_PATH) as _file:
+    CONFIG_DATA = json.load(_file)
+
+
+def is_tender_processed_by_auction(_tender, auction_type):
+    if auction_type not in VALID_AUCTION_TYPES:
+        raise ValueError("Auction type must be one of ".format(VALID_AUCTION_TYPES))
+
+    tender_period_start_date_str = _tender.get("tenderPeriod", {}).get("startDate", None)
+    if not tender_period_start_date_str:
+        LOGGER.error("There is no tenderPeriod startDate in tender {}".format(_tender["id"]))
+        tender_period_start_date_str = "2000-01-01T00:00:00+00:00"
+
+    tender_period_start_date = parse(tender_period_start_date_str)
+    _filters_statuses = []
+
+    for filter_key, filter_data in CONFIG_DATA.items():
+        _filters_statuses.append(
+            is_match_criteria(filter_data, _tender, filter_key, tender_period_start_date)
+        )
+
+    if auction_type == VALID_AUCTION_TYPES[0]:
+        return any(_filters_statuses)
+    else:
+        return all(map(operator.not_, _filters_statuses))
+
+
+def is_match_criteria(filter_data, _tender, _filter_key, tender_period_start_date):
+    try:
+        tender_field_value = _tender[_filter_key]
+    except KeyError:
+        LOGGER.error("There is no {} field in tender {}".format(_filter_key, _tender["id"]))
+        return False
+    else:
+        param_start_date = filter_data.get(tender_field_value, None)
+
+        try:
+            config_start_date = parse(param_start_date)
+        except ValueError:
+            LOGGER.error("Invalid Date string {} for {} filter key".format(param_start_date, _filter_key))
+            return False
+        except TypeError:
+            return False
+        else:
+            if tender_period_start_date >= config_start_date:
+                return True
+            else:
+                return False

openprocurement/auction/worker/forms.py

@@ -34,7 +34,9 @@ def validate_bid_change_on_bidding(form, field):
             raise ValidationError(u'Too high value')
     else:
         minimal_bid = form.document['stages'][stage_id]['amount']
-        if field.data > (minimal_bid - form.document['minimalStep']['amount']):
+        max_allowed = minimal_bid - form.document['minimalStep']['amount']
+        max_allowed = float(str(max_allowed))  # convert floats to more likely values, ex 0.19999999999999996 to 0.2
+        if field.data > max_allowed:
             raise ValidationError(u'Too high value')
 
 

openprocurement/auction/worker/includeme.py

@@ -27,3 +27,15 @@ def competitiveDialogueUA(components):
 
 def aboveThresholdUAdefense(components):
     _register(components, 'aboveThresholdUA.defense')
+
+
+def simpledefense(components):
+    _register(components, 'simple.defense')
+
+
+def closeFrameworkAgreementUA(components):
+    _register(components, 'closeFrameworkAgreementUA')
+
+
+def closeFrameworkAgreementSelectionUA(components):
+    _register(components, 'closeFrameworkAgreementSelectionUA')

openprocurement/auction/worker/mixins.py

@@ -34,6 +34,7 @@
     AUCTION_WORKER_SERVICE_START_STAGE,
     AUCTION_WORKER_SERVICE_START_NEXT_STAGE,
 )
+from openprocurement.auction.worker.deprecated_auction_config_filter import is_tender_processed_by_auction
 
 
 LOGGER = logging.getLogger("Auction Worker")
@@ -133,6 +134,13 @@ def prepare_auction_document(self):
             self.auction_document['test_auction_data'] = deepcopy(self._auction_data)
 
         self.get_auction_info(prepare=True)
+
+        if not is_tender_processed_by_auction(self._auction_data['data'], auction_type="deprecated"):
+            LOGGER.info('Skip tender {} as that tender work with new auctions'.format(
+                self._auction_data['data'].get('id')))
+
+            return
+
         if self.worker_defaults.get('sandbox_mode', False):
             submissionMethodDetails = self._auction_data['data'].get('submissionMethodDetails', '')
             if submissionMethodDetails == 'quick(mode:no-auction)':

openprocurement/auction/worker/server.py

@@ -1,4 +1,4 @@
-from flask_oauthlib.client import OAuth
+from flask_oauthlib.client import OAuth, OAuthException
 from flask import Flask, request, jsonify, url_for, session, abort, redirect
 import os
 from urlparse import urljoin
@@ -69,6 +69,11 @@ def log_request(self):
             log.write(self.format_request(), extra=extra)
 
 
+def return_oauth_exception(e):
+    app.logger.warning("Failed auth response {}".format(e))
+    return abort(503)
+
+
 @app.route('/login')
 def login():
     if 'bidder_id' in request.args and 'hash' in request.args:
@@ -82,11 +87,15 @@ def login():
                     )
                 else:
                     callback_url = url_for('authorized', next=next_url, _external=True)
-                response = app.remote_oauth.authorize(
-                    callback=callback_url,
-                    bidder_id=request.args['bidder_id'],
-                    hash=request.args['hash']
-                )
+
+                try:
+                    response = app.remote_oauth.authorize(
+                        callback=callback_url,
+                        bidder_id=request.args['bidder_id'],
+                        hash=request.args['hash']
+                    )
+                except OAuthException as e:
+                    return return_oauth_exception(e)
                 if 'return_url' in request.args:
                     session['return_url'] = request.args['return_url']
                 session['login_bidder_id'] = request.args['bidder_id']
@@ -100,7 +109,10 @@ def login():
 @app.route('/authorized')
 def authorized():
     if not('error' in request.args and request.args['error'] == 'access_denied'):
-        resp = app.remote_oauth.authorized_response()
+        try:
+            resp = app.remote_oauth.authorized_response()
+        except OAuthException as e:
+            return return_oauth_exception(e)
         if resp is None or hasattr(resp, 'data'):
             app.logger.info("Error Response from Oauth: {}".format(resp))
             return abort(403, 'Access denied')
@@ -138,12 +150,16 @@ def relogin():
         app.logger.info("Bidder {} with login_hash {} start re-login".format(
                         session['login_bidder_id'], session['login_hash'],
                         ), extra=prepare_extra_journal_fields(request.headers))
-        return app.remote_oauth.authorize(
-            callback=session['login_callback'],
-            bidder_id=session['login_bidder_id'],
-            hash=session['login_hash'],
-            auto_allow='1'
-        )
+        try:
+            resp = app.remote_oauth.authorize(
+                callback=session['login_callback'],
+                bidder_id=session['login_bidder_id'],
+                hash=session['login_hash'],
+                auto_allow='1'
+            )
+        except OAuthException as e:
+            return return_oauth_exception(e)
+        return resp
     return redirect(
         urljoin(request.headers['X-Forwarded-Path'], '.').rstrip('/')
     )

openprocurement/auction/worker/tests/unit/test_forms.py

@@ -188,6 +188,32 @@ def test_bids_form(auction, features_auction):
         'bids'
 
 
+def test_bids_form_float(auction):
+    from copy import deepcopy
+
+    # test values
+    bid = 1772091.11
+    step = 23062.86
+    prev_bid = 1795153.97
+
+    #  the problem and the solution
+    assert prev_bid - step == 1772091.1099999999  # not 1772091.11
+    assert str(prev_bid - step) == "1772091.11"
+    assert float(str(prev_bid - step)) == 1772091.11
+
+    # test that validation actually works as the example above
+    form_data = {
+        'bid': bid,
+        'bidder_id': 'f7c8cd1d56624477af8dc3aa9c4b3ea3',
+    }
+    form = BidsForm().from_json(form_data)
+    form.document = deepcopy(test_auction_document)
+    form.document["minimalStep"]["amount"] = step
+    form.document["stages"][-1]["amount"] = prev_bid
+    form.auction = auction
+    assert form.validate() is True
+
+
 def test_form_handler(app):
     app.application.form_handler = form_handler
     headers = {'Content-Type': 'application/json'}

openprocurement/auction/worker/tests/unit/test_server.py

@@ -3,9 +3,8 @@
 from datetime import datetime, timedelta
 from dateutil.tz import tzlocal
 from mock import MagicMock, patch
-from openprocurement.auction.worker.server import (
-    _LoggerStream
-)
+from openprocurement.auction.worker.server import _LoggerStream
+from flask_oauthlib.client import OAuthException
 
 
 def test_logger_stream_write():
@@ -71,6 +70,12 @@ def test_server_login(app):
         session['login_hash'] = u'bd4a790aac32b73e853c26424b032e5a29143d1f'
         session['login_callback'] = 'http://localhost/authorized'
 
+    app.application.remote_oauth.authorize.side_effect = OAuthException("Invalid response")
+    res = app.get('/login?bidder_id=5675acc9232942e8940a034994ad883e&'
+                  'hash=bd4a790aac32b73e853c26424b032e5a29143d1f',
+                  headers=headers)
+    assert res.status == "503 SERVICE UNAVAILABLE"
+
 
 def test_server_authorized(app):
     headers = {
@@ -110,6 +115,10 @@ def test_server_authorized(app):
     assert auctions_loggedin is True
     assert path is True
 
+    app.application.remote_oauth.authorized_response.side_effect = OAuthException("Invalid response")
+    res = app.get('/authorized', headers=headers)
+    assert res.status == "503 SERVICE UNAVAILABLE"
+
 
 def test_server_relogin(app):
     headers = {
@@ -139,6 +148,11 @@ def test_server_relogin(app):
     assert res.status == '302 FOUND'
     assert res.location == 'https://my.test.url'
 
+    app.application.remote_oauth.authorize.side_effect = OAuthException("Invalid response")
+    with patch('openprocurement.auction.worker.server.session', s):
+        res = app.get('/relogin', headers=headers)
+    assert res.status == "503 SERVICE UNAVAILABLE"
+
 
 def test_server_check_authorization(app):
 

setup.py

@@ -1,7 +1,7 @@
 from setuptools import setup, find_packages
 import os
 
-VERSION = '0.1.1'
+VERSION = '0.1.7dp'
 
 INSTALL_REQUIRES = [
     'setuptools',
@@ -27,6 +27,9 @@
         'competitiveDialogueEU.stage2 = openprocurement.auction.worker.includeme:competitiveDialogueEU',
         'competitiveDialogueUA.stage2 = openprocurement.auction.worker.includeme:competitiveDialogueUA',
         'aboveThresholdUA.defense = openprocurement.auction.worker.includeme:aboveThresholdUAdefense',
+        'simple.defense = openprocurement.auction.worker.includeme:simpledefense',
+        'closeFrameworkAgreementUA = openprocurement.auction.worker.includeme:closeFrameworkAgreementUA',
+        'closeFrameworkAgreementSelectionUA = openprocurement.auction.worker.includeme:closeFrameworkAgreementSelectionUA',
     ],
     'openprocurement.auction.robottests': [
         'auction_test = openprocurement.auction.worker.tests.functional.main:includeme'
@@ -52,6 +55,7 @@
       packages=find_packages(exclude=['ez_setup']),
       namespace_packages=['openprocurement', 'openprocurement.auction'],
       include_package_data=True,
+      package_data={'openprocurement.auction.worker': ['*.json']},
       zip_safe=False,
       install_requires=INSTALL_REQUIRES,
       extras_require=EXTRAS_REQUIRE,