Update with changes from monorepo

bugswarm/common/artifact_processing/runners.py

@@ -6,7 +6,7 @@
 from typing import List
 
 from . import utils as procutils
-from .. import log
+from bugswarm.common import log
 
 
 class ParallelArtifactRunner(object):

bugswarm/common/artifact_processing/utils.py

@@ -5,7 +5,7 @@
 import shutil
 
 from ..shell_wrapper import ShellWrapper
-from bugswarm.common.rest_api.database_api import DatabaseAPI
+from ..rest_api.database_api import DatabaseAPI
 
 REPOS_DIR = '/home/travis/build'
 _SANDBOX = 'bugswarm-sandbox'

bugswarm/common/credentials.py

@@ -1,3 +1,9 @@
 # DockerHub
 DOCKER_HUB_USERNAME = 'bugswarm'
-DOCKER_HUB_REPO = '{}/{}'.format(DOCKER_HUB_USERNAME, 'images')
+DOCKER_HUB_REPO = 'bugswarm/images'
+DOCKER_HUB_CACHED_REPO = 'bugswarm/cached-images'
+
+COMMON_HOSTNAME = 'www.api.bugswarm.org'
+
+# Travis
+TRAVIS_TOKENS = []

bugswarm/common/decorators/classproperty.py

@@ -4,11 +4,13 @@
 These decorators are adapted from answers to the Stack Overflow question at https://stackoverflow.com/q/3203286.
 """
 
+
 class classproperty:
     """
     Same as property(), but passes obj.__class__ instead of obj to fget/fset/fdel.
     Original code for property emulation at https://docs.python.org/3.5/howto/descriptor.html#properties.
     """
+
     def __init__(self, fget=None, fset=None, fdel=None, doc=None):
         self.fget = fget
         self.fset = fset

bugswarm/common/filter_reasons.py

@@ -0,0 +1,19 @@
+"""
+String constants for each reason that Pair Filter filters a pair. Also used in the evaluation script.
+
+Think carefully before changing these constants! Any changes will create inconsistency in the 'filtered_reason'
+attribute in the metadata database. In turn, changes will break the evaluation script. In short, if these constants
+change, then to remove inconsistencies between previously created artifacts and newly created artifacts, either every
+artifact will need to be recreated or the metadata database will need to be updated so that previously created artifacts
+use the new constants in their metadata.
+"""
+
+NO_HEAD_SHA = 'no head sha'
+NO_ORIGINAL_LOG = 'do not have original log'
+ERROR_READING_ORIGINAL_LOG = 'error when reading original log'
+NO_IMAGE_PROVISION_TIMESTAMP = 'original log does not have provisioned datetime string'
+INACCESSIBLE_IMAGE = 'do not have image'
+NOT_RESETTABLE = 'not resettable'  # Deprecated.
+NOT_ACQUIRABLE = 'not acquirable'  # Deprecated.
+NOT_AVAILABLE = 'not available'
+SAME_COMMIT_PAIR = 'failed build has same sha with passed build'

bugswarm/common/github_wrapper.py

@@ -10,7 +10,7 @@
 import requests
 import copy
 
-from . import log
+from bugswarm.common import log
 
 
 class GitHubWrapper(object):
@@ -71,6 +71,9 @@ def get(self, url: str):
                 elif response.status_code == 451:  # Repository access blocked.
                     log.error('Repository access blocked:', url)
                     return None, None
+                elif response.status_code == 401:  # Not authorized.
+                    log.error('Invalid GitHub API token: ', self._session.headers['Authorization'])
+                    return None, None
                 elif response.status_code == 422:
                     return None, None
                 else:

bugswarm/common/log_downloader.py

@@ -8,7 +8,7 @@
 from typing import Union
 from urllib.error import URLError
 
-from . import log
+from bugswarm.common import log
 
 _DEFAULT_RETRIES = 3
 
@@ -40,14 +40,15 @@ def download_log(job_id: Union[str, int],
         raise FileExistsError
 
     job_id = str(job_id)
-
-    aws_log_link = 'https://s3.amazonaws.com/archive.travis-ci.org/jobs/{}/log.txt'.format(job_id)
     travis_log_link = 'https://api.travis-ci.org/jobs/{}/log.txt'.format(job_id)
-
-    content = _get_log_from_url(aws_log_link, retries) or _get_log_from_url(travis_log_link, retries)
+    content = _get_log_from_url(travis_log_link, retries)
 
     if not content:
-        return False
+        travis_log_link = 'https://api.travis-ci.com/v3/job/{}/log.txt'.format(job_id)
+        content = _get_log_from_url(travis_log_link, retries)
+        # If this endpoint fails, the log is not on either endpoint and does not exist
+        if not content:
+            return False
 
     with open(destination, 'wb') as f:
         f.write(content)
@@ -111,12 +112,12 @@ def _get_log_from_url(log_url: str, max_retries: int, retry_count: int = 0):
             result = url.read()
             log.info('Downloaded log from {}.'.format(log_url))
             return result
-    except URLError:
-        log.info('Could not download log from {}.'.format(log_url))
+    except URLError as e:
+        log.error('Could not download log from {}.'.format(log_url, e.reason))
         return None
     except ConnectionResetError:
         if retry_count == max_retries:
-            log.info('Could not download log from', log_url, 'after retrying', max_retries, 'times.')
+            log.warning('Could not download log from', log_url, 'after retrying', max_retries, 'times.')
             return None
         log.warning('The server reset the connection. Retrying after', sleep_duration, 'seconds.')
         time.sleep(sleep_duration)

bugswarm/common/outdated.py

@@ -6,7 +6,7 @@
 
 import requests
 
-from . import log
+from bugswarm.common import log
 from .shell_wrapper import ShellWrapper