Python3 db backend

.travis.yml

@@ -2,11 +2,11 @@ language: python
 python:
     - "2.7"
 before_install:
-    - sudo apt-get update
-    - sudo apt-get -o Dpkg::Options::="--force-confnew" -qq -y upgrade
+    - sudo apt-get update -qq
     - sudo apt-get install -qq protobuf-compiler
 services:
     - riak
+    - mysql
 install:
-    - pip install -q -e . --use-mirrors
+    - pip install -q -e .
 script: python setup.py nosetests

docs/source/conf.py

@@ -11,7 +11,10 @@
 # All configuration values have a default; values that are commented out
 # serve to show the default.
 
-import sys, os
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+import os
+import sys
 
 FILE_ROOT = os.path.abspath(os.path.dirname(__file__))
 #add the apps dir to the python path.
@@ -46,8 +49,8 @@
 master_doc = 'index'
 
 # General information about the project.
-project = u'Sunspear'
-copyright = u'2013, Numan Sachwani'
+project = 'Sunspear'
+copyright = '2013, Numan Sachwani'
 
 # The version info for the project you're documenting, acts as replacement for
 # |version| and |release|, also used in various other places throughout the
@@ -189,8 +192,8 @@
 # Grouping the document tree into LaTeX files. List of tuples
 # (source start file, target name, title, author, documentclass [howto/manual]).
 latex_documents = [
-  ('index', 'Sunspear.tex', u'Sunspear Documentation',
-   u'Numan Sachwani', 'manual'),
+  ('index', 'Sunspear.tex', 'Sunspear Documentation',
+   'Numan Sachwani', 'manual'),
 ]
 
 # The name of an image file (relative to this directory) to place at the top of
@@ -219,8 +222,8 @@
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [
-    ('index', 'sunspear', u'Sunspear Documentation',
-     [u'Numan Sachwani'], 1)
+    ('index', 'sunspear', 'Sunspear Documentation',
+     ['Numan Sachwani'], 1)
 ]
 
 # If true, show URL addresses after external links.
@@ -233,8 +236,8 @@
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-  ('index', 'Sunspear', u'Sunspear Documentation',
-   u'Numan Sachwani', 'Sunspear', 'One line description of project.',
+  ('index', 'Sunspear', 'Sunspear Documentation',
+   'Numan Sachwani', 'Sunspear', 'One line description of project.',
    'Miscellaneous'),
 ]
 

docs/source/userguide/introduction.rst

@@ -49,11 +49,11 @@ The main takeaway points are:
 .. note::
     For more info, see the specifications for `activity <http://activitystrea.ms/specs/json/1.0/#activity>`_ and `object <http://activitystrea.ms/specs/json/1.0/#object>`_.
 
-**Sunspear** also implements parts of some extensions to the specificiations. More specifically, `Audience Targeting <http://activitystrea.ms/specs/json/targeting/1.0/>`_ and `Responses <http://activitystrea.ms/specs/json/replies/1.0/>`_.
+**Sunspear** also implements parts of some extensions to the specification. More specifically, `Audience Targeting <http://activitystrea.ms/specs/json/targeting/1.0/>`_ and `Responses <http://activitystrea.ms/specs/json/replies/1.0/>`_.
 
 What it isn't
 --------------
 
 **Sunspear** strictly deals with storage and retrival of JSON activity stream items. It does not include all adquate indexes that allow you to build a fully fledged feed system.
 
-For indexing, you'll probably want to use something like `Sandsnake <https://github.com/numan/sandsnake>`_, a sorted index backed by `redis <http://redis.io>`_.
+For indexing, you'll probably want to use something like `Sandsnake <https://github.com/numan/sandsnake>`_, a sorted index backed by `redis <http://redis.io>`_.

setup.cfg

@@ -1,2 +1,23 @@
 [nosetests]
 where=tests
+
+[isort]
+line_length=120
+known_standard_library=
+known_third_party=
+known_first_party=sunspear
+balanced_wrapping=true
+combine_star=true
+# 0: grid
+# 1: vertical
+# 2: hanging
+# 3: vert-hanging
+# 4: vert-grid
+# 5: vert-grid-grouped
+multi_line_output=4
+not_skip=__init__.py
+
+# Don't sort one-letter classes (like Q) first
+order_by_type=false
+
+enforce_white_space=true

setup.py

@@ -1,6 +1,8 @@
 #!/usr/bin/python
 
-from setuptools import setup, find_packages
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+from setuptools import find_packages, setup
 
 tests_require=[
     'nose',
@@ -20,9 +22,13 @@
     packages=find_packages(exclude=['tests']),
     test_suite='nose.collector',
     install_requires=[
-        'riak==2.5.4',
-        'python-dateutil==1.5',
+        'python-dateutil>=1.5, != 2.0',
+        'riak',
+        'six',
         'protobuf==2.6.1',
+        'sqlalchemy',
+        'mysqlclient',
+        'six'
     ],
     options={'easy_install': {'allow_hosts': 'pypi.python.org'}},
     tests_require=tests_require,

sunspear/activitystreams/models.py

@@ -1,16 +1,19 @@
-from sunspear.exceptions import SunspearValidationException
+from __future__ import absolute_import, division, print_function, unicode_literals
 
-from sunspear.lib.rfc3339 import rfc3339
+import datetime
 
+import six
 from dateutil.parser import parse
 
-import datetime
+from sunspear.exceptions import SunspearValidationException
+from sunspear.lib.rfc3339 import rfc3339
 
 __all__ = ('Model', 'Activity', 'ReplyActivity', 'LikeActivity',
     'Object', 'MediaLink', )
 
 
 class Model(object):
+
     _required_fields = []
     _media_fields = []
     _reserved_fields = []
@@ -46,7 +49,7 @@ def validate(self):
         for field in self._reserved_fields:
             if self._dict.get(field, None) is not None\
                 and field not in ['updated', 'published']:
-                #updated and publised are special eceptions because if they are in reserved fields, the'll be overridden
+                # updated and publised are special eceptions because if they are in reserved fields, the'll be overridden
                 raise SunspearValidationException("Reserved field name used: %s" % field)
 
         for field in self._media_fields:
@@ -64,40 +67,45 @@ def validate(self):
                         Object(sub_obj, backend=self._backend).validate()
 
     def parse_data(self, data, *args, **kwargs):
-        #TODO Rename to jsonify_dict
+        # TODO Rename to jsonify_dict
         _parsed_data = data.copy()
 
-        #parse datetime fields
+        # parse datetime fields
         for d in self._datetime_fields:
             if d in _parsed_data and _parsed_data[d]:
                 _parsed_data[d] = self._parse_date(_parsed_data[d], utc=True, use_system_timezone=False)
 
-        #parse object fields
+        # parse object fields
         for c in self._object_fields:
             if c in _parsed_data and _parsed_data[c] and isinstance(_parsed_data[c], Model):
                 _parsed_data[c] = _parsed_data[c].parse_data(_parsed_data[c].get_dict())
 
-        #parse direct and indirect audience targeting
+        # parse direct and indirect audience targeting
         for c in self._indirect_audience_targeting_fields + self._direct_audience_targeting_fields:
             if c in _parsed_data and _parsed_data[c]:
-                _parsed_data[c] = [obj.parse_data(obj.get_dict()) if isinstance(obj, Model) else obj\
+                _parsed_data[c] = [obj.parse_data(obj.get_dict()) if isinstance(obj, Model) else obj
                     for obj in _parsed_data[c]]
 
-        #parse media fields
+        # parse media fields
         for c in self._media_fields:
             if c in _parsed_data and _parsed_data[c] and isinstance(_parsed_data[c], Model):
                 _parsed_data[c] = _parsed_data[c].parse_data(_parsed_data[c].get_dict())
 
-        #parse anything that is a dictionary for things like datetime fields that are datetime objects
-        for k, v in _parsed_data.items():
+        # parse anything that is a dictionary for things like datetime fields that are datetime objects
+        for k, v in list(_parsed_data.items()):
             if isinstance(v, dict) and k not in self._response_fields:
                 _parsed_data[k] = self.parse_data(v)
 
+        if 'id' in _parsed_data:
+            # we need to let the database take care of generating ids
+            # since there are size constraints that the sunspear convention violates,
+            # however to be safe we'll store the sunspear id as well.
+            _parsed_data['sunspear_id'] = _parsed_data['id']
+            del _parsed_data['id']
         return _parsed_data
 
     def get_parsed_dict(self, *args, **kwargs):
-
-        #we are suppose to maintain our own published and updated fields
+        # we are suppose to maintain our own published and updated fields
         if not self._dict.get('published', None):
             self._dict['published'] = datetime.datetime.utcnow()
         elif 'updated' in self._reserved_fields:
@@ -113,7 +121,7 @@ def get_dict(self):
     def _parse_date(self, date=None, utc=True, use_system_timezone=False):
         dt = None
         if date is None or not isinstance(date, datetime.datetime):
-            if isinstance(date, basestring):
+            if isinstance(date, six.string_types):
                 try:
                     dt = parse(date)
                 except ValueError:
@@ -138,6 +146,7 @@ def __getitem__(self, key):
 
 
 class Activity(Model):
+
     _required_fields = ['verb', 'actor', 'object']
     _media_fields = ['icon']
     _reserved_fields = ['updated']
@@ -156,9 +165,17 @@ def _set_defaults(self, model_dict):
 
         return model_dict
 
-    def get_parsed_sub_activity_dict(self, actor, content="", verb="reply", object_type="reply", \
-        collection="replies", activity_class=None, extra={}, published=None, **kwargs):
-        #TODO: Doesn't feel like this should be here Feels like it belongs in the backend.
+    def get_parsed_sub_activity_dict(self,
+                                     actor,
+                                     content="",
+                                     verb="reply",
+                                     object_type="reply",
+                                     collection="replies",
+                                     activity_class=None,
+                                     extra={},
+                                     published=None,
+                                     **kwargs):
+        # TODO: Doesn't feel like this should be here Feels like it belongs in the backend.
 
         if published is None:
             published = datetime.datetime.utcnow()
@@ -205,15 +222,15 @@ def get_parsed_sub_activity_dict(self, actor, content="", verb="reply", object_t
         }
 
         self._dict[collection]['totalItems'] += 1
-        #insert the newest comment at the top of the list
+        # insert the newest comment at the top of the list
         self._dict[collection]['items'].insert(0, _sub_dict)
 
         parent_activity = self.parse_data(self._dict, **kwargs)
 
         return _activity, parent_activity
 
     def parse_data(self, data, *args, **kwargs):
-        #TODO Rename to jsonify_dict
+        # TODO Rename to jsonify_dict
         _parsed_data = super(Activity, self).parse_data(data, *args, **kwargs)
         for response_field in self._response_fields:
             if response_field in _parsed_data:

sunspear/aggregators/base.py

@@ -1,3 +1,6 @@
+from __future__ import absolute_import, division, print_function, unicode_literals
+
+
 class BaseAggregator(object):
     def __init__(self, *args, **kwargs):
         pass

sunspear/aggregators/property.py

@@ -1,10 +1,11 @@
-from sunspear.aggregators.base import BaseAggregator
-from sunspear.lib.dotdict import dotdictify
-
-from itertools import groupby
+from __future__ import absolute_import, division, print_function, unicode_literals
 
 import copy
 import re
+from itertools import groupby
+
+from sunspear.aggregators.base import BaseAggregator
+from sunspear.lib.dotdict import dotdictify
 
 
 class PropertyAggregator(BaseAggregator):
@@ -48,12 +49,12 @@ def _listify_attributes(self, group_by_attributes=[], activity={}):
                     nested_root, rest = attr.split('.', 1)
                     #store a list of nested roots. We'll have to be careful not to listify these
                     nested_root_attributes.append(nested_root)
-                    for nested_dict_key, nested_dict_value in activity.get(nested_dict).items():
+                    for nested_dict_key, nested_dict_value in list(activity.get(nested_dict).items()):
                         if nested_dict_key != deepest_attr:
                             listified_dict['.'.join([nested_dict, nested_dict_key])] = [nested_dict_value]
 
         #now we listify all other non nested attributes
-        for key, val in activity.items():
+        for key, val in list(activity.items()):
             if key not in group_by_attributes and key not in nested_root_attributes:
                 listified_dict[key] = [val]
 
@@ -96,7 +97,7 @@ def _aggregate_activities(self, group_by_attributes=[], grouped_activities=[]):
                 #aggregate the rest of the activities into lists
                 for activity in group_list[1:]:
                     activity = dotdictify(activity)
-                    for key in aggregated_activity.keys():
+                    for key in list(aggregated_activity.keys()):
                         if key not in group_by_attributes and key not in nested_root_attributes:
                             aggregated_activity[key].append(activity.get(key))
 
@@ -107,7 +108,7 @@ def _aggregate_activities(self, group_by_attributes=[], grouped_activities=[]):
                             if nested_val is not None:
                                 nested_dict, deepest_attr = attr.rsplit('.', 1)
 
-                                for nested_dict_key, nested_dict_value in activity.get(nested_dict).items():
+                                for nested_dict_key, nested_dict_value in list(activity.get(nested_dict).items()):
                                     if nested_dict_key != deepest_attr:
                                         aggregated_activity['.'.join([nested_dict, nested_dict_key])].append(nested_dict_value)