diff --git a/.travis.yml b/.travis.yml index 6006c82..76deffc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,11 +2,11 @@ language: python python: - "2.7" before_install: - - sudo apt-get update - - sudo apt-get -o Dpkg::Options::="--force-confnew" -qq -y upgrade + - sudo apt-get update -qq - sudo apt-get install -qq protobuf-compiler services: - riak + - mysql install: - - pip install -q -e . --use-mirrors + - pip install -q -e . script: python setup.py nosetests diff --git a/docs/source/conf.py b/docs/source/conf.py index b435b95..96c6570 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -11,7 +11,10 @@ # All configuration values have a default; values that are commented out # serve to show the default. -import sys, os +from __future__ import absolute_import, division, print_function, unicode_literals + +import os +import sys FILE_ROOT = os.path.abspath(os.path.dirname(__file__)) #add the apps dir to the python path. @@ -46,8 +49,8 @@ master_doc = 'index' # General information about the project. -project = u'Sunspear' -copyright = u'2013, Numan Sachwani' +project = 'Sunspear' +copyright = '2013, Numan Sachwani' # The version info for the project you're documenting, acts as replacement for # |version| and |release|, also used in various other places throughout the @@ -189,8 +192,8 @@ # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'Sunspear.tex', u'Sunspear Documentation', - u'Numan Sachwani', 'manual'), + ('index', 'Sunspear.tex', 'Sunspear Documentation', + 'Numan Sachwani', 'manual'), ] # The name of an image file (relative to this directory) to place at the top of @@ -219,8 +222,8 @@ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ('index', 'sunspear', u'Sunspear Documentation', - [u'Numan Sachwani'], 1) + ('index', 'sunspear', 'Sunspear Documentation', + ['Numan Sachwani'], 1) ] # If true, show URL addresses after external links. @@ -233,8 +236,8 @@ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'Sunspear', u'Sunspear Documentation', - u'Numan Sachwani', 'Sunspear', 'One line description of project.', + ('index', 'Sunspear', 'Sunspear Documentation', + 'Numan Sachwani', 'Sunspear', 'One line description of project.', 'Miscellaneous'), ] diff --git a/docs/source/userguide/introduction.rst b/docs/source/userguide/introduction.rst index 67cb03a..7f47158 100644 --- a/docs/source/userguide/introduction.rst +++ b/docs/source/userguide/introduction.rst @@ -49,11 +49,11 @@ The main takeaway points are: .. note:: For more info, see the specifications for `activity `_ and `object `_. -**Sunspear** also implements parts of some extensions to the specificiations. More specifically, `Audience Targeting `_ and `Responses `_. +**Sunspear** also implements parts of some extensions to the specification. More specifically, `Audience Targeting `_ and `Responses `_. What it isn't -------------- **Sunspear** strictly deals with storage and retrival of JSON activity stream items. It does not include all adquate indexes that allow you to build a fully fledged feed system. -For indexing, you'll probably want to use something like `Sandsnake `_, a sorted index backed by `redis `_. \ No newline at end of file +For indexing, you'll probably want to use something like `Sandsnake `_, a sorted index backed by `redis `_. diff --git a/setup.cfg b/setup.cfg index ce26f6a..b6c1cfb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,2 +1,23 @@ [nosetests] where=tests + +[isort] +line_length=120 +known_standard_library= +known_third_party= +known_first_party=sunspear +balanced_wrapping=true +combine_star=true +# 0: grid +# 1: vertical +# 2: hanging +# 3: vert-hanging +# 4: vert-grid +# 5: vert-grid-grouped +multi_line_output=4 +not_skip=__init__.py + +# Don't sort one-letter classes (like Q) first +order_by_type=false + +enforce_white_space=true diff --git a/setup.py b/setup.py index ad645f1..037a2ab 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,8 @@ #!/usr/bin/python -from setuptools import setup, find_packages +from __future__ import absolute_import, division, print_function, unicode_literals + +from setuptools import find_packages, setup tests_require=[ 'nose', @@ -20,9 +22,13 @@ packages=find_packages(exclude=['tests']), test_suite='nose.collector', install_requires=[ - 'riak==2.5.4', - 'python-dateutil==1.5', + 'python-dateutil>=1.5, != 2.0', + 'riak', + 'six', 'protobuf==2.6.1', + 'sqlalchemy', + 'mysqlclient', + 'six' ], options={'easy_install': {'allow_hosts': 'pypi.python.org'}}, tests_require=tests_require, diff --git a/sunspear/activitystreams/models.py b/sunspear/activitystreams/models.py index 02ca5e4..8c733cc 100644 --- a/sunspear/activitystreams/models.py +++ b/sunspear/activitystreams/models.py @@ -1,16 +1,19 @@ -from sunspear.exceptions import SunspearValidationException +from __future__ import absolute_import, division, print_function, unicode_literals -from sunspear.lib.rfc3339 import rfc3339 +import datetime +import six from dateutil.parser import parse -import datetime +from sunspear.exceptions import SunspearValidationException +from sunspear.lib.rfc3339 import rfc3339 __all__ = ('Model', 'Activity', 'ReplyActivity', 'LikeActivity', 'Object', 'MediaLink', ) class Model(object): + _required_fields = [] _media_fields = [] _reserved_fields = [] @@ -46,7 +49,7 @@ def validate(self): for field in self._reserved_fields: if self._dict.get(field, None) is not None\ and field not in ['updated', 'published']: - #updated and publised are special eceptions because if they are in reserved fields, the'll be overridden + # updated and publised are special eceptions because if they are in reserved fields, the'll be overridden raise SunspearValidationException("Reserved field name used: %s" % field) for field in self._media_fields: @@ -64,40 +67,45 @@ def validate(self): Object(sub_obj, backend=self._backend).validate() def parse_data(self, data, *args, **kwargs): - #TODO Rename to jsonify_dict + # TODO Rename to jsonify_dict _parsed_data = data.copy() - #parse datetime fields + # parse datetime fields for d in self._datetime_fields: if d in _parsed_data and _parsed_data[d]: _parsed_data[d] = self._parse_date(_parsed_data[d], utc=True, use_system_timezone=False) - #parse object fields + # parse object fields for c in self._object_fields: if c in _parsed_data and _parsed_data[c] and isinstance(_parsed_data[c], Model): _parsed_data[c] = _parsed_data[c].parse_data(_parsed_data[c].get_dict()) - #parse direct and indirect audience targeting + # parse direct and indirect audience targeting for c in self._indirect_audience_targeting_fields + self._direct_audience_targeting_fields: if c in _parsed_data and _parsed_data[c]: - _parsed_data[c] = [obj.parse_data(obj.get_dict()) if isinstance(obj, Model) else obj\ + _parsed_data[c] = [obj.parse_data(obj.get_dict()) if isinstance(obj, Model) else obj for obj in _parsed_data[c]] - #parse media fields + # parse media fields for c in self._media_fields: if c in _parsed_data and _parsed_data[c] and isinstance(_parsed_data[c], Model): _parsed_data[c] = _parsed_data[c].parse_data(_parsed_data[c].get_dict()) - #parse anything that is a dictionary for things like datetime fields that are datetime objects - for k, v in _parsed_data.items(): + # parse anything that is a dictionary for things like datetime fields that are datetime objects + for k, v in list(_parsed_data.items()): if isinstance(v, dict) and k not in self._response_fields: _parsed_data[k] = self.parse_data(v) + if 'id' in _parsed_data: + # we need to let the database take care of generating ids + # since there are size constraints that the sunspear convention violates, + # however to be safe we'll store the sunspear id as well. + _parsed_data['sunspear_id'] = _parsed_data['id'] + del _parsed_data['id'] return _parsed_data def get_parsed_dict(self, *args, **kwargs): - - #we are suppose to maintain our own published and updated fields + # we are suppose to maintain our own published and updated fields if not self._dict.get('published', None): self._dict['published'] = datetime.datetime.utcnow() elif 'updated' in self._reserved_fields: @@ -113,7 +121,7 @@ def get_dict(self): def _parse_date(self, date=None, utc=True, use_system_timezone=False): dt = None if date is None or not isinstance(date, datetime.datetime): - if isinstance(date, basestring): + if isinstance(date, six.string_types): try: dt = parse(date) except ValueError: @@ -138,6 +146,7 @@ def __getitem__(self, key): class Activity(Model): + _required_fields = ['verb', 'actor', 'object'] _media_fields = ['icon'] _reserved_fields = ['updated'] @@ -156,9 +165,17 @@ def _set_defaults(self, model_dict): return model_dict - def get_parsed_sub_activity_dict(self, actor, content="", verb="reply", object_type="reply", \ - collection="replies", activity_class=None, extra={}, published=None, **kwargs): - #TODO: Doesn't feel like this should be here Feels like it belongs in the backend. + def get_parsed_sub_activity_dict(self, + actor, + content="", + verb="reply", + object_type="reply", + collection="replies", + activity_class=None, + extra={}, + published=None, + **kwargs): + # TODO: Doesn't feel like this should be here Feels like it belongs in the backend. if published is None: published = datetime.datetime.utcnow() @@ -205,7 +222,7 @@ def get_parsed_sub_activity_dict(self, actor, content="", verb="reply", object_t } self._dict[collection]['totalItems'] += 1 - #insert the newest comment at the top of the list + # insert the newest comment at the top of the list self._dict[collection]['items'].insert(0, _sub_dict) parent_activity = self.parse_data(self._dict, **kwargs) @@ -213,7 +230,7 @@ def get_parsed_sub_activity_dict(self, actor, content="", verb="reply", object_t return _activity, parent_activity def parse_data(self, data, *args, **kwargs): - #TODO Rename to jsonify_dict + # TODO Rename to jsonify_dict _parsed_data = super(Activity, self).parse_data(data, *args, **kwargs) for response_field in self._response_fields: if response_field in _parsed_data: diff --git a/sunspear/aggregators/base.py b/sunspear/aggregators/base.py index a2d8991..49b9b3d 100644 --- a/sunspear/aggregators/base.py +++ b/sunspear/aggregators/base.py @@ -1,3 +1,6 @@ +from __future__ import absolute_import, division, print_function, unicode_literals + + class BaseAggregator(object): def __init__(self, *args, **kwargs): pass diff --git a/sunspear/aggregators/property.py b/sunspear/aggregators/property.py index 277743f..fcc3389 100644 --- a/sunspear/aggregators/property.py +++ b/sunspear/aggregators/property.py @@ -1,10 +1,11 @@ -from sunspear.aggregators.base import BaseAggregator -from sunspear.lib.dotdict import dotdictify - -from itertools import groupby +from __future__ import absolute_import, division, print_function, unicode_literals import copy import re +from itertools import groupby + +from sunspear.aggregators.base import BaseAggregator +from sunspear.lib.dotdict import dotdictify class PropertyAggregator(BaseAggregator): @@ -48,12 +49,12 @@ def _listify_attributes(self, group_by_attributes=[], activity={}): nested_root, rest = attr.split('.', 1) #store a list of nested roots. We'll have to be careful not to listify these nested_root_attributes.append(nested_root) - for nested_dict_key, nested_dict_value in activity.get(nested_dict).items(): + for nested_dict_key, nested_dict_value in list(activity.get(nested_dict).items()): if nested_dict_key != deepest_attr: listified_dict['.'.join([nested_dict, nested_dict_key])] = [nested_dict_value] #now we listify all other non nested attributes - for key, val in activity.items(): + for key, val in list(activity.items()): if key not in group_by_attributes and key not in nested_root_attributes: listified_dict[key] = [val] @@ -96,7 +97,7 @@ def _aggregate_activities(self, group_by_attributes=[], grouped_activities=[]): #aggregate the rest of the activities into lists for activity in group_list[1:]: activity = dotdictify(activity) - for key in aggregated_activity.keys(): + for key in list(aggregated_activity.keys()): if key not in group_by_attributes and key not in nested_root_attributes: aggregated_activity[key].append(activity.get(key)) @@ -107,7 +108,7 @@ def _aggregate_activities(self, group_by_attributes=[], grouped_activities=[]): if nested_val is not None: nested_dict, deepest_attr = attr.rsplit('.', 1) - for nested_dict_key, nested_dict_value in activity.get(nested_dict).items(): + for nested_dict_key, nested_dict_value in list(activity.get(nested_dict).items()): if nested_dict_key != deepest_attr: aggregated_activity['.'.join([nested_dict, nested_dict_key])].append(nested_dict_value) diff --git a/sunspear/backends/base.py b/sunspear/backends/base.py index 3e415f9..a841062 100644 --- a/sunspear/backends/base.py +++ b/sunspear/backends/base.py @@ -1,9 +1,14 @@ -from sunspear.activitystreams.models import Activity, ReplyActivity, LikeActivity -from sunspear.exceptions import (SunspearDuplicateEntryException, SunspearInvalidActivityException, - SunspearInvalidObjectException) +from __future__ import absolute_import, division, print_function, unicode_literals -import uuid import copy +import uuid + +from sunspear.activitystreams.models import (Activity, LikeActivity, Model, + ReplyActivity) +from sunspear.exceptions import (SunspearDuplicateEntryException, + SunspearInvalidActivityException, + SunspearInvalidObjectException, + SunspearOperationNotSupportedException) __all__ = ('BaseBackend', 'SUB_ACTIVITY_MAP') @@ -14,6 +19,7 @@ class BaseBackend(object): + def clear_all_objects(self): """ Clears all objects from the backend. @@ -48,7 +54,17 @@ def activity_exists(self, activity, **kwargs): """ raise NotImplementedError() - #TODO: Tests + def _resolve_activity_id(self, activity, **kwargs): + activity_id = self._extract_id(activity) + if activity_id: + if self.activity_exists(activity, **kwargs): + raise SunspearDuplicateEntryException() + else: + activity_id = self.get_new_id() + + return activity_id + + # TODO: Tests def create_activity(self, activity, **kwargs): """ Stores a new ``activity`` in the backend. If an object with the same id already exists in @@ -64,18 +80,14 @@ def create_activity(self, activity, **kwargs): :raises: ``SunspearDuplicateEntryException`` if the record already exists in the database. :return: dict representing the new activity. """ - activity_id = self._extract_id(activity) - if activity_id: - if self.activity_exists(activity, **kwargs): - raise SunspearDuplicateEntryException() - else: - activity['id'] = self.get_new_id() + activity_id = self._resolve_activity_id(activity, **kwargs) + activity['id'] = activity_id activity_copy = copy.copy(activity) objs_created = [] objs_modified = [] - for key, value in activity_copy.items(): + for key, value in list(activity_copy.items()): if key in Activity._object_fields and isinstance(value, dict): if self.obj_exists(value): previous_value = self.get_obj([self._extract_id(value)])[0] @@ -90,7 +102,7 @@ def create_activity(self, activity, **kwargs): new_obj = self.create_obj(value) objs_created.append(new_obj) except Exception: - #there was an error, undo everything we just did + # there was an error, undo everything we just did self._rollback(objs_created, objs_modified) raise @@ -218,7 +230,6 @@ def create_obj(self, obj, **kwargs): :type obj: dict :param obj: obj we want to store in the backend - :raises: ``SunspearDuplicateEntryException`` if the record already exists in the database. :return: dict representing the new obj. """ obj_id = self._extract_id(obj) @@ -300,6 +311,15 @@ def get_obj(self, obj_ids=[], **kwargs): def obj_get(self, obj, **kwargs): raise NotImplementedError() + def is_sub_activity_verb_valid(self, sub_activity_verb): + return sub_activity_verb.lower() in SUB_ACTIVITY_MAP + + def get_sub_activity_model(self, sub_activity_verb): + return SUB_ACTIVITY_MAP[sub_activity_verb.lower()][0] + + def get_sub_activity_attribute(self, sub_activity_verb): + return SUB_ACTIVITY_MAP[sub_activity_verb.lower()][1] + def create_sub_activity(self, activity, actor, content, extra={}, sub_activity_verb="", **kwargs): """ Creates a new sub-activity as a child of ``activity``. @@ -326,6 +346,9 @@ def create_sub_activity(self, activity, actor, content, extra={}, sub_activity_v if not activity_id: raise SunspearInvalidActivityException() + if not self.is_sub_activity_verb_valid(sub_activity_verb): + raise SunspearOperationNotSupportedException('Verb not supported') + return self.sub_activity_create(activity, actor, content, extra=extra, sub_activity_verb=sub_activity_verb, **kwargs) @@ -379,21 +402,9 @@ def _listify(self, list_or_string): """ if not isinstance(list_or_string, (list, tuple, set)): list_or_string = [list_or_string] - else: - list_or_string = list_or_string return list_or_string - def _extract_id(self, activity_or_id): - """ - Helper that returns an id if the activity has one. - """ - this_id = activity_or_id - if isinstance(activity_or_id, dict): - this_id = activity_or_id.get('id', None) - - return this_id - def get_new_id(self): """ Generates a new unique ID. The default implementation uses uuid1 to @@ -402,3 +413,78 @@ def get_new_id(self): :return: a new id """ return uuid.uuid1().hex + + def _extract_object_keys(self, activity, skip_sub_activities=False): + keys = [] + for object_key in Model._object_fields + Activity._direct_audience_targeting_fields \ + + Activity._indirect_audience_targeting_fields: + if object_key not in activity: + continue + objects = activity.get(object_key) + if isinstance(objects, dict): + if objects.get('objectType', None) == 'activity': + keys = keys + self._extract_object_keys(objects) + if objects.get('inReplyTo', None): + [keys.extend(self._extract_object_keys(in_reply_to_obj, skip_sub_activities=skip_sub_activities)) \ + for in_reply_to_obj in objects['inReplyTo']] + if isinstance(objects, list): + for item in objects: + if isinstance(item, basestring): + keys.append(item) + if isinstance(objects, basestring): + keys.append(objects) + + if not skip_sub_activities: + for collection in Activity._response_fields: + if collection in activity and activity[collection]['items']: + for item in activity[collection]['items']: + keys.extend(self._extract_object_keys(item)) + return keys + + def _extract_id(self, activity_or_id): + """ + Helper that returns an id if the activity has one. + """ + if isinstance(activity_or_id, basestring): + return activity_or_id + elif isinstance(activity_or_id, dict): + this_id = activity_or_id.get('id', None) + if this_id is None: + return None + try: + return str(this_id) + except Exception: + return None + else: + try: + return str(activity_or_id) + except Exception: + return None + + def _dehydrate_object_keys(self, activity, objects_dict, skip_sub_activities=False): + for object_key in Model._object_fields + Activity._direct_audience_targeting_fields \ + + Activity._indirect_audience_targeting_fields: + if object_key not in activity: + continue + activity_objects = activity.get(object_key) + if isinstance(activity_objects, dict): + if activity_objects.get('objectType', None) == 'activity': + activity[object_key] = self._dehydrate_object_keys(activity_objects, objects_dict, skip_sub_activities=skip_sub_activities) + if activity_objects.get('inReplyTo', None): + for i, in_reply_to_obj in enumerate(activity_objects['inReplyTo']): + activity_objects['inReplyTo'][i] = \ + self._dehydrate_object_keys(activity_objects['inReplyTo'][i], \ + objects_dict, skip_sub_activities=skip_sub_activities) + if isinstance(activity_objects, list): + for i, obj_id in enumerate(activity_objects): + if isinstance(activity[object_key][i], basestring): + activity[object_key][i] = objects_dict.get(obj_id, {}) + if isinstance(activity_objects, basestring): + activity[object_key] = objects_dict.get(activity_objects, {}) + + if not skip_sub_activities: + for collection in Activity._response_fields: + if collection in activity and activity[collection]['items']: + for i, item in enumerate(activity[collection]['items']): + activity[collection]['items'][i] = self._dehydrate_object_keys(item, objects_dict) + return activity diff --git a/sunspear/backends/database/__init__.py b/sunspear/backends/database/__init__.py new file mode 100644 index 0000000..eda7f14 --- /dev/null +++ b/sunspear/backends/database/__init__.py @@ -0,0 +1 @@ +from db import DatabaseBackend diff --git a/sunspear/backends/database/db.py b/sunspear/backends/database/db.py new file mode 100644 index 0000000..d3da99c --- /dev/null +++ b/sunspear/backends/database/db.py @@ -0,0 +1,492 @@ +""" +Copyright 2016 Numan Sachwani + +This file is provided to you under the Apache License, +Version 2.0 (the "License"); you may not use this file +except in compliance with the License. You may obtain +a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, +software distributed under the License is distributed on an +"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +KIND, either express or implied. See the License for the +specific language governing permissions and limitations +under the License. +""" +from __future__ import absolute_import, unicode_literals + +import copy +import datetime +import json +import logging + +import six +from dateutil import tz +from dateutil.parser import parse + +from sqlalchemy import create_engine, sql +from sqlalchemy.pool import QueuePool +from sunspear.activitystreams.models import Activity, Model, Object +from sunspear.backends.base import BaseBackend +from sunspear.exceptions import SunspearOperationNotSupportedException + +from . import schema + + +logger = logging.getLogger(__name__) + +DB_OBJ_FIELD_MAPPING = { + 'id': 'id', + 'objectType': 'object_type', + 'displayName': 'display_name', + 'content': 'content', + 'published': 'published', + 'image': 'image', +} + +DB_ACTIVITY_FIELD_MAPPING = { + 'id': 'id', + 'verb': 'verb', + 'actor': 'actor_id', + 'object': 'object_id', + 'target': 'target_id', + 'author': 'author_id', + 'generator': 'generator_id', + 'provider': 'provider_id', + 'content': 'content', + 'published': 'published', + 'updated': 'updated', + 'icon': 'icon', +} + +DICT_FIELDS = Activity._media_fields + Object._media_fields + Activity._object_fields + ['other_data',] + + +class DatabaseBackend(BaseBackend): + + def __init__(self, db_connection_string=None, verbose=False, poolsize=10, max_overflow=5, **kwargs): + if db_connection_string is None: + logger.info("WARNING: Not given a valid db connection string") + self._engine = create_engine( + db_connection_string, + echo=verbose, + poolclass=QueuePool, + pool_size=poolsize, + max_overflow=max_overflow, + convert_unicode=True) + + @property + def engine(self): + return self._engine + + @property + def activities_table(self): + return schema.tables['activities'] + + @property + def objects_table(self): + return schema.tables['objects'] + + @property + def likes_table(self): + return schema.tables['likes'] + + @property + def replies_table(self): + return schema.tables['replies'] + + @property + def cc_table(self): + return schema.tables['cc'] + + @property + def bcc_table(self): + return schema.tables['bcc'] + + @property + def to_table(self): + return schema.tables['to'] + + @property + def bto_table(self): + return schema.tables['bto'] + + def _get_connection(self): + return self.engine.connect() + + def create_tables(self): + schema.metadata.create_all(self.engine) + + def drop_tables(self): + schema.metadata.drop_all(self.engine) + + def clear_all(self): + self.drop_tables() + self.create_tables() + + def clear_all_objects(self): + raise SunspearOperationNotSupportedException() + + def clear_all_activities(self): + self.engine.execute(self.activities_table.delete()) + + def obj_create(self, obj, **kwargs): + obj_dict = self._get_parsed_and_validated_obj_dict(obj) + obj_db_schema_dict = self._obj_dict_to_db_schema(obj_dict) + + if self.obj_exists(obj): + self.obj_update(obj) + else: + self.engine.execute(self.objects_table.insert(), [obj_db_schema_dict]).close() + + return obj_dict + + def obj_exists(self, obj, **kwargs): + obj_id = self._extract_id(obj) + objs_db_table = self.objects_table + # this probably needs to use sunspear id now? i think... + # mmm.. + return self.engine.execute(sql.select([sql.exists().where(objs_db_table.c.id == obj_id)])).scalar() + + def obj_update(self, obj, **kwargs): + obj_dict = self._get_parsed_and_validated_obj_dict(obj) + obj_id = self._extract_id(obj_dict) + obj_db_schema_dict = self._obj_dict_to_db_schema(obj_dict) + + self.engine.execute( + self.objects_table.update().where(self.objects_table.c.id == obj_id).values(**obj_db_schema_dict)) + + def obj_get(self, obj, **kwargs): + """ + Given a list of object ids, returns a list of objects + """ + if not obj: + return obj + + obj_ids = [self._extract_id(o) for o in obj] + s = self._get_select_multiple_objects_query(obj_ids) + + results = self.engine.execute(s).fetchall() + results = map(self._db_schema_to_obj_dict, results) + + return results + + def obj_delete(self, obj, **kwargs): + obj_id = self._extract_id(obj) + + stmt = self.objects_table.delete().where(self.objects_table.c.id == obj_id) + self.engine.execute(stmt) + + def activity_exists(self, activity, **kwargs): + activity_id = self._extract_id(activity) + activities_db_table = self.activities_table + + return self.engine.execute(sql.select([sql.exists().where(activities_db_table.c.id == activity_id)])).scalar() + + def activity_create(self, activity_dict, **kwargs): + """ + Creates an activity. This assumes the activity is already dehydrated (ie has references + to the objects and not the actual objects itself) + """ + activity = Activity(activity_dict, backend=self) + + activity.validate() + activity_dict = activity.get_parsed_dict() + + activity_db_schema_dict = self._activity_dict_to_db_schema(activity_dict) + + self.engine.execute(self.activities_table.insert(), [activity_db_schema_dict]).close() + + obj_id = activity_dict['object'] + activity_id = activity_dict['id'] + for audience_field in ['cc', 'bcc', 'to', 'bto']: + if audience_field in activity_dict: + table = schema.tables[audience_field] + self.engine.execute(table.insert(), dict( + obj_id=obj_id, + activity_id=activity_id + )).close() + + return self.get_activity(activity_dict) + + def activity_delete(self, activity, **kwargs): + activity_id = activity['id'] + statement = self.activities_table.delete().where( + self.activities_table.c.id == activity_id) + self.engine.execute(statement) + + def _extract_activity_obj_key(self, obj_or_value): + activity_obj = None + + if isinstance(obj_or_value, dict): + activity_obj_id = self._extract_id(obj_or_value) + activity_obj = obj_or_value + else: + activity_obj_id = obj_or_value + + return activity_obj, activity_obj_id + + def create_activity(self, activity, **kwargs): + activity_id = self._resolve_activity_id(activity, **kwargs) + activity['id'] = activity_id + + activity_copy = copy.copy(activity) + + activity_objs = {} + ids_of_objs_with_no_dict = [] + + audience_targeting_fields = Activity._direct_audience_targeting_fields + Activity._indirect_audience_targeting_fields + + for key, value in activity_copy.items(): + if key in Activity._object_fields: + activity_obj, activity_obj_id = self._extract_activity_obj_key(value) + if activity_obj: + activity_objs[activity_obj_id] = activity_obj + activity[key] = activity_obj_id + else: + ids_of_objs_with_no_dict.append(activity_obj_id) + + if key in audience_targeting_fields and value: + activity_audience_targeting_objs = [] + for activity_obj_or_value in value: + activity_obj, activity_obj_id = self._extract_activity_obj_key(activity_obj_or_value) + if activity_obj: + activity_objs[activity_obj_id] = activity_obj + activity_audience_targeting_objs.append(activity_obj_id) + else: + ids_of_objs_with_no_dict.append(activity_obj_id) + activity[key] = activity_audience_targeting_objs + + for obj_id, obj in activity_objs.items(): + self.obj_create(obj) + + return self.activity_create(activity, **kwargs) + + def activity_get(self, + activity_ids, + raw_filters=None, + filters="", + include_public=False, + audience_targeting=None, + aggregation_pipeline=None, + **kwargs): + + if filters is None: + filters = {} + if audience_targeting is None: + audience_targeting = {} + if aggregation_pipeline is None: + aggregation_pipeline = [] + activity_ids = self._listify(activity_ids) # TODO: likely don't need to listify here. + + assert len(audience_targeting) == 1 or len(audience_targeting) == 0, "I can't be wrong about this. I hope" + + """ + audience_activity_ids = None + for audience_type, object_id in audience_targeting.items(): + # audience_table = schema.tables[audience_type] + # audience_query = sql.select([audience_table.c.activity_id]).where( + # audience_table.c.obj_id == object_id) + + audience_activity_ids = self.engine.execute(sql.select([self.activities_table.c.id]).where( + self.activities_table.c.actor_id.in_(audience_targeting[audience_type]) + )).fetchall() + + # audience_activities_query = sql.select(['*']).where(self.activities_table.c.id.in_(audience_query)) + # result_proxy = self.engine.execute(audience_activities_query) + """ + # if audience_activity_ids is not None and not include_public: # only filter then?? + # activity_ids = list(set(audience_activity_ids) & (set(activity_ids))) + self.filter_activities_by_audience(activity_ids, audience_targeting) + + activities = self._get_raw_activities(activity_ids, **kwargs) + activities = self.hydrate_activities(activities) + + return activities + + def filter_activities_by_audience(self, activity_ids, audience_targeting): + # s = sql.select(['*']).where(self.objects_table.c.id.in_(obj_ids)) + # TODO: bcc, bto, etc.. + cc_query = sql.select(['*']).where(self.cc_table.c.activity_id.in_(activity_ids)) + res = self.engine.execute(cc_query).fetchall() + return None + + def sub_activity_create(self, activity, actor, content, extra={}, sub_activity_verb="", published=None, **kwargs): + object_type = kwargs.get('object_type', sub_activity_verb) + sub_activity_model = self.get_sub_activity_model(sub_activity_verb) + sub_activity_attribute = self.get_sub_activity_attribute(sub_activity_verb) + + activity_id = self._extract_id(activity) + raw_activity = self._get_raw_activities([activity_id])[0] + activity_model = Activity(raw_activity, backend=self) + + sub_activity_table = getattr(self, '{}_table'.format(sub_activity_attribute)) + + sub_activity, original_activity = activity_model\ + .get_parsed_sub_activity_dict( + actor=actor, content=content, verb=sub_activity_verb, + object_type=object_type, collection=sub_activity_attribute, + activity_class=sub_activity_model, published=published, extra=extra) + + sub_activity = self.create_activity(sub_activity)[0] + sub_activity_db_schema = self._convert_sub_activity_to_db_schema(sub_activity, original_activity) + self.engine.execute(sub_activity_table.insert(), [sub_activity_db_schema]) + + return sub_activity, original_activity + + def hydrate_activities(self, activities): + """ + Takes a raw list of activities returned from riak and replace keys with contain ids for riak objects with actual riak object + TODO: This can probably be refactored out of the riak backend once everything like + sub activities and shared with fields are implemented + """ + # collect a list of unique object ids. We only iterate through the fields that we know + # for sure are objects. User is responsible for hydrating all other fields. + object_ids = set() + for activity in activities: + object_ids.update(self._extract_object_keys(activity)) + + # Get the objects for the ids we have collected + objects = self.get_obj(object_ids) + objects_dict = dict(((obj["id"], obj,) for obj in objects)) + + # replace the object ids with the hydrated objects + for activity in activities: + self._dehydrate_object_keys(activity, objects_dict) + + return activities + + def _get_raw_activities(self, activity_ids, **kwargs): + activity_ids = map(self._extract_id, activity_ids) # Likely don't need to do this + if not activity_ids: + return [] + + s = self._get_select_multiple_activities_query(activity_ids) + activities = self.engine.execute(s).fetchall() + activities = [self._db_schema_to_activity_dict(activity) for activity in activities] + + return activities + + def _convert_sub_activity_to_db_schema(self, sub_activity, activity): + # Find all the fields in the sub activity that aren't part of the standard activity object + converted_subactivity = self._activity_dict_to_db_schema(sub_activity) + other_data = converted_subactivity.get('other_data') + sub_activity = { + 'id': sub_activity['id'], + 'in_reply_to': activity['id'], + 'actor': sub_activity['actor']['id'], + 'published': self._get_db_compatiable_date_string(sub_activity['published']), + 'updated': self._get_db_compatiable_date_string(sub_activity['published']), + 'content': sub_activity['object']['content'], + } + if other_data: + sub_activity['other_data'] = other_data + return sub_activity + + def _convert_to_db_schema(self, obj, field_mapping): + # we make a copy because we will be mutating the dict. + # we will map official fields to db fields, and put the rest in `other_data` + obj_copy = copy.deepcopy(obj) + schema_dict = {} + + for obj_field, db_schema_field in field_mapping.items(): + if obj_field in obj_copy: + data = obj_copy.pop(obj_field) + + # SQLAlchemy requires datetime fields to be datetime strings + if obj_field in Model._datetime_fields: + data = self._get_datetime_obj(data) + data = self._get_db_compatiable_date_string(data) + + schema_dict[db_schema_field] = data + + # all standard fields should no longer be part of the dictionary + if obj_copy: + schema_dict['other_data'] = obj_copy + + return schema_dict + + def _need_to_parse_json(self, schema_field_name, data): + if schema_field_name in DICT_FIELDS and isinstance(data, six.string_types) and data: + # TODO: This seems hacky. Is there a better way to do this? + if '{' in data or '[' in data: + return True + return False + + def _convert_to_activity_stream_schema(self, schema_dict, field_mapping): + # we make a copy because we will be mutating the dict. + # we will map official fields to db fields, and put the rest in `other_data` + obj_dict = {} + + for obj_field, db_schema_field in field_mapping.items(): + if db_schema_field in schema_dict: + data = schema_dict[db_schema_field] + if self._need_to_parse_json(db_schema_field, data): + data = json.loads(data) + + # SQLAlchemy requires datetime fields to be datetime instances + if obj_field in Model._datetime_fields: + data = self._get_datetime_obj(data) + data = '{}Z'.format(data.isoformat()) + + obj_dict[obj_field] = data + + if 'other_data' in schema_dict and schema_dict['other_data'] is not None: + other_data = schema_dict['other_data'] + if self._need_to_parse_json('other_data', other_data): + other_data = json.loads(other_data) + obj_dict.update(other_data) + + return obj_dict + + def _obj_dict_to_db_schema(self, obj): + return self._convert_to_db_schema(obj, DB_OBJ_FIELD_MAPPING) + + def _activity_dict_to_db_schema(self, activity): + return self._convert_to_db_schema(activity, DB_ACTIVITY_FIELD_MAPPING) + + def _db_schema_to_obj_dict(self, obj): + return self._convert_to_activity_stream_schema(obj, DB_OBJ_FIELD_MAPPING) + + def _db_schema_to_activity_dict(self, activity): + return self._convert_to_activity_stream_schema(activity, DB_ACTIVITY_FIELD_MAPPING) + + def _get_datetime_obj(self, datetime_instance): + if isinstance(datetime_instance, basestring): + datetime_instance = parse(datetime_instance) + utctimezone = tz.tzutc() + + # Assume UTC if we don't have a timezone + if datetime_instance.tzinfo is None: + datetime_instance.replace(tzinfo=utctimezone) + # If we do have a timezone, convert it to UTC + elif datetime.tzinfo != utctimezone: + datetime_instance.astimezone(utctimezone) + + return datetime_instance + + def _get_db_compatiable_date_string(self, datetime_instance): + datetime_instance = self._get_datetime_obj(datetime_instance) + + return datetime_instance.strftime('%Y-%m-%d %H:%M:%S') + + def _flatten(self, list_of_lists): + return [item for sublist in list_of_lists for item in sublist] + + def _get_parsed_and_validated_obj_dict(self, obj): + obj = Object(obj, backend=self) + + obj.validate() + obj_dict = obj.get_parsed_dict() + + return obj_dict + + def _get_select_multiple_objects_query(self, obj_ids): + s = sql.select(['*']).where(self.objects_table.c.id.in_(obj_ids)) + return s + + def _get_select_multiple_activities_query(self, activity_ids): + s = sql.select(['*']).where(self.activities_table.c.id.in_(activity_ids)) + return s diff --git a/sunspear/backends/database/schema.py b/sunspear/backends/database/schema.py new file mode 100644 index 0000000..8b11f06 --- /dev/null +++ b/sunspear/backends/database/schema.py @@ -0,0 +1,92 @@ +from datetime import datetime +from sqlalchemy import Table, Column, DateTime, Integer, String, Text, MetaData, ForeignKey, UniqueConstraint +import types as custom_types + + +metadata = MetaData() + +objects_table = Table('sgactivitystream_streamobject', metadata, + Column('id', String(32), primary_key=True, nullable=True), + Column('object_type', String(256), nullable=False), + Column('display_name', String(256), default=''), + Column('sunspear_id', String(256), nullable=True), + Column('badge_id', ForeignKey('sgrecognition_badge.id', ondelete='SET NULL')), + Column('badgerecipient_id', ForeignKey('sgrecognition_badgerecipient.id', ondelete='SET NULL')), + Column('checkin_id', ForeignKey('sgcheckin_checkin.id', ondelete='SET NULL')), + Column('goal_id', ForeignKey('sggoals_goal.id', ondelete='SET NULL')), + Column('keyresult_id', ForeignKey('sggoals_kyresult.id', ondelete='SET NULL')), + Column('oneonone_id', ForeignKey('sgoneonone_oneonone.id', ondelete='SET NULL')), + Column('sgnetwork_id', ForeignKey('sgnetworks_sgnetwork.container_ptr_id', ondelete='SET NULL')), + Column('team_id', ForeignKey('sgteam_team.id', ondelete='SET NULL')), + Column('userprofile_id', ForeignKey('core_userprofile.id', ondelete='SET NULL')), + Column('content', Text, default=''), + Column('published', DateTime(timezone=True), nullable=False), + Column('updated', DateTime(timezone=True), default=datetime.now(), onupdate=datetime.now()), + Column('image', custom_types.JSONSmallDict(4096), default={}), + Column('other_data', custom_types.JSONDict(), default={})) + +activities_table = Table('sgactivitystream_streamactivity', metadata, + Column('id', String(32), primary_key=True), + Column('verb', String(256), nullable=False), + Column('unique_verb', String(256), nullable=False), + Column('actor_id', ForeignKey('sgactivitystream_objects.id', ondelete='CASCADE'), nullable=False), + Column('object_id', ForeignKey('sgactivitystream_objects.id', ondelete='SET NULL')), + Column('target_id', ForeignKey('sgactivitystream_objects.id', ondelete='SET NULL')), + Column('author_id', ForeignKey('sgactivitystream_objects.id', ondelete='SET NULL')), + Column('sgnetwork_id', ForeignKey('sgnetworks_sgnetwork.container_ptr_id', ondelete='CASCADE'), nullable=True), + Column('generator_id', ForeignKey('sgactivitystream_objects.id', ondelete='SET NULL')), + Column('provider_id', ForeignKey('sgactivitystream_objects.id', ondelete='SET NULL')), + Column('content', Text, default=''), + Column('published', DateTime(timezone=True), nullable=False), + Column('updated', DateTime(timezone=True), default=datetime.now(), onupdate=datetime.now()), + Column('icon', custom_types.JSONSmallDict(4096), default={}), + Column('other_data', custom_types.JSONDict(), default={})) + +replies_table = Table('replies', metadata, + Column('id', String(32), primary_key=True), + Column('in_reply_to', ForeignKey('sgactivitiystream_activities.id', ondelete='CASCADE'), nullable=False), + Column('actor', ForeignKey('sgactivitystream_objects.id', ondelete='CASCADE'), nullable=False), + Column('published', DateTime(timezone=True), nullable=False), + Column('updated', DateTime(timezone=True)), + Column('content', Text, nullable=False), + Column('other_data', custom_types.JSONDict())) + +likes_table = Table('likes', metadata, + Column('id', String(32), primary_key=True), + Column('in_reply_to', ForeignKey('sgactivitystream_activities.id', ondelete='CASCADE'), nullable=False), + Column('actor', ForeignKey('sgactivitystream_objects.id', ondelete='CASCADE'), nullable=False), + Column('published', DateTime(timezone=True), nullable=False), + Column('content', Text), + Column('other_data', custom_types.JSONDict()), + UniqueConstraint('actor', 'in_reply_to')) + +to_table = Table('sgactivitystream_to', metadata, + Column('id', Integer, primary_key=True), + Column('obj_id', ForeignKey('sgactivitystream_objects.id', ondelete='CASCADE')), + Column('activity_id', ForeignKey('sgactivitystream_activities.id', ondelete='CASCADE'))) + +bto_table = Table('sgactivitystream_bto', metadata, + Column('id', Integer, primary_key=True), + Column('obj_id', ForeignKey('sgactivitystream_objects.id', ondelete='CASCADE')), + Column('activity_id', ForeignKey('sgactivitystream_activities.id', ondelete='CASCADE'))) + +cc_table = Table('sgactivitystream_cc', metadata, + Column('id', Integer, primary_key=True), + Column('obj_id', ForeignKey('sgactivitystream_objects.id', ondelete='CASCADE')), + Column('activity_id', ForeignKey('sgactivitystream_activities.id', ondelete='CASCADE'))) + +bcc_table = Table('sgactivitystream_bcc', metadata, + Column('id', Integer, primary_key=True), + Column('obj_id', ForeignKey('sgactivitystream_objects.id', ondelete='CASCADE')), + Column('activity_id', ForeignKey('sgactivitystream_activities.id', ondelete='CASCADE'))) + +tables = { + 'objects': objects_table, + 'activities': activities_table, + 'replies': replies_table, + 'likes': likes_table, + 'to': to_table, + 'bto': bto_table, + 'cc': cc_table, + 'bcc': bcc_table, +} diff --git a/sunspear/backends/database/types.py b/sunspear/backends/database/types.py new file mode 100644 index 0000000..a2b5347 --- /dev/null +++ b/sunspear/backends/database/types.py @@ -0,0 +1,81 @@ +from sqlalchemy.sql import operators +from sqlalchemy import String +from sqlalchemy.types import TypeDecorator, CHAR, VARCHAR, TEXT +from sqlalchemy.dialects.postgresql import UUID + +import json +import uuid + +__all__ = ['GUID', 'JSONDict', 'JSONSmallDict'] + + +class GUID(TypeDecorator): + """Platform-independent GUID type. + + Uses PostgreSQL's UUID type, otherwise uses + CHAR(32), storing as stringified hex values. + + """ + impl = CHAR + + def load_dialect_impl(self, dialect): + if dialect.name == 'postgresql': + return dialect.type_descriptor(UUID()) + else: + return dialect.type_descriptor(CHAR(32)) + + def process_bind_param(self, value, dialect): + if value is None: + return value + elif dialect.name == 'postgresql': + return str(value) + else: + if not isinstance(value, uuid.UUID): + return "%.32x" % uuid.UUID(value).int + else: + # hexstring + return "%.32x" % value.int + + def process_result_value(self, value, dialect): + if value is None: + return value + else: + return uuid.UUID(value) + + +class JSONEncodedSmallDict(TypeDecorator): + """Represents an immutable structure as a json-encoded string. + + Usage:: + + JSONEncodedDict(255) + + """ + + impl = VARCHAR + + def coerce_compared_value(self, op, value): + if op in (operators.like_op, operators.notlike_op): + return String() + else: + return self + + def process_bind_param(self, value, dialect): + if value is not None: + value = json.dumps(value) + + return value + + def process_result_value(self, value, dialect): + if value is not None: + value = json.loads(value) + return value + + +class JSONEncodedBigDict(JSONEncodedSmallDict): + + impl = TEXT + + +JSONDict = JSONEncodedBigDict +JSONSmallDict = JSONEncodedSmallDict diff --git a/sunspear/backends/riak.py b/sunspear/backends/riak.py index b3271f9..89603ab 100644 --- a/sunspear/backends/riak.py +++ b/sunspear/backends/riak.py @@ -1,5 +1,5 @@ """ -Copyright 2013 Numan Sachwani +Copyright 2016 Numan Sachwani This file is provided to you under the Apache License, Version 2.0 (the "License"); you may not use this file @@ -15,18 +15,19 @@ specific language governing permissions and limitations under the License. """ -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function, unicode_literals -from sunspear.activitystreams.models import Object, Activity, Model -from sunspear.exceptions import (SunspearValidationException) -from sunspear.backends.base import BaseBackend, SUB_ACTIVITY_MAP - -from riak import RiakClient - -import uuid +import calendar import copy import datetime -import calendar +import uuid +import six +from riak import RiakClient + +from sunspear.activitystreams.models import Activity, Model, Object +from sunspear.backends.base import BaseBackend, SUB_ACTIVITY_MAP +from sunspear.compat import must_be_str +from sunspear.exceptions import SunspearValidationException __all__ = ('RiakBackend', ) @@ -145,42 +146,35 @@ class RiakBackend(BaseBackend): - custom_epoch = datetime.datetime(month=1, day=1, year=2013) def __init__( self, protocol="pbc", nodes=[], objects_bucket_name="objects", - activities_bucket_name="activities", **kwargs): - - self._riak_backend = RiakClient(protocol=protocol, nodes=nodes) + activities_bucket_name="activities", r=None, w=None, dw=None, + pr=None, pw=None, **kwargs): - r_value = kwargs.get("r") - w_value = kwargs.get("w") - dw_value = kwargs.get("dw") - pr_value = kwargs.get("pr") - pw_value = kwargs.get("pw") + self._riak_backend = RiakClient(protocol=protocol, nodes=nodes, **kwargs) + self._objects = self._riak_backend.bucket(must_be_str(objects_bucket_name)) + self._activities = self._riak_backend.bucket(must_be_str(activities_bucket_name)) - self._objects = self._riak_backend.bucket(objects_bucket_name) - self._activities = self._riak_backend.bucket(activities_bucket_name) + if r: + self._objects.r = r + self._activities.r = r - if r_value: - self._objects.r = r_value - self._activities.r = r_value + if w: + self._objects.w = w + self._activities.w = w - if w_value: - self._objects.w = w_value - self._activities.w = w_value + if dw: + self._objects.dw = dw + self._activities.dw = dw - if dw_value: - self._objects.dw = dw_value - self._activities.dw = dw_value + if pr: + self._objects.pr = pr + self._activities.pr = pr - if pr_value: - self._objects.pr = pr_value - self._activities.pr = pr_value - - if pw_value: - self._objects.pw = pw_value - self._activities.pw = pw_value + if pw: + self._objects.pw = pw + self._activities.pw = pw def clear_all(self, **kwargs): """ @@ -227,7 +221,7 @@ def obj_create(self, obj, **kwargs): riak_obj.store() - #finally save the data + # finally save the data return obj_dict def set_general_indexes(self, riak_object): @@ -237,11 +231,11 @@ def set_general_indexes(self, riak_object): :type riak_object: RiakObject :param riak_object: a RiakObject representing the model of the class """ - if not filter(lambda x: x[0] == "timestamp_int", riak_object.indexes): - riak_object.add_index("timestamp_int", self._get_timestamp()) + if not any(must_be_str(name) == must_be_str('timestamp_int') for name, value in riak_object.indexes): + riak_object.add_index(must_be_str("timestamp_int"), self._get_timestamp()) - riak_object.remove_index('modified_int') - riak_object.add_index("modified_int", self._get_timestamp()) + riak_object.remove_index(must_be_str('modified_int')) + riak_object.add_index(must_be_str("modified_int"), self._get_timestamp()) return riak_object def obj_update(self, obj, **kwargs): @@ -306,15 +300,15 @@ def set_activity_indexes(self, riak_object): """ _dict = riak_object.data - riak_object.remove_index('verb_bin') - riak_object.remove_index('actor_bin') - riak_object.remove_index('object_bin') - riak_object.add_index("verb_bin", self._extract_id(_dict['verb'])) - riak_object.add_index("actor_bin", self._extract_id(_dict['actor'])) - riak_object.add_index("object_bin", self._extract_id(_dict['object'])) + riak_object.remove_index(must_be_str('verb_bin')) + riak_object.remove_index(must_be_str('actor_bin')) + riak_object.remove_index(must_be_str('object_bin')) + riak_object.add_index(must_be_str("verb_bin"), self._extract_id(_dict['verb'])) + riak_object.add_index(must_be_str("actor_bin"), self._extract_id(_dict['actor'])) + riak_object.add_index(must_be_str("object_bin"), self._extract_id(_dict['object'])) if 'target' in _dict and _dict.get("target"): - riak_object.remove_index('target_bin') - riak_object.add_index("target_bin", self._extract_id(_dict['target'])) + riak_object.remove_index(must_be_str('target_bin')) + riak_object.add_index(must_be_str("target_bin"), self._extract_id(_dict['target'])) return riak_object @@ -333,7 +327,7 @@ def activity_update(self, activity, **kwargs): return self.activity_create(activity, **kwargs) def activity_get( - self, activity_ids=[], raw_filter="", filters={}, include_public=False, + self, activity_ids, raw_filter="", filters={}, include_public=False, audience_targeting={}, aggregation_pipeline=[], **kwargs): """ Gets a list of activities. You can also group activities by providing a list of attributes to group @@ -346,7 +340,7 @@ def activity_get( Filters do not work for nested dictionaries. :type raw_filter: string :param raw_filter: allows you to specify a javascript function as a string. The function should return ``true`` if the activity should be included in the result set - or ``false`` it shouldn't. If you specify a raw filter, the filters specified in ``filters`` will not run. How ever, the results will still be filtered based on + or ``false`` it shouldn't. If you specify a raw filter, the filters specified in ``filters`` will not run. However, the results will still be filtered based on the ``audience_targeting`` parameter. :type include_public: boolean :param include_public: If ``True``, and the ``audience_targeting`` dictionary is defined, activities that are @@ -360,7 +354,7 @@ def activity_get( :return: list -- a list of activities matching ``activity_ids``. If the activities is not found, it is not included in the result set. Activities are returned in the order of ids provided. """ - activity_ids = map(self._extract_id, activity_ids) + activity_ids = list(map(self._extract_id, activity_ids)) if not activity_ids: return [] @@ -376,18 +370,11 @@ def activity_get( activities = aggregator.process(activities, original_activities, aggregation_pipeline) return activities - def create_sub_activity(self, activity, actor, content, extra={}, sub_activity_verb="", **kwargs): - if sub_activity_verb.lower() not in SUB_ACTIVITY_MAP: - raise Exception('Verb not supported') - return super(RiakBackend, self).create_sub_activity( - activity, actor, content, extra=extra, - sub_activity_verb=sub_activity_verb, **kwargs) - def sub_activity_create( self, activity, actor, content, extra={}, sub_activity_verb="", published=None, **kwargs): - sub_activity_model = SUB_ACTIVITY_MAP[sub_activity_verb.lower()][0] - sub_activity_attribute = SUB_ACTIVITY_MAP[sub_activity_verb.lower()][1] + sub_activity_model = self.get_sub_activity_model(sub_activity_verb) + sub_activity_attribute = self.get_sub_activity_attribute(sub_activity_verb) object_type = kwargs.get('object_type', sub_activity_verb) activity_id = self._extract_id(activity) @@ -433,9 +420,7 @@ def sub_activity_delete(self, sub_activity, sub_activity_verb, **kwargs): activity = self._activities.get(key=in_reply_to_key) activity_data = activity.data activity_data[sub_activity_model.sub_item_key]['totalItems'] -= 1 - activity_data[sub_activity_model.sub_item_key]['items'] = filter( - lambda x: x["id"] != sub_activity_id, - activity_data[sub_activity_model.sub_item_key]['items']) + activity_data[sub_activity_model.sub_item_key]['items'] = [x for x in activity_data[sub_activity_model.sub_item_key]['items'] if x["id"] != sub_activity_id] updated_activity = self.update_activity(activity_data, **kwargs) self.delete_activity(sub_activity_id) @@ -453,8 +438,8 @@ def set_sub_item_indexes(self, riak_object, **kwargs): original_activity_id = kwargs.get('activity_id') if not original_activity_id: raise SunspearValidationException() - riak_object.remove_index('inreplyto_bin') - riak_object.add_index("inreplyto_bin", str(original_activity_id)) + riak_object.remove_index(must_be_str('inreplyto_bin')) + riak_object.add_index(must_be_str("inreplyto_bin"), str(original_activity_id)) return riak_object @@ -464,39 +449,39 @@ def dehydrate_activities(self, activities): """ activities = self._extract_sub_activities(activities) - #collect a list of unique object ids. We only iterate through the fields that we know - #for sure are objects. User is responsible for hydrating all other fields. + # collect a list of unique object ids. We only iterate through the fields that we know + # for sure are objects. User is responsible for hydrating all other fields. object_ids = set() for activity in activities: object_ids.update(self._extract_object_keys(activity)) - #Get the objects for the ids we have collected + # Get the objects for the ids we have collected objects = self.get_obj(object_ids) objects_dict = dict(((obj["id"], obj,) for obj in objects)) - #We also need to extract any activities that were diguised as objects. IE activities with - #objectType=activity + # We also need to extract any activities that were diguised as objects. IE activities with + # objectType=activity activities_in_objects_ids = set() - #replace the object ids with the hydrated objects + # replace the object ids with the hydrated objects for activity in activities: activity = self._dehydrate_object_keys(activity, objects_dict) - #Extract keys of any activities that were objects + # Extract keys of any activities that were objects activities_in_objects_ids.update(self._extract_activity_keys(activity, skip_sub_activities=True)) - #If we did have activities that were objects, we need to hydrate those activities and - #the objects for those activities + # If we did have activities that were objects, we need to hydrate those activities and + # the objects for those activities if activities_in_objects_ids: sub_activities = self._get_many_activities(activities_in_objects_ids) activities_in_objects_dict = dict(((sub_activity["id"], sub_activity,) for sub_activity in sub_activities)) for activity in activities: activity = self._dehydrate_sub_activity(activity, activities_in_objects_dict, skip_sub_activities=True) - #we have to do one more round of object dehydration for our new sub-activities + # we have to do one more round of object dehydration for our new sub-activities object_ids.update(self._extract_object_keys(activity)) - #now get all the objects we don't already have and for sub-activities and and hydrate them into - #our list of activities + # now get all the objects we don't already have and for sub-activities and and hydrate them into + # our list of activities object_ids -= set(objects_dict.keys()) objects = self.get_obj(object_ids) for obj in objects: @@ -526,7 +511,7 @@ def _extract_sub_activities(self, activities): for sub_activity in sub_activities: activities_dict[sub_activity["id"]] = sub_activity - #Dehydrate out any subactivities we may have + # Dehydrate out any subactivities we may have for activity in activities: activity = self._dehydrate_sub_activity(activity, activities_dict) @@ -570,7 +555,7 @@ def _dehydrate_sub_activity(self, sub_activity, obj_list, skip_sub_activities=Fa for i, item in enumerate(sub_activity[collection]['items']): try: dehydrated_sub_items.append(self._dehydrate_sub_activity(item, obj_list)) - except KeyError, e: + except KeyError as e: pass sub_activity[collection]['items'] = dehydrated_sub_items sub_activity[collection]['totalItems'] = len(dehydrated_sub_items) @@ -592,9 +577,9 @@ def _extract_object_keys(self, activity, skip_sub_activities=False): for in_reply_to_obj in objects['inReplyTo']] if isinstance(objects, list): for item in objects: - if isinstance(item, basestring): + if isinstance(item, six.string_types): keys.append(item) - if isinstance(objects, basestring): + if isinstance(objects, six.string_types): keys.append(objects) if not skip_sub_activities: @@ -620,9 +605,9 @@ def _dehydrate_object_keys(self, activity, objects_dict, skip_sub_activities=Fal objects_dict, skip_sub_activities=skip_sub_activities) if isinstance(activity_objects, list): for i, obj_id in enumerate(activity_objects): - if isinstance(activity[object_key][i], basestring): + if isinstance(activity[object_key][i], six.string_types): activity[object_key][i] = objects_dict.get(obj_id, {}) - if isinstance(activity_objects, basestring): + if isinstance(activity_objects, six.string_types): activity[object_key] = objects_dict.get(activity_objects, {}) if not skip_sub_activities: @@ -640,7 +625,7 @@ def _get_many_activities(self, activity_ids=[], raw_filter="", filters=None, inc :param activity_ids: The list of activities you want to retrieve :type raw_filter: string :param raw_filter: allows you to specify a javascript function as a string. The function should return ``true`` if the activity should be included in the result set - or ``false`` it shouldn't. If you specify a raw filter, the filters specified in ``filters`` will not run. How ever, the results will still be filtered based on + or ``false`` it shouldn't. If you specify a raw filter, the filters specified in ``filters`` will not run. However, the results will still be filtered based on the ``audience_targeting`` parameter. :type filters: dict :param filters: filters list of activities by key, value pair. For example, ``{'verb': 'comment'}`` would only return activities where the ``verb`` was ``comment``. @@ -663,9 +648,10 @@ def _get_many_activities(self, activity_ids=[], raw_filter="", filters=None, inc results = results.reduce(JS_REDUCE_FILTER_AUD_TARGETTING, options={'arg': {'public': include_public, 'filters': audience_targeting}}) if filters or raw_filter: - # An empty `filters` dict would cause all activities to be filtered out. If you wanted that effect, you - # wouldn't have to call this function, so let's assume that an empty dict is a typical default value and - # should denote that there are no filters to apply. + # An empty `filters` dict would cause all activities to be filtered out. If you + # wanted that effect, you wouldn't have to call this function, so let's assume that + # an empty dict is a typical default value and should denote that there are no + # filters to apply. filters = filters or None results = results.reduce(JS_REDUCE_FILTER_PROP, options={'arg': {'raw_filter': raw_filter, 'filters': filters}}) @@ -674,7 +660,7 @@ def _get_many_activities(self, activity_ids=[], raw_filter="", filters=None, inc #riak does not return the results in any particular order (unless we sort). So, #we have to put the objects returned by riak back in order - results_map = dict(map(lambda result: (result['id'], result,), results)) + results_map = dict([(result['id'], result,) for result in results]) reordered_results = [results_map[id] for id in activity_ids if id in results_map] return reordered_results @@ -684,7 +670,7 @@ def _extract_id(self, activity_or_id): Helper that returns an id if the activity has one. """ this_id = None - if isinstance(activity_or_id, basestring): + if isinstance(activity_or_id, six.string_types): this_id = activity_or_id elif isinstance(activity_or_id, dict): this_id = activity_or_id.get('id', None) @@ -704,15 +690,5 @@ def _get_timestamp(self): returns a unix timestamp representing the ``datetime`` object """ dt_obj = datetime.datetime.utcnow() - return long((calendar.timegm(dt_obj.utctimetuple()) * 1000)) + (dt_obj.microsecond / 1000) - - def get_new_id(self): - """ - Generates a new unique ID. The default implementation uses uuid1 to - generate a unique ID. + return int((calendar.timegm(dt_obj.utctimetuple()) * 1000)) + (dt_obj.microsecond // 1000) - :return: a new id - """ - return uuid.uuid1().hex - # now = datetime.datetime.utcnow() - # return str(long(calendar.timegm(now.utctimetuple()) - calendar.timegm(self.custom_epoch.utctimetuple())) + now.microsecond) diff --git a/sunspear/clients.py b/sunspear/clients.py index 491e4ac..3d7a2cc 100644 --- a/sunspear/clients.py +++ b/sunspear/clients.py @@ -1,3 +1,6 @@ +from __future__ import absolute_import, division, print_function, unicode_literals + + class SunspearClient(object): """ The class is used to create, delete, remove and update activity stream items. @@ -120,7 +123,7 @@ def get_objects(self, object_ids=[]): """ return self._backend.get_obj(object_ids) - def get_activities(self, activity_ids=[], **kwargs): + def get_activities(self, activity_ids, **kwargs): """ Gets a list of activities. Specific backends may support other arguments. Please see reference of the specific backends to see all ``kwargs`` supported. @@ -128,7 +131,10 @@ def get_activities(self, activity_ids=[], **kwargs): :type activity_ids: list :param activity_ids: The list of activities you want to retrieve """ - return self._backend.get_activity(activity_ids=activity_ids, **kwargs) + # if activity_ids is None: + # activity_ids = [] + + return self._backend.get_activity(activity_ids, **kwargs) def get_backend(self): """ diff --git a/sunspear/compat.py b/sunspear/compat.py new file mode 100644 index 0000000..61060ca --- /dev/null +++ b/sunspear/compat.py @@ -0,0 +1,25 @@ +from __future__ import absolute_import, division, print_function, unicode_literals + +import traceback +from logging import getLogger + +import six + +logger = getLogger(__name__) + + +def must_be_str(arg): + """ + Some functions require `str` in Python 2, i.e. its binary type, + but also `str` in Python 3, which is its text type... + + Accommodate both. + """ + if six.PY2: + if isinstance(arg, six.text_type): + return arg.encode('utf-8') + # The idea of this function is to simply remove all function calls once we're on python 3, so let's be sure we + # always have the right type passed in in python 2, i.e. `unicode`. + traceback.print_stack() + logger.warn('Unexpectedly got non-unicode in `must_be_str`...', extra={'stack': True}) + return arg diff --git a/sunspear/exceptions.py b/sunspear/exceptions.py index 47eb1bc..89422c6 100644 --- a/sunspear/exceptions.py +++ b/sunspear/exceptions.py @@ -1,3 +1,6 @@ +from __future__ import absolute_import, division, print_function, unicode_literals + + class SunspearBaseException(Exception): pass @@ -25,5 +28,10 @@ class SunspearDuplicateEntryException(SunspearBaseException): class SunspearInvalidActivityException(SunspearBaseException): pass + class SunspearInvalidObjectException(SunspearBaseException): - pass \ No newline at end of file + pass + + +class SunspearOperationNotSupportedException(SunspearBaseException): + pass diff --git a/sunspear/lib/dotdict.py b/sunspear/lib/dotdict.py index 50e4ce6..78576c1 100644 --- a/sunspear/lib/dotdict.py +++ b/sunspear/lib/dotdict.py @@ -1,5 +1,6 @@ #Originally 'borrowed' from http://stackoverflow.com/questions/3797957/python-easily-access-deeply-nested-dict-get-and-set #Some modifications mad to suit the needs of this project +from __future__ import absolute_import, division, print_function, unicode_literals class dotdictify(dict): diff --git a/sunspear/lib/rfc3339.py b/sunspear/lib/rfc3339.py index bfca1f4..46e525d 100644 --- a/sunspear/lib/rfc3339.py +++ b/sunspear/lib/rfc3339.py @@ -20,6 +20,7 @@ .. _BitBucket: https://bitbucket.org/henry/clan.cx/issues ''' +from __future__ import absolute_import, division, print_function, unicode_literals __author__ = 'Henry Precheur ' __license__ = 'ISCL' @@ -28,7 +29,6 @@ import datetime import time -import unittest def _timezone(utc_offset): diff --git a/tests/test_activitystreams.py b/tests/test_activitystreams.py index 7f492f0..b741b2c 100644 --- a/tests/test_activitystreams.py +++ b/tests/test_activitystreams.py @@ -1,12 +1,13 @@ -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function, unicode_literals -from nose.tools import ok_, eq_, raises, set_trace -from mock import MagicMock, call, ANY +import datetime -from sunspear.activitystreams.models import Activity, MediaLink, Object, Model, ReplyActivity, LikeActivity -from sunspear.exceptions import SunspearValidationException +import six +from mock import MagicMock +from nose.tools import eq_, ok_, raises -import datetime +from sunspear.activitystreams.models import Activity, MediaLink, Model, Object +from sunspear.exceptions import SunspearValidationException class TestActivityModel(object): @@ -214,7 +215,7 @@ def test__set_defaults(self): obj = Model({}, backend=MagicMock()) obj_dict = obj._set_defaults({'id': 12}) - ok_(isinstance(obj_dict.get('id'), basestring)) + ok_(isinstance(obj_dict.get('id'), six.string_types)) def test__set_defaults_no_id_does_not_fail(self): obj = Model({}, backend=MagicMock()) @@ -229,7 +230,7 @@ def test__parse_date(self): eq_(obj._parse_date(d), d.strftime('%Y-%m-%dT%H:%M:%S') + "Z") #badly formatted string date - ok_(isinstance(obj._parse_date(date="qwerty"), basestring)) + ok_(isinstance(obj._parse_date(date="qwerty"), six.string_types)) #no date passed - ok_(isinstance(obj._parse_date(date=None), basestring)) + ok_(isinstance(obj._parse_date(date=None), six.string_types)) diff --git a/tests/test_aggregators.py b/tests/test_aggregators.py index d493b62..91ece85 100644 --- a/tests/test_aggregators.py +++ b/tests/test_aggregators.py @@ -1,10 +1,10 @@ -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function, unicode_literals -from nose.tools import ok_, eq_, raises, set_trace +from itertools import groupby -from sunspear.aggregators.property import PropertyAggregator +from nose.tools import eq_, ok_, raises, set_trace -from itertools import groupby +from sunspear.aggregators.property import PropertyAggregator class TestPropertyAggregator(object): diff --git a/tests/test_client.py b/tests/test_client.py index 26ec940..3da4b3a 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -1,14 +1,14 @@ -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function, unicode_literals -from nose.tools import ok_, eq_, set_trace, raises -from mock import MagicMock, call, ANY +import datetime + +from mock import ANY, call, MagicMock +from nose.tools import eq_, ok_, raises, set_trace from sunspear.aggregators.property import PropertyAggregator from sunspear.backends.riak import RiakBackend from sunspear.clients import SunspearClient -import datetime - riak_connection_options = { "nodes": [ {'http_port': 8098, 'host': '127.0.0.1'}], @@ -614,5 +614,4 @@ def test_get_activities_with_aggregation_pipline(self): activities = self._client.get_activities(activity_ids=activity_ids, aggregation_pipeline=[PropertyAggregator(properties=['verb', 'actor'])]) - eq_([{u'id': u'7779', u'verb': u'like', u'target': {u'objectType': u'something', u'id': u'31415', u'published': u'2012-07-05T12:00:00Z'}, u'object': {u'inReplyTo': [], u'objectType': u'like', u'id': u'6669', u'published': u'2012-08-05T12:00:00Z'}, u'actor': {u'objectType': u'something', u'id': u'1234', u'published': u'2012-07-05T12:00:00Z'}}, {u'id': u'8889', u'verb': u'reply', u'target': {u'objectType': u'something', u'id': u'31415', u'published': u'2012-07-05T12:00:00Z'}, u'object': {u'content': u'This is my first reply', u'inReplyTo': [], u'objectType': u'reply', u'id': u'9999', u'published': u'2012-08-05T12:00:00Z'}, u'actor': {u'objectType': u'something', u'id': u'1234', u'published': u'2012-07-05T12:00:00Z'}}, {'grouped_by_attributes': ['verb', 'actor'], u'title': [u'Stream Item', u'Stream Item'], u'object': [{u'objectType': u'something', u'id': u'4353', u'published': u'2012-07-05T12:00:00Z'}, {u'published': u'2012-07-05T12:00:00Z', u'id': u'4353', u'objectType': u'something'}], u'actor': {u'published': u'2012-07-05T12:00:00Z', u'id': u'4321', u'objectType': u'something'}, u'verb': u'post', u'replies': [{u'totalItems': 2, u'items': [{u'verb': u'reply', u'actor': {u'objectType': u'something', u'id': u'1234', u'published': u'2012-07-05T12:00:00Z'}, u'object': {u'target': {u'objectType': u'something', u'id': u'31415', u'published': u'2012-07-05T12:00:00Z'}, u'object': {u'content': u'This is my first reply', u'inReplyTo': [], u'objectType': u'reply', u'id': u'9999', u'published': u'2012-08-05T12:00:00Z'}, u'actor': {u'objectType': u'something', u'id': u'1234', u'published': u'2012-07-05T12:00:00Z'}, u'verb': u'reply', u'id': u'8889', u'objectType': u'activity'}}, {u'verb': u'reply', u'actor': {u'objectType': u'something', u'id': u'1234', u'published': u'2012-07-05T12:00:00Z'}, u'object': {u'target': {u'objectType': u'something', u'id': u'1234', u'published': u'2012-07-05T12:00:00Z'}, u'object': {u'content': u'This is my second reply', u'inReplyTo': [], u'objectType': u'reply', u'id': u'9998', u'published': u'2012-08-05T12:05:00Z'}, u'actor': {u'objectType': u'something', u'id': u'4321', u'published': u'2012-07-05T12:00:00Z'}, u'verb': u'reply', u'id': u'8888', u'objectType': u'activity'}}]}, {u'totalItems': 2, u'items': [{u'verb': u'reply', u'actor': {u'objectType': u'something', u'id': u'1234', u'published': u'2012-07-05T12:00:00Z'}, u'object': {u'target': {u'objectType': u'something', u'id': u'31415', u'published': u'2012-07-05T12:00:00Z'}, u'object': {u'content': u'This is my first reply', u'inReplyTo': [], u'objectType': u'reply', u'id': u'9999', u'published': u'2012-08-05T12:00:00Z'}, u'actor': {u'objectType': u'something', u'id': u'1234', u'published': u'2012-07-05T12:00:00Z'}, u'verb': u'reply', u'id': u'8889', u'objectType': u'activity'}}, {u'verb': u'reply', u'actor': {u'objectType': u'something', u'id': u'1234', u'published': u'2012-07-05T12:00:00Z'}, u'object': {u'target': {u'objectType': u'something', u'id': u'1234', u'published': u'2012-07-05T12:00:00Z'}, u'object': {u'content': u'This is my second reply', u'inReplyTo': [], u'objectType': u'reply', u'id': u'9998', u'published': u'2012-08-05T12:05:00Z'}, u'actor': {u'objectType': u'something', u'id': u'4321', u'published': u'2012-07-05T12:00:00Z'}, u'verb': u'reply', u'id': u'8888', u'objectType': u'activity'}}]}], u'id': [u'5555', u'5556'], 'grouped_by_values': [u'post', {u'published': u'2012-07-05T12:00:00Z', u'id': u'4321', u'objectType': u'something'}]}, {u'id': u'7778', u'verb': u'like', u'target': {u'objectType': u'something', u'id': u'31415', u'published': u'2012-07-05T12:00:00Z'}, u'object': {u'inReplyTo': [], u'objectType': u'like', u'id': u'6669', u'published': u'2012-08-05T12:00:00Z'}, u'actor': {u'objectType': u'something', u'id': u'1234', u'published': u'2012-07-05T12:00:00Z'}}, {u'id': u'8888', u'verb': u'reply', u'target': {u'objectType': u'something', u'id': u'1234', u'published': u'2012-07-05T12:00:00Z'}, u'object': {u'content': u'This is my second reply', u'inReplyTo': [], u'objectType': u'reply', u'id': u'9998', u'published': u'2012-08-05T12:05:00Z'}, u'actor': {u'objectType': u'something', u'id': u'4321', u'published': u'2012-07-05T12:00:00Z'}}], activities) - + eq_([{'id': '7779', 'verb': 'like', 'target': {'objectType': 'something', 'id': '31415', 'published': '2012-07-05T12:00:00Z'}, 'object': {'inReplyTo': [], 'objectType': 'like', 'id': '6669', 'published': '2012-08-05T12:00:00Z'}, 'actor': {'objectType': 'something', 'id': '1234', 'published': '2012-07-05T12:00:00Z'}}, {'id': '8889', 'verb': 'reply', 'target': {'objectType': 'something', 'id': '31415', 'published': '2012-07-05T12:00:00Z'}, 'object': {'content': 'This is my first reply', 'inReplyTo': [], 'objectType': 'reply', 'id': '9999', 'published': '2012-08-05T12:00:00Z'}, 'actor': {'objectType': 'something', 'id': '1234', 'published': '2012-07-05T12:00:00Z'}}, {'grouped_by_attributes': ['verb', 'actor'], 'title': ['Stream Item', 'Stream Item'], 'object': [{'objectType': 'something', 'id': '4353', 'published': '2012-07-05T12:00:00Z'}, {'published': '2012-07-05T12:00:00Z', 'id': '4353', 'objectType': 'something'}], 'actor': {'published': '2012-07-05T12:00:00Z', 'id': '4321', 'objectType': 'something'}, 'verb': 'post', 'replies': [{'totalItems': 2, 'items': [{'verb': 'reply', 'actor': {'objectType': 'something', 'id': '1234', 'published': '2012-07-05T12:00:00Z'}, 'object': {'target': {'objectType': 'something', 'id': '31415', 'published': '2012-07-05T12:00:00Z'}, 'object': {'content': 'This is my first reply', 'inReplyTo': [], 'objectType': 'reply', 'id': '9999', 'published': '2012-08-05T12:00:00Z'}, 'actor': {'objectType': 'something', 'id': '1234', 'published': '2012-07-05T12:00:00Z'}, 'verb': 'reply', 'id': '8889', 'objectType': 'activity'}}, {'verb': 'reply', 'actor': {'objectType': 'something', 'id': '1234', 'published': '2012-07-05T12:00:00Z'}, 'object': {'target': {'objectType': 'something', 'id': '1234', 'published': '2012-07-05T12:00:00Z'}, 'object': {'content': 'This is my second reply', 'inReplyTo': [], 'objectType': 'reply', 'id': '9998', 'published': '2012-08-05T12:05:00Z'}, 'actor': {'objectType': 'something', 'id': '4321', 'published': '2012-07-05T12:00:00Z'}, 'verb': 'reply', 'id': '8888', 'objectType': 'activity'}}]}, {'totalItems': 2, 'items': [{'verb': 'reply', 'actor': {'objectType': 'something', 'id': '1234', 'published': '2012-07-05T12:00:00Z'}, 'object': {'target': {'objectType': 'something', 'id': '31415', 'published': '2012-07-05T12:00:00Z'}, 'object': {'content': 'This is my first reply', 'inReplyTo': [], 'objectType': 'reply', 'id': '9999', 'published': '2012-08-05T12:00:00Z'}, 'actor': {'objectType': 'something', 'id': '1234', 'published': '2012-07-05T12:00:00Z'}, 'verb': 'reply', 'id': '8889', 'objectType': 'activity'}}, {'verb': 'reply', 'actor': {'objectType': 'something', 'id': '1234', 'published': '2012-07-05T12:00:00Z'}, 'object': {'target': {'objectType': 'something', 'id': '1234', 'published': '2012-07-05T12:00:00Z'}, 'object': {'content': 'This is my second reply', 'inReplyTo': [], 'objectType': 'reply', 'id': '9998', 'published': '2012-08-05T12:05:00Z'}, 'actor': {'objectType': 'something', 'id': '4321', 'published': '2012-07-05T12:00:00Z'}, 'verb': 'reply', 'id': '8888', 'objectType': 'activity'}}]}], 'id': ['5555', '5556'], 'grouped_by_values': ['post', {'published': '2012-07-05T12:00:00Z', 'id': '4321', 'objectType': 'something'}]}, {'id': '7778', 'verb': 'like', 'target': {'objectType': 'something', 'id': '31415', 'published': '2012-07-05T12:00:00Z'}, 'object': {'inReplyTo': [], 'objectType': 'like', 'id': '6669', 'published': '2012-08-05T12:00:00Z'}, 'actor': {'objectType': 'something', 'id': '1234', 'published': '2012-07-05T12:00:00Z'}}, {'id': '8888', 'verb': 'reply', 'target': {'objectType': 'something', 'id': '1234', 'published': '2012-07-05T12:00:00Z'}, 'object': {'content': 'This is my second reply', 'inReplyTo': [], 'objectType': 'reply', 'id': '9998', 'published': '2012-08-05T12:05:00Z'}, 'actor': {'objectType': 'something', 'id': '4321', 'published': '2012-07-05T12:00:00Z'}}], activities) diff --git a/tests/test_db.py b/tests/test_db.py new file mode 100644 index 0000000..8a783dc --- /dev/null +++ b/tests/test_db.py @@ -0,0 +1,514 @@ +from __future__ import absolute_import + +import copy +import datetime +import os + +from sqlalchemy import create_engine, sql +from sqlalchemy.exc import IntegrityError +from sunspear.activitystreams.models import Model +from sunspear.backends.database.db import * +from sunspear.exceptions import SunspearOperationNotSupportedException + +from nose.tools import assert_raises, eq_, ok_, raises + +DB_CONNECTION_STRING = os.environ.get('DB_CONNECTION_STRING', 'mysql://root:@localhost') +DB_TYPE = os.environ.get('DB_TYPE', 'mysql') +DB_USER = os.environ.get('DB_USER', 'root') +DB_PASS = os.environ.get('DB_PASSWORD', '') +DB_HOST = os.environ.get('DB_HOST', 'localhost') +DB_PORT = int(os.environ.get('DB_PORT', 3306)) + +DB_NAME = os.environ.get('DB_NAME', 'sunspear_test_database') + + +class TestDatabaseBackend(object): + @classmethod + def setUpClass(cls): + database_name = DB_NAME + cls._setup_db(database_name) + database_connection_string = cls.get_connection_string_with_database(database_name) + + cls._backend = DatabaseBackend(db_connection_string=database_connection_string, verbose=False) + cls._backend.drop_tables() + cls._engine = cls._backend.engine + cls.now = datetime.datetime.utcnow() + + @classmethod + def tearDownClass(cls): + database_name = DB_NAME + cls._cleanup_db(database_name) + + @classmethod + def get_connection_string(cls): + return '{0}://{1}:{2}@{3}:{4}'.format(DB_TYPE, DB_USER, DB_PASS, DB_HOST, DB_PORT) + + @classmethod + def get_connection_string_with_database(cls, database_name): + return '{0}/{1}?charset=utf8'.format(cls.get_connection_string(), database_name) + + @classmethod + def _cleanup_db(cls, db_name): + connection_string = cls.get_connection_string() + + # This engine just used to query for list of databases + engine = create_engine(connection_string) + + engine.execute("DROP DATABASE {};".format(db_name)) + + @classmethod + def _setup_db(cls, db_name): + connection_string = cls.get_connection_string() + + # This engine just used to query for list of databases + engine = create_engine(connection_string) + conn = engine.connect() + + # Query for existing databases + existing_databases = conn.execute("SHOW DATABASES;") + # Results are a list of single item tuples, so unpack each tuple + existing_databases = [d[0] for d in existing_databases] + + # Create database if not exists + if db_name not in existing_databases: + conn.execute("CREATE DATABASE {0} character set UTF8mb4 collate utf8mb4_bin".format(db_name)) + print("Created database {0}".format(db_name)) + + conn.close() + + def setUp(self): + self._backend.create_tables() + self._setup_objs() + self._setup_activities() + self._setup_db_schema_dicts() + + def tearDown(self): + self._backend.drop_tables() + + def _setup_db_schema_dicts(self): + self.test_db_schema_dicts = [{ + 'id': 'AxsdSG244BfduiIZ', + 'object_type': u'use\u0403', + 'display_name': u'\u019duman S', + 'content': u'Foo bar!\u03ee', + 'published': self._datetime_to_string(self.now), + 'image': { + 'url': 'https://www.google.com/cool_image.png', + 'displayName': u'Cool \u0268mage', + 'width': '500px', + 'height': '500px' + }, + 'other_data': { + 'foo': 'bar', + 'baz': u'go\u0298', + 'zoo': {'zee': 12, 'tim': {'zde': u'\u0268\u0298'}} + } + }] + + self.test_db_schema_dict = self.test_db_schema_dicts[0] + + def _setup_objs(self): + self.test_objs = [ + { + 'id': 'AxsdSG244BfduiIZ', + 'objectType': u'use\u0403', + 'displayName': u'\u019duman S', + 'content': u'Foo bar!\u03ee', + 'published': self._datetime_to_string(self.now), + 'image': { + 'url': 'https://www.google.com/cool_image.png', + 'displayName': u'Cool \u0268mage', + 'width': '500px', + 'height': '500px' + }, + 'foo': 'bar', + 'baz': u'go\u0298', + 'zoo': {'zee': 12, 'tim': {'zde': u'\u0268\u0298'}} + }, + { + 'id': '1000', + 'objectType': u'user', + 'displayName': u'\u019duman S2', + 'content': u'Foo bar!\u03ee', + 'published': self._datetime_to_string(self.now), + 'image': { + 'url': 'https://www.google.com/cool_image.png', + 'displayName': u'Cool \u0268mage', + 'width': '500px', + 'height': '500px' + }, + 'foo': 'bar', + 'baz': u'go\u0298', + 'zoo': {'zee': 12, 'tim': {'zde': u'\u0268\u0298'}} + }, + { + 'id': '1001', + 'objectType': u'user', + 'displayName': u'\u019duman S3', + 'content': u'Foo bar!\u03ee', + 'published': self._datetime_to_string(self.now), + 'image': { + 'url': 'https://www.google.com/cool_image.png', + 'displayName': u'Cool \u0268mage', + 'width': '500px', + 'height': '500px' + }, + 'foo': 'bar', + 'baz': u'go\u0298', + 'zoo': {'zee': 12, 'tim': {'zde': u'\u0268\u0298'}} + }, + { + 'id': '1002', + 'objectType': u'user', + 'displayName': u'\u019duman S4', + 'content': u'Foo bar!\u03ee', + 'published': self._datetime_to_string(self.now), + 'image': { + 'url': 'https://www.google.com/cool_image.png', + 'displayName': u'Cool \u0268mage', + 'width': '500px', + 'height': '500px' + }, + 'foo': 'bar', + 'baz': u'go\u0298', + 'zoo': {'zee': 12, 'tim': {'zde': u'\u0268\u0298'}} + } + ] + + self.test_obj = self.test_objs[0] + + def _setup_activities(self): + self.test_activities = [{ + 'id': 'WvgYP43bfg64fsdDHt3', + 'verb': 'join', + 'actor': 'user:1', + 'object': 'recognition:1', + 'target': 'badge:2', + 'author': 'user:435', + 'generator': 'mobile:phone:android', + 'provider': 'mobile:phone:android', + 'content': 'foo baz', + 'published': self._datetime_to_string(self.now), + 'updated': self._datetime_to_string(self.now), + 'icon': { + 'url': 'https://www.google.com/cool_image.png', + 'displayName': u'Cool \u0268mage', + 'width': '500px', + 'height': '500px', + 'id': 'icon:1' + }, + 'foo': 'bar', + 'baz': u'go\u0298', + 'zoo': {'zee': 12, 'tim': {'zde': u'\u0268\u0298'}}, + }] + + self.test_objs_for_activities = [{ + 'id': 'user:1', + 'objectType': u'use\u0403', + 'displayName': u'\u019duman S1', + 'content': u'Foo bar!\u03ee', + 'published': self._datetime_to_string(self.now), + 'image': { + 'url': 'https://www.google.com/cool_image.png', + 'displayName': u'Cool \u0268mage', + 'width': '500px', + 'height': '500px', + 'id': 'img:1', + }, + 'foo': 'bar', + 'baz': u'go\u0298', + 'zoo': {'zee': 12, 'tim': {'zde': u'\u0268\u0298'}} + }, { + 'id': 'recognition:1', + 'objectType': u'use\u0403', + 'displayName': u'\u019dRecognitionBadge', + 'content': u'Good Work on everything\u03ee', + 'published': self._datetime_to_string(self.now), + 'image': { + 'url': 'https://www.google.com/cool_image.png', + 'displayName': u'Cool \u0268mage', + 'width': '500px', + 'height': '500px', + 'id': 'img:2', + }, + 'foo': 'bar', + 'baz': u'go\u0298', + 'zoo': {'zee': 12, 'tim': {'zde': u'\u0268\u0298'}} + }, { + 'id': 'badge:2', + 'objectType': u'use\u0403', + 'displayName': u'\u019dAwesomeness', + 'content': u'Just for being awesome\u03ee', + 'published': self._datetime_to_string(self.now), + 'image': { + 'url': 'https://www.google.com/cool_image.png', + 'displayName': u'Cool \u0268mage', + 'width': '500px', + 'height': '500px', + 'id': 'img:3', + }, + 'foo': 'bar', + 'baz': u'go\u0298', + 'zoo': {'zee': 12, 'tim': {'zde': u'\u0268\u0298'}} + }, { + 'id': 'user:435', + 'objectType': u'use\u0403', + 'displayName': u'\u019duman S435', + 'content': u'Foo bar!\u03ee', + 'published': self._datetime_to_string(self.now), + 'image': { + 'url': 'https://www.google.com/cool_image.png', + 'displayName': u'Cool \u0268mage', + 'width': '500px', + 'height': '500px', + 'id': 'img:4', + }, + 'foo': 'bar', + 'baz': u'go\u0298', + 'zoo': {'zee': 12, 'tim': {'zde': u'\u0268\u0298'}} + }, { + 'id': 'mobile:phone:android', + 'objectType': u'androidmobilephone\u0403', + 'displayName': u'\u019dobile Phone Android', + 'content': u'Foo bar!\u03ee', + 'published': self._datetime_to_string(self.now), + 'image': { + 'url': 'https://www.google.com/cool_image.png', + 'displayName': u'Cool \u0268mage', + 'width': '500px', + 'height': '500px', + 'id': 'img:5', + }, + 'foo': 'bar', + 'baz': u'go\u0298', + 'zoo': {'zee': 12, 'tim': {'zde': u'\u0268\u0298'}} + }] + + self.test_activity = self.test_activities[0] + + self.hydrated_test_activity = self._build_hydrated_activity(self.test_activity, self.test_objs_for_activities) + + def _build_hydrated_activity(self, dehydrated_activity, objs): + hydrated_activity = copy.deepcopy(dehydrated_activity) + for obj_field in Model._object_fields: + if obj_field in hydrated_activity: + obj_id = hydrated_activity[obj_field] + obj = [obj for obj in objs if obj['id'] == obj_id][0] + hydrated_activity[obj_field] = obj + + return hydrated_activity + + def _insert_obj(self, obj): + db_obj = self._backend._obj_dict_to_db_schema(obj) + + objects_table = self._backend.objects_table + + self._engine.execute(objects_table.insert(), [ + db_obj + ]) + + @raises(SunspearOperationNotSupportedException) + def test_sample_test(self): + self._backend.clear_all_objects() + + def test__obj_dict_to_db_schema(self): + obj_dict = self.test_obj + obj_dict_copy = copy.deepcopy(obj_dict) + + db_schema_dict = self._backend._obj_dict_to_db_schema(obj_dict) + + # Confirm the original dict was not modified + eq_(obj_dict, obj_dict_copy) + + for obj_field, db_schema_field in DB_OBJ_FIELD_MAPPING.items(): + data = obj_dict[obj_field] + if obj_field in Model._datetime_fields: + data = self._backend._get_db_compatiable_date_string(data) + + eq_(data, db_schema_dict[db_schema_field]) + # Remove all "supported" fields. What we have left should be what went to `other_data` + obj_dict_copy.pop(obj_field) + + # Everything was placed in other_data + eq_(obj_dict_copy, db_schema_dict['other_data']) + + def test_db_schema_to_obj_dict(self): + db_schema_dict = self.test_db_schema_dict + db_schema_dict_copy = copy.deepcopy(db_schema_dict) + + obj_dict = self._backend._db_schema_to_obj_dict(db_schema_dict) + + # Confirm the original dict was not modified + eq_(db_schema_dict, db_schema_dict_copy) + + for obj_field, db_schema_field in DB_OBJ_FIELD_MAPPING.items(): + data = db_schema_dict[db_schema_field] + if obj_field in Model._datetime_fields: + data = self._backend._get_datetime_obj(data) + data = '{}Z'.format(data.isoformat()) + + eq_(data, obj_dict[obj_field]) + + for key, value in db_schema_dict['other_data'].items(): + eq_(obj_dict[key], value) + + def test_obj_create(self): + self._backend.obj_create(self.test_obj) + + obj_exists = self._engine.execute( + sql.select([sql.exists().where(self._backend.objects_table.c.id == self.test_obj['id'])])) + + ok_(obj_exists) + + def test_obj_exists(self): + self._insert_obj(self.test_obj) + + ok_(self._backend.obj_exists(self.test_obj)) + ok_(not self._backend.obj_exists('someunknownid')) + + def test_obj_delete(self): + objects_table = self._backend.objects_table + obj_id = self.test_obj['id'] + + self._insert_obj(self.test_obj) + + self._backend.obj_delete(obj_id) + + exists = self._engine.execute(sql.select([sql.exists().where(objects_table.c.id == obj_id)])).scalar() + ok_(not exists) + + def test_obj_get(self): + obj_id = self.test_obj['id'] + + self._insert_obj(self.test_obj) + + objs = self._backend.obj_get([obj_id]) + eq_(objs[0], self.test_obj) + + def test_activity_exists(self): + db_activity = self._backend._activity_dict_to_db_schema(self.test_activity) + db_objs = map(self._backend._obj_dict_to_db_schema, self.test_objs_for_activities) + + activities_table = self._backend.activities_table + objects_table = self._backend.objects_table + + self._engine.execute(objects_table.insert(), db_objs) + + self._engine.execute(activities_table.insert(), [ + db_activity + ]) + + ok_(self._backend.activity_exists(self.test_activity)) + ok_(not self._backend.activity_exists('someunknownid')) + + def test_activity_create(self): + db_objs = map(self._backend._obj_dict_to_db_schema, self.test_objs_for_activities) + + objects_table = self._backend.objects_table + self._engine.execute(objects_table.insert(), db_objs) + + self._backend.activity_create(self.test_activity) + + ok_(self._backend.activity_exists(self.test_activity)) + + def test_create_activity(self): + self._backend.create_activity(self.hydrated_test_activity) + ok_(self._backend.activity_exists(self.hydrated_test_activity)) + + def test_create_activity_with_audience_targeting(self): + db_obj = self._backend._obj_dict_to_db_schema(self.test_objs[3]) + objects_table = self._backend.objects_table + self._engine.execute(objects_table.insert(), db_obj) + + self.hydrated_test_activity['to'] = [self.test_objs[0]] + self.hydrated_test_activity['bto'] = [self.test_objs[1]] + self.hydrated_test_activity['cc'] = [self.test_objs[0], self.test_objs[1]] + self.hydrated_test_activity['bcc'] = [self.test_objs[2], self.test_objs[3]['id']] + + self._backend.create_activity(self.hydrated_test_activity) + + ok_(self._backend.activity_exists(self.hydrated_test_activity)) + + ok_(self._backend.obj_exists(self.test_objs[0])) + ok_(self._backend.obj_exists(self.test_objs[1])) + ok_(self._backend.obj_exists(self.test_objs[2])) + ok_(self._backend.obj_exists(self.test_objs[3])) + + def test_create_activity_with_already_existing_objs(self): + db_objs = map(self._backend._obj_dict_to_db_schema, self.test_objs_for_activities) + objects_table = self._backend.objects_table + self._engine.execute(objects_table.insert(), db_objs) + + self._backend.create_activity(self.hydrated_test_activity) + ok_(self._backend.activity_exists(self.hydrated_test_activity)) + + def test_create_activity_with_some_already_existing_objs(self): + db_objs = map(self._backend._obj_dict_to_db_schema, self.test_objs_for_activities) + objects_table = self._backend.objects_table + self._engine.execute(objects_table.insert(), db_objs[1:]) + + self._backend.create_activity(self.hydrated_test_activity) + ok_(self._backend.activity_exists(self.hydrated_test_activity)) + + def test_create_activity_with_only_ids_for_objs(self): + db_objs = map(self._backend._obj_dict_to_db_schema, self.test_objs_for_activities) + objects_table = self._backend.objects_table + self._engine.execute(objects_table.insert(), db_objs) + + self._backend.create_activity(self.test_activity) + ok_(self._backend.activity_exists(self.test_activity)) + + def test_create_activity_with_non_existing_objects_doesnt_work(self): + assert_raises(IntegrityError, self._backend.create_activity, self.test_activity) + ok_(not self._backend.activity_exists(self.test_activity)) + + def test_get_activities(self): + activity_copy = copy.deepcopy(self.hydrated_test_activity) + + self._backend.create_activity(self.hydrated_test_activity) + activity = self._backend.get_activity(self.hydrated_test_activity['id'])[0] + + # Updated is changed when an activity is saved + activity_copy['updated'] = activity['updated'] + + eq_(activity, activity_copy) + + def test_create_reply(self): + actor_id = '1234' + published_time = datetime.datetime.utcnow() + + actor = {"objectType": "something", "id": actor_id, "published": published_time} + + # create the activity + self._backend.create_activity(self.hydrated_test_activity) + + # now create a reply for the activity + reply_activity_dict, activity_obj_dict = self._backend.sub_activity_create( + self.hydrated_test_activity, actor, "This is a reply.", + sub_activity_verb='reply') + + sub_activity_exists = self._engine.execute(sql.select([sql.exists().where(self._backend.replies_table.c.id == reply_activity_dict['id'])])).scalar() + ok_(sub_activity_exists) + + def test_create_like(self): + actor_id = '1234' + published_time = datetime.datetime.utcnow() + + actor = {"objectType": "something", "id": actor_id, "published": published_time} + + # create the activity + self._backend.create_activity(self.hydrated_test_activity) + + # now create a reply for the activity + like_activity_dict, activity_obj_dict = self._backend.sub_activity_create( + self.hydrated_test_activity, actor, "This is a like.", + sub_activity_verb='like') + + sub_activity_exists = self._engine.execute(sql.select([sql.exists().where(self._backend.likes_table.c.id == like_activity_dict['id'])])).scalar() + ok_(sub_activity_exists) + + def _datetime_to_db_compatibal_str(self, datetime_instance): + return datetime_instance.strftime('%Y-%m-%d %H:%M:%S') + + def _datetime_to_string(self, datetime_instance): + return datetime_instance.strftime('%Y-%m-%dT%H:%M:%S') + "Z" diff --git a/tests/test_dotdict.py b/tests/test_dotdict.py index 602704e..f387db5 100644 --- a/tests/test_dotdict.py +++ b/tests/test_dotdict.py @@ -1,8 +1,8 @@ -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function, unicode_literals -from sunspear.lib.dotdict import dotdictify +from nose.tools import eq_, ok_, raises -from nose.tools import ok_, eq_, raises +from sunspear.lib.dotdict import dotdictify class TestDotDictify(object): diff --git a/tests/test_rfc3339.py b/tests/test_rfc3339.py index a7bc723..022720d 100644 --- a/tests/test_rfc3339.py +++ b/tests/test_rfc3339.py @@ -1,18 +1,18 @@ -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function, unicode_literals -from sunspear.lib.rfc3339 import rfc3339, _timezone, _utc_offset, _timedelta_to_seconds +import datetime +import time from nose.tools import eq_, ok_ -import datetime -import time +from sunspear.lib.rfc3339 import _timedelta_to_seconds, _timezone, _utc_offset, rfc3339 class TestRFC3339(object): - ''' + """ Test the use of the timezone saved locally. Since it is hard to test using doctest. - ''' + """ def setUp(self): local_utcoffset = _utc_offset(datetime.datetime.now(), True) @@ -22,7 +22,7 @@ def setUp(self): def test_datetime(self): d = datetime.datetime.now() eq_(rfc3339(d), - d.strftime('%Y-%m-%dT%H:%M:%S') + self.local_timezone) + d.strftime('%Y-%m-%dT%H:%M:%S') + self.local_timezone) def test_datetime_timezone(self): @@ -91,13 +91,13 @@ def test_timestamp_utc(self): self.local_timezone)) def test_before_1970(self): - d = datetime.date(1885, 01, 04) + d = datetime.date(1885, 0o1, 0o4) ok_(rfc3339(d).startswith('1885-01-04T00:00:00')) eq_(rfc3339(d, utc=True, use_system_timezone=False), '1885-01-04T00:00:00Z') def test_1920(self): - d = datetime.date(1920, 02, 29) + d = datetime.date(1920, 0o2, 29) x = rfc3339(d, utc=False, use_system_timezone=True) ok_(x.startswith('1920-02-29T00:00:00')) diff --git a/tests/test_backend.py b/tests/test_riak.py similarity index 91% rename from tests/test_backend.py rename to tests/test_riak.py index 7b160ef..f1ec793 100644 --- a/tests/test_backend.py +++ b/tests/test_riak.py @@ -1,23 +1,25 @@ -from __future__ import absolute_import +from __future__ import absolute_import, division, print_function, unicode_literals -from nose.tools import ok_, eq_, set_trace, raises +import datetime +import six from mock import MagicMock, call, ANY +import os -from sunspear.exceptions import SunspearValidationException from sunspear.aggregators.property import PropertyAggregator from sunspear.backends.riak import RiakBackend - -import datetime +from sunspear.compat import must_be_str +from sunspear.exceptions import SunspearValidationException +from nose.tools import eq_, ok_, raises riak_connection_options = { "nodes": [ - {'http_port': 8098, 'host': '127.0.0.1'}], + {'http_port': 8098, 'host': os.environ.get('RIAK_HOST', '127.0.0.1')}], 'protocol': 'http', - # "nodes": [{'host': '127.0.0.1', 'pb_port': 10017}, {'host': '127.0.0.1', 'pb_port': 10027}, {'host': '127.0.0.1', 'pb_port': 10037}], } class TestRiakBackend(object): + def setUp(self): backend = RiakBackend(**riak_connection_options) self._backend = backend @@ -172,7 +174,7 @@ def test_create_activity_stored_as_sparse(self): riak_obj = self._backend._activities.get('5') riak_obj_data = riak_obj.data - ok_(isinstance(riak_obj_data.get("target"), basestring)) + ok_(isinstance(riak_obj_data.get("target"), six.string_types)) def test_delete_activity(self): self._backend._activities.get('5').delete() @@ -730,8 +732,8 @@ def test_create_reply_maintains_dehydrate_state(self): self._backend.create_activity({"id": 5, "title": "Stream Item", "verb": "post", "actor": actor, "object": obj}) riak_obj_data = self._backend._activities.get(key="5").data - ok_(isinstance(riak_obj_data.get("actor"), basestring)) - ok_(isinstance(riak_obj_data.get("object"), basestring)) + ok_(isinstance(riak_obj_data.get("actor"), six.string_types)) + ok_(isinstance(riak_obj_data.get("object"), six.string_types)) #now create a reply for the activity reply_activity_dict, activity_obj_dict = self._backend.sub_activity_create( @@ -739,8 +741,8 @@ def test_create_reply_maintains_dehydrate_state(self): sub_activity_verb='reply') riak_obj_data = self._backend._activities.get(key="5").data - ok_(isinstance(riak_obj_data.get("actor"), basestring)) - ok_(isinstance(riak_obj_data.get("object"), basestring)) + ok_(isinstance(riak_obj_data.get("actor"), six.string_types)) + ok_(isinstance(riak_obj_data.get("object"), six.string_types)) def test_create_reply_with_extra_data(self): self._backend._activities.get('5').delete() @@ -843,8 +845,8 @@ def test_create_like_maintains_dehydrate_state(self): self._backend.create_activity({"id": 5, "title": "Stream Item", "verb": "post", "actor": actor, "object": obj}) riak_obj_data = self._backend._activities.get(key="5").data - ok_(isinstance(riak_obj_data.get("actor"), basestring)) - ok_(isinstance(riak_obj_data.get("object"), basestring)) + ok_(isinstance(riak_obj_data.get("actor"), six.string_types)) + ok_(isinstance(riak_obj_data.get("object"), six.string_types)) #now create a reply for the activity like_activity_dict, activity_obj_dict = self._backend.sub_activity_create( @@ -852,8 +854,8 @@ def test_create_like_maintains_dehydrate_state(self): sub_activity_verb='like') riak_obj_data = self._backend._activities.get(key="5").data - ok_(isinstance(riak_obj_data.get("actor"), basestring)) - ok_(isinstance(riak_obj_data.get("object"), basestring)) + ok_(isinstance(riak_obj_data.get("actor"), six.string_types)) + ok_(isinstance(riak_obj_data.get("object"), six.string_types)) def test_create_like(self): self._backend._activities.get('5').delete() @@ -950,15 +952,15 @@ def test_delete_like_maintains_dehydrated_state(self): 5, actor2_id, "", sub_activity_verb='like') riak_obj_data = self._backend._activities.get(key="5").data - ok_(isinstance(riak_obj_data.get("actor"), basestring)) - ok_(isinstance(riak_obj_data.get("object"), basestring)) + ok_(isinstance(riak_obj_data.get("actor"), six.string_types)) + ok_(isinstance(riak_obj_data.get("object"), six.string_types)) #now delete the like and make sure everything is ok: self._backend.sub_activity_delete(like_activity_dict['id'], 'like') riak_obj_data = self._backend._activities.get(key="5").data - ok_(isinstance(riak_obj_data.get("actor"), basestring)) - ok_(isinstance(riak_obj_data.get("object"), basestring)) + ok_(isinstance(riak_obj_data.get("actor"), six.string_types)) + ok_(isinstance(riak_obj_data.get("object"), six.string_types)) def test_reply_delete_maintains_dehydrated_state(self): self._backend._activities.get('5').delete() @@ -983,16 +985,16 @@ def test_reply_delete_maintains_dehydrated_state(self): 5, actor2_id, "This is a reply.", sub_activity_verb='reply') riak_obj_data = self._backend._activities.get(key="5").data - ok_(isinstance(riak_obj_data.get("actor"), basestring)) - ok_(isinstance(riak_obj_data.get("object"), basestring)) + ok_(isinstance(riak_obj_data.get("actor"), six.string_types)) + ok_(isinstance(riak_obj_data.get("object"), six.string_types)) #now delete the reply and make sure everything is ok: self._backend.sub_activity_delete( reply_activity_dict['id'], 'reply') riak_obj_data = self._backend._activities.get(key="5").data - ok_(isinstance(riak_obj_data.get("actor"), basestring)) - ok_(isinstance(riak_obj_data.get("object"), basestring)) + ok_(isinstance(riak_obj_data.get("actor"), six.string_types)) + ok_(isinstance(riak_obj_data.get("object"), six.string_types)) def test_reply_delete(self): self._backend._activities.get('5').delete() @@ -1576,7 +1578,7 @@ def test_get_activities_with_aggregation_pipline(self): activities = self._backend.activity_get(activity_ids, aggregation_pipeline=[PropertyAggregator(properties=['verb', 'actor'])]) - eq_([{u'id': u'7779', u'verb': u'like', u'target': {u'objectType': u'something', u'id': u'31415', u'published': u'2012-07-05T12:00:00Z'}, u'object': {u'inReplyTo': [], u'objectType': u'like', u'id': u'6669', u'published': u'2012-08-05T12:00:00Z'}, u'actor': {u'objectType': u'something', u'id': u'1234', u'published': u'2012-07-05T12:00:00Z'}}, {u'id': u'8889', u'verb': u'reply', u'target': {u'objectType': u'something', u'id': u'31415', u'published': u'2012-07-05T12:00:00Z'}, u'object': {u'content': u'This is my first reply', u'inReplyTo': [], u'objectType': u'reply', u'id': u'9999', u'published': u'2012-08-05T12:00:00Z'}, u'actor': {u'objectType': u'something', u'id': u'1234', u'published': u'2012-07-05T12:00:00Z'}}, {'grouped_by_attributes': ['verb', 'actor'], u'title': [u'Stream Item', u'Stream Item'], u'object': [{u'objectType': u'something', u'id': u'4353', u'published': u'2012-07-05T12:00:00Z'}, {u'published': u'2012-07-05T12:00:00Z', u'id': u'4353', u'objectType': u'something'}], u'actor': {u'published': u'2012-07-05T12:00:00Z', u'id': u'4321', u'objectType': u'something'}, u'verb': u'post', u'replies': [{u'totalItems': 2, u'items': [{u'verb': u'reply', u'actor': {u'objectType': u'something', u'id': u'1234', u'published': u'2012-07-05T12:00:00Z'}, u'object': {u'target': {u'objectType': u'something', u'id': u'31415', u'published': u'2012-07-05T12:00:00Z'}, u'object': {u'content': u'This is my first reply', u'inReplyTo': [], u'objectType': u'reply', u'id': u'9999', u'published': u'2012-08-05T12:00:00Z'}, u'actor': {u'objectType': u'something', u'id': u'1234', u'published': u'2012-07-05T12:00:00Z'}, u'verb': u'reply', u'id': u'8889', u'objectType': u'activity'}}, {u'verb': u'reply', u'actor': {u'objectType': u'something', u'id': u'1234', u'published': u'2012-07-05T12:00:00Z'}, u'object': {u'target': {u'objectType': u'something', u'id': u'1234', u'published': u'2012-07-05T12:00:00Z'}, u'object': {u'content': u'This is my second reply', u'inReplyTo': [], u'objectType': u'reply', u'id': u'9998', u'published': u'2012-08-05T12:05:00Z'}, u'actor': {u'objectType': u'something', u'id': u'4321', u'published': u'2012-07-05T12:00:00Z'}, u'verb': u'reply', u'id': u'8888', u'objectType': u'activity'}}]}, {u'totalItems': 2, u'items': [{u'verb': u'reply', u'actor': {u'objectType': u'something', u'id': u'1234', u'published': u'2012-07-05T12:00:00Z'}, u'object': {u'target': {u'objectType': u'something', u'id': u'31415', u'published': u'2012-07-05T12:00:00Z'}, u'object': {u'content': u'This is my first reply', u'inReplyTo': [], u'objectType': u'reply', u'id': u'9999', u'published': u'2012-08-05T12:00:00Z'}, u'actor': {u'objectType': u'something', u'id': u'1234', u'published': u'2012-07-05T12:00:00Z'}, u'verb': u'reply', u'id': u'8889', u'objectType': u'activity'}}, {u'verb': u'reply', u'actor': {u'objectType': u'something', u'id': u'1234', u'published': u'2012-07-05T12:00:00Z'}, u'object': {u'target': {u'objectType': u'something', u'id': u'1234', u'published': u'2012-07-05T12:00:00Z'}, u'object': {u'content': u'This is my second reply', u'inReplyTo': [], u'objectType': u'reply', u'id': u'9998', u'published': u'2012-08-05T12:05:00Z'}, u'actor': {u'objectType': u'something', u'id': u'4321', u'published': u'2012-07-05T12:00:00Z'}, u'verb': u'reply', u'id': u'8888', u'objectType': u'activity'}}]}], u'id': [u'5555', u'5556'], 'grouped_by_values': [u'post', {u'published': u'2012-07-05T12:00:00Z', u'id': u'4321', u'objectType': u'something'}]}, {u'id': u'7778', u'verb': u'like', u'target': {u'objectType': u'something', u'id': u'31415', u'published': u'2012-07-05T12:00:00Z'}, u'object': {u'inReplyTo': [], u'objectType': u'like', u'id': u'6669', u'published': u'2012-08-05T12:00:00Z'}, u'actor': {u'objectType': u'something', u'id': u'1234', u'published': u'2012-07-05T12:00:00Z'}}, {u'id': u'8888', u'verb': u'reply', u'target': {u'objectType': u'something', u'id': u'1234', u'published': u'2012-07-05T12:00:00Z'}, u'object': {u'content': u'This is my second reply', u'inReplyTo': [], u'objectType': u'reply', u'id': u'9998', u'published': u'2012-08-05T12:05:00Z'}, u'actor': {u'objectType': u'something', u'id': u'4321', u'published': u'2012-07-05T12:00:00Z'}}], activities) + eq_([{'id': '7779', 'verb': 'like', 'target': {'objectType': 'something', 'id': '31415', 'published': '2012-07-05T12:00:00Z'}, 'object': {'inReplyTo': [], 'objectType': 'like', 'id': '6669', 'published': '2012-08-05T12:00:00Z'}, 'actor': {'objectType': 'something', 'id': '1234', 'published': '2012-07-05T12:00:00Z'}}, {'id': '8889', 'verb': 'reply', 'target': {'objectType': 'something', 'id': '31415', 'published': '2012-07-05T12:00:00Z'}, 'object': {'content': 'This is my first reply', 'inReplyTo': [], 'objectType': 'reply', 'id': '9999', 'published': '2012-08-05T12:00:00Z'}, 'actor': {'objectType': 'something', 'id': '1234', 'published': '2012-07-05T12:00:00Z'}}, {'grouped_by_attributes': ['verb', 'actor'], 'title': ['Stream Item', 'Stream Item'], 'object': [{'objectType': 'something', 'id': '4353', 'published': '2012-07-05T12:00:00Z'}, {'published': '2012-07-05T12:00:00Z', 'id': '4353', 'objectType': 'something'}], 'actor': {'published': '2012-07-05T12:00:00Z', 'id': '4321', 'objectType': 'something'}, 'verb': 'post', 'replies': [{'totalItems': 2, 'items': [{'verb': 'reply', 'actor': {'objectType': 'something', 'id': '1234', 'published': '2012-07-05T12:00:00Z'}, 'object': {'target': {'objectType': 'something', 'id': '31415', 'published': '2012-07-05T12:00:00Z'}, 'object': {'content': 'This is my first reply', 'inReplyTo': [], 'objectType': 'reply', 'id': '9999', 'published': '2012-08-05T12:00:00Z'}, 'actor': {'objectType': 'something', 'id': '1234', 'published': '2012-07-05T12:00:00Z'}, 'verb': 'reply', 'id': '8889', 'objectType': 'activity'}}, {'verb': 'reply', 'actor': {'objectType': 'something', 'id': '1234', 'published': '2012-07-05T12:00:00Z'}, 'object': {'target': {'objectType': 'something', 'id': '1234', 'published': '2012-07-05T12:00:00Z'}, 'object': {'content': 'This is my second reply', 'inReplyTo': [], 'objectType': 'reply', 'id': '9998', 'published': '2012-08-05T12:05:00Z'}, 'actor': {'objectType': 'something', 'id': '4321', 'published': '2012-07-05T12:00:00Z'}, 'verb': 'reply', 'id': '8888', 'objectType': 'activity'}}]}, {'totalItems': 2, 'items': [{'verb': 'reply', 'actor': {'objectType': 'something', 'id': '1234', 'published': '2012-07-05T12:00:00Z'}, 'object': {'target': {'objectType': 'something', 'id': '31415', 'published': '2012-07-05T12:00:00Z'}, 'object': {'content': 'This is my first reply', 'inReplyTo': [], 'objectType': 'reply', 'id': '9999', 'published': '2012-08-05T12:00:00Z'}, 'actor': {'objectType': 'something', 'id': '1234', 'published': '2012-07-05T12:00:00Z'}, 'verb': 'reply', 'id': '8889', 'objectType': 'activity'}}, {'verb': 'reply', 'actor': {'objectType': 'something', 'id': '1234', 'published': '2012-07-05T12:00:00Z'}, 'object': {'target': {'objectType': 'something', 'id': '1234', 'published': '2012-07-05T12:00:00Z'}, 'object': {'content': 'This is my second reply', 'inReplyTo': [], 'objectType': 'reply', 'id': '9998', 'published': '2012-08-05T12:05:00Z'}, 'actor': {'objectType': 'something', 'id': '4321', 'published': '2012-07-05T12:00:00Z'}, 'verb': 'reply', 'id': '8888', 'objectType': 'activity'}}]}], 'id': ['5555', '5556'], 'grouped_by_values': ['post', {'published': '2012-07-05T12:00:00Z', 'id': '4321', 'objectType': 'something'}]}, {'id': '7778', 'verb': 'like', 'target': {'objectType': 'something', 'id': '31415', 'published': '2012-07-05T12:00:00Z'}, 'object': {'inReplyTo': [], 'objectType': 'like', 'id': '6669', 'published': '2012-08-05T12:00:00Z'}, 'actor': {'objectType': 'something', 'id': '1234', 'published': '2012-07-05T12:00:00Z'}}, {'id': '8888', 'verb': 'reply', 'target': {'objectType': 'something', 'id': '1234', 'published': '2012-07-05T12:00:00Z'}, 'object': {'content': 'This is my second reply', 'inReplyTo': [], 'objectType': 'reply', 'id': '9998', 'published': '2012-08-05T12:05:00Z'}, 'actor': {'objectType': 'something', 'id': '4321', 'published': '2012-07-05T12:00:00Z'}}], activities) class TestIndexes(object): @@ -1591,10 +1593,10 @@ def test_set_sub_item_indexes(self): self._backend.set_activity_indexes(riak_obj_mock) calls = [ - call.add_index('verb_bin', 'post'), - call.add_index('actor_bin', '1234'), - call.add_index('object_bin', '5678'), - call.add_index('target_bin', '4333'), + call.add_index(must_be_str('verb_bin'), 'post'), + call.add_index(must_be_str('actor_bin'), '1234'), + call.add_index(must_be_str('object_bin'), '5678'), + call.add_index(must_be_str('target_bin'), '4333'), ] riak_obj_mock.assert_has_calls(calls, any_order=True) @@ -1607,7 +1609,7 @@ def test_set_sub_item_indexes_reply(self): self._backend.set_sub_item_indexes(riak_obj_mock, activity_id=1234) calls = [ - call.add_index('inreplyto_bin', ANY), + call.add_index(must_be_str('inreplyto_bin'), ANY), ] riak_obj_mock.assert_has_calls(calls, any_order=True) @@ -1627,8 +1629,8 @@ def test_set_general_indexes_not_already_created_set(self): self._backend.set_general_indexes(riak_obj_mock) calls = [ - call.add_index('timestamp_int', ANY), - call.add_index('modified_int', ANY), + call.add_index(must_be_str('timestamp_int'), ANY), + call.add_index(must_be_str('modified_int'), ANY), ] riak_obj_mock.assert_has_calls(calls, any_order=True) @@ -1636,12 +1638,12 @@ def test_set_general_indexes_not_already_created_set(self): def test_set_general_indexes_already_created(self): riak_obj_mock = MagicMock() - riak_obj_mock.indexes = [('timestamp_int', 12343214,)] + riak_obj_mock.indexes = [(must_be_str('timestamp_int'), 12343214,)] self._backend.set_general_indexes(riak_obj_mock) calls = [ - call.add_index('modified_int', ANY), + call.add_index(must_be_str('modified_int'), ANY), ] riak_obj_mock.assert_has_calls(calls, any_order=True) @@ -1655,8 +1657,8 @@ def test_create_obj_indexes(self): riak_obj = self._backend._objects.get(key=actstream_obj['id']) riak_obj.data - ok_(filter(lambda x: x[0] == 'timestamp_int', riak_obj.indexes) != []) - ok_(filter(lambda x: x[0] == 'modified_int', riak_obj.indexes) != []) + ok_([x for x in riak_obj.indexes if x[0] == 'timestamp_int'] != []) + ok_([x for x in riak_obj.indexes if x[0] == 'modified_int'] != []) def test_create_activity_indexes(self): self._backend._activities.get('5').delete() @@ -1678,8 +1680,8 @@ def test_create_activity_indexes(self): riak_obj = self._backend._activities.get(key=act_obj_dict['id']) riak_obj.data - ok_(filter(lambda x: x[0] == 'timestamp_int', riak_obj.indexes) != []) - ok_(filter(lambda x: x[0] == 'modified_int', riak_obj.indexes) != []) + ok_([x for x in riak_obj.indexes if x[0] == 'timestamp_int'] != []) + ok_([x for x in riak_obj.indexes if x[0] == 'modified_int'] != []) eq_(filter(lambda x: x[0] == 'verb_bin', riak_obj.indexes)[0][1], 'post') eq_(filter(lambda x: x[0] == 'actor_bin', riak_obj.indexes)[0][1], actor_id) eq_(filter(lambda x: x[0] == 'object_bin', riak_obj.indexes)[0][1], object_id) @@ -1703,20 +1705,19 @@ def test_create_sub_activity_indexes(self): self._backend.create_obj(actor2) - #create the activity + # create the activity self._backend.create_activity({"id": 5, "title": "Stream Item", "verb": "post", "actor": actor, "object": obj}) - #now create a reply for the activity + # now create a reply for the activity like_activity_dict, activity_obj_dict = self._backend.sub_activity_create( 5, actor2_id, "", sub_activity_verb='like') riak_obj = self._backend._activities.get(key=like_activity_dict['id']) riak_obj.data - ok_(filter(lambda x: x[0] == 'timestamp_int', riak_obj.indexes) != []) - ok_(filter(lambda x: x[0] == 'modified_int', riak_obj.indexes) != []) + ok_([x for x in riak_obj.indexes if x[0] == 'timestamp_int'] != []) + ok_([x for x in riak_obj.indexes if x[0] == 'modified_int'] != []) eq_(filter(lambda x: x[0] == 'verb_bin', riak_obj.indexes)[0][1], 'like') eq_(filter(lambda x: x[0] == 'actor_bin', riak_obj.indexes)[0][1], actor2_id) eq_(filter(lambda x: x[0] == 'object_bin', riak_obj.indexes)[0][1], like_activity_dict['object']['id']) eq_(filter(lambda x: x[0] == 'inreplyto_bin', riak_obj.indexes)[0][1], '5') -