"""

**********

Provenance

**********

The Activity object represents the source of a data set or the data processing

steps used to produce it. Using `W3C provenance ontology <http://www.w3.org/2011/prov/wiki/Main_Page>`_

terms, a result is **generated by** a combination of data and code which are either

**used** or **executed**.

~~~~~~~

Imports

~~~~~~~

::

from synapseclient import Activity

~~~~~~~~~~~~~~~~~~~~~~~~~~~

Creating an activity object

~~~~~~~~~~~~~~~~~~~~~~~~~~~

::

act = Activity(name='clustering',

description='whizzy clustering',

used=['syn1234','syn1235'],

executed='syn4567')

Here, syn1234 and syn1235 might be two types of measurements on a common set of

samples. Some whizzy clustering code might be referred to by syn4567. The used and

executed can reference entities in Synapse or URLs.

Alternatively, you can build an activity up piecemeal::

act = Activity(name='clustering', description='whizzy clustering')

act.used(['syn12345', 'syn12346'])

act.executed('https://raw.githubusercontent.com/Sage-Bionetworks/synapsePythonClient/develop/tests/unit/unit_test_client.py')

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

Storing entities with provenance

~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~

The activity can be passed in when storing an Entity to set the Entity's provenance::

clustered_samples = syn.store(clustered_samples, activity=act)

We've now recorded that `clustered_samples` is the output of our whizzy clustering algorithm

applied to the data stored in syn1234 and syn1235.

~~~~~~~~~~~~~~~~~~~~~

Recording data source

~~~~~~~~~~~~~~~~~~~~~

The :py:meth:`synapseclient.Synapse.store` has shortcuts for specifying the used and executed lists directly. For example, when

storing a data entity, it's a good idea to record its source::

excellent_data = syn.store(excellent_data,

activityName='data-r-us'

activityDescription='downloaded from data-r-us',

used='http://data-r-us.com/excellent/data.xyz')

~~~~~~~~

Activity

~~~~~~~~

.. autoclass:: synapseclient.activity.Activity

:members:

"""

from __future__ import absolute_import

from __future__ import division

from __future__ import print_function

from __future__ import unicode_literals

import six

import collections

from synapseclient.utils import is_url, id_of

from synapseclient.entity import is_synapse_entity

from synapseclient.exceptions import *

def is_used_entity(x):

"""Returns True if the given object represents a UsedEntity."""

# A UsedEntity must be a dictionary with a 'reference' field, with a 'targetId' field

if not isinstance(x, collections.Mapping) \

or 'reference' not in x \

or 'targetId' not in x['reference']:

return False

# Must only have three keys

if not all(key in ('reference', 'wasExecuted', 'concreteType') for key in x.keys()):

return False

# 'reference' field can only have two keys

if not all(key in ('targetId', 'targetVersionNumber') for key in x['reference'].keys()):

return False

return True

def is_used_url(x):

"""Returns True if the given object represents a UsedURL."""

# A UsedURL must be a dictionary with a 'url' field

if not isinstance(x, collections.Mapping) or 'url' not in x:

return False

# Must only have four keys

if not all(key in ('url', 'name', 'wasExecuted', 'concreteType') for key in x.keys()):

return False

return True

def _get_any_bad_args(badargs, dictionary):

"""Returns the intersection of 'badargs' and the non-Null keys of 'dictionary'."""

return list(illegal for illegal in badargs \

if illegal in dictionary and dictionary[illegal] is not None)

def _raise_incorrect_used_usage(badargs, message):

"""Raises an informative exception about Activity.used()."""

if any(badargs):

raise SynapseMalformedEntityError("The parameter%s '%s' %s not allowed in combination with a %s." \

% ("s" if len(badargs) > 1 else "", \

badargs, \

"are" if len(badargs) > 1 else "is", \

message))

class Activity(dict):

"""

Represents the provenance of a Synapse Entity.

:param name: name of the Activity

:param description: a short text description of the Activity

:param used: Either a list of:

- `reference objects <http://rest.synapse.org/org/sagebionetworks/repo/model/Reference.html>`_ (e.g. ``[{'targetId':'syn123456', 'targetVersionNumber':1}]``)

- a list of Synapse Entities or Entity IDs

- a list of URL's

:param executed: A code resource that was executed to generate the Entity.

:param data: A dictionary representation of an Activity,

with fields 'name', 'description' and 'used'

(a list of reference objects)

See also: `Provenance in Synapse <https://sagebionetworks.jira.com/wiki/display/PLFM/Analysis+Provenance+in+Synapse>`_

and the `W3C's provenance ontology <http://www.w3.org/TR/prov-o/>`_

"""

## TODO: make constructors from JSON consistent across objects

def __init__(self, name=None, description=None, used=None, executed=None, data={}):

super(Activity, self).__init__(data)

if 'used' not in self:

self['used'] = []

if name is not None:

self['name'] = name

if description is not None:

self['description'] = description

if used is not None:

self.used(used)

if executed is not None:

self.executed(executed)

def used(self, target=None, targetVersion=None, wasExecuted=None, url=None, name=None):

"""

Add a resource used by the activity.

This method tries to be as permissive as possible. It accepts a string which might

be a synapse ID or a URL, a synapse entity, a UsedEntity or UsedURL dictionary or

a list containing any combination of these.

In addition, named parameters can be used to specify the fields of either a

UsedEntity or a UsedURL. If target and optionally targetVersion are specified,

create a UsedEntity. If url and optionally name are specified, create a UsedURL.

It is an error to specify both target/targetVersion parameters and url/name

parameters in the same call. To add multiple UsedEntities and UsedURLs, make a

separate call for each or pass in a list.

In case of conflicting settings for wasExecuted both inside an object and with a

parameter, the parameter wins. For example, this UsedURL will have wasExecuted set

to False::

activity.used({'url':'http://google.com', 'name':'Goog', 'wasExecuted':True}, wasExecuted=False)

Entity examples::

activity.used('syn12345')

activity.used(entity)

activity.used(target=entity, targetVersion=2)

activity.used(codeEntity, wasExecuted=True)

activity.used({'reference':{'target':'syn12345', 'targetVersion':1}, 'wasExecuted':False})

URL examples::

activity.used('http://mydomain.com/my/awesome/data.RData')

activity.used(url='http://mydomain.com/my/awesome/data.RData', name='Awesome Data')

activity.used(url='https://github.com/joe_hacker/code_repo', name='Gnarly hacks', wasExecuted=True)

activity.used({'url':'https://github.com/joe_hacker/code_repo', 'name':'Gnarly hacks'}, wasExecuted=True)

List example::

activity.used(['syn12345', 'syn23456', entity,

{'reference':{'target':'syn100009', 'targetVersion':2}, 'wasExecuted':True},

'http://mydomain.com/my/awesome/data.RData'])

"""

# -- A list of targets

if isinstance(target, list):

badargs = _get_any_bad_args(['targetVersion', 'url', 'name'], locals())

_raise_incorrect_used_usage(badargs, 'list of used resources')

for item in target:

self.used(item, wasExecuted=wasExecuted)

return

# -- UsedEntity

elif is_used_entity(target):

badargs = _get_any_bad_args(['targetVersion', 'url', 'name'], locals())

_raise_incorrect_used_usage(badargs, 'dictionary representing a used resource')

resource = target

if 'concreteType' not in resource:

resource['concreteType'] = 'org.sagebionetworks.repo.model.provenance.UsedEntity'

# -- Used URL

elif is_used_url(target):

badargs = _get_any_bad_args(['targetVersion', 'url', 'name'], locals())

_raise_incorrect_used_usage(badargs, 'URL')

resource = target

if 'concreteType' not in resource:

resource['concreteType'] = 'org.sagebionetworks.repo.model.provenance.UsedURL'

# -- Synapse Entity

elif is_synapse_entity(target):

badargs = _get_any_bad_args(['url', 'name'], locals())

_raise_incorrect_used_usage(badargs, 'Synapse entity')

reference = {'targetId':target['id']}

if 'versionNumber' in target:

reference['targetVersionNumber'] = target['versionNumber']

if targetVersion:

reference['targetVersionNumber'] = int(targetVersion)

resource = {'reference':reference, 'concreteType':'org.sagebionetworks.repo.model.provenance.UsedEntity'}

# -- URL parameter

elif url:

badargs = _get_any_bad_args(['target', 'targetVersion'], locals())

_raise_incorrect_used_usage(badargs, 'URL')

resource = {'url':url, 'name':name if name else target, 'concreteType':'org.sagebionetworks.repo.model.provenance.UsedURL'}

# -- URL as a string

elif is_url(target):

badargs = _get_any_bad_args(['targetVersion'], locals())

_raise_incorrect_used_usage(badargs, 'URL')

resource = {'url':target, 'name':name if name else target, 'concreteType':'org.sagebionetworks.repo.model.provenance.UsedURL'}

# -- Synapse Entity ID (assuming the string is an ID)

elif isinstance(target, six.string_types):

badargs = _get_any_bad_args(['url', 'name'], locals())

_raise_incorrect_used_usage(badargs, 'Synapse entity')

reference = {'targetId':target}

if targetVersion:

reference['targetVersionNumber'] = int(targetVersion)

resource = {'reference':reference, 'concreteType':'org.sagebionetworks.repo.model.provenance.UsedEntity'}

else:

raise SynapseError('Unexpected parameters in call to Activity.used().')

# Set wasExecuted

if wasExecuted is None:

# Default to False

if 'wasExecuted' not in resource:

resource['wasExecuted'] = False

else:

# wasExecuted parameter overrides setting in an object

resource['wasExecuted'] = wasExecuted

# Add the used resource to the activity

self['used'].append(resource)

def usedEntity(self, target, targetVersion=None, wasExecuted=False):

"""

**Deprecated**

See :py:func:`synapseclient.Activity.used`.

"""

self.used(target=target, targetVersion=targetVersion, wasExecuted=wasExecuted)

def usedURL(self, url, name=None, wasExecuted=False):

"""

**Deprecated**

See :py:func:`synapseclient.Activity.used`.

"""

self.used(url=url, name=name, wasExecuted=wasExecuted)

def executed(self, target=None, targetVersion=None, url=None, name=None):

"""

Add a code resource that was executed during the activity.

See :py:func:`synapseclient.activity.Activity.used`

"""

self.used(target=target, targetVersion=targetVersion, url=url, name=name, wasExecuted=True)

def __str__(self):

#user = syn.getUserProfile(self['createdBy'])

#str = ' Added by: %s %s (%s)' %(user['firstName'], user['lastName'], user['userName']))

str = '%s\n Executed:\n' % self.get('name', '')

for source in [source for source in self['used'] if source.get('wasExecuted', False)]:

if source['concreteType'].endswith('UsedURL'):

str +=' %s\n' %source['name']

else: #It is an entity for now

str +=' %s.%i\n' %(source['reference']['targetId'], source['reference']['targetVersionNumber'])

str += ' Used:\n'

for source in [source for source in self['used'] if not source.get('wasExecuted', False)]:

if source['concreteType'].endswith('UsedURL'):

str += ' %s\n' %source['name']

else: #It is an entity for now

str += ' %s.%i\n' %(source['reference']['targetId'], source['reference']['targetVersionNumber'])

return str