Test dev

.github/workflows/dev-test.yml

@@ -0,0 +1,32 @@
+name: Dev Test/Deploy
+
+on:
+  push: {}
+  pull_request:
+    branches: dev
+
+jobs:
+  test:
+    uses: openworm/owmeta/.github/workflows/test.yml@github-workflows
+    with:
+      coveralls: ${{github.event_name != 'schedule'}}
+      ref: ${{ github.ref }}
+    secrets:
+      my_github_token: ${{secrets.GITHUB_TOKEN}}
+      pubmed_api_key: ${{secrets.PUBMED_API_KEY}}
+  dev-deploy:
+    if: github.event_name == 'push' && github.ref == 'refs/heads/dev'
+    uses: openworm/owmeta-core/.github/workflows/deploy.yml@github-workflows
+    needs: test
+    with:
+      ref: refs/heads/dev
+    secrets:
+      twine_token: ${{secrets.TWINE_PASSWORD}}
+
+  coveralls:
+    if: github.event_name == 'push'
+    needs: test
+    uses: openworm/owmeta/.github/workflows/coveralls-upload.yml@github-workflows
+    secrets:
+      my_github_token: ${{secrets.GITHUB_TOKEN}}
+

.github/workflows/scheduled-dev-build.yml

@@ -0,0 +1,13 @@
+name: Build develop
+
+on:
+  schedule:
+    - cron:  '39 12 10,23 * *'
+
+jobs:
+  test:
+    uses: openworm/owmeta/.github/workflows/test.yml@github-workflows
+    with:
+      ref: refs/heads/dev
+    secrets:
+      pubmed_api_key: ${{secrets.PUBMED_API_KEY}}

.github/workflows/scheduled-master-build.yml

@@ -0,0 +1,13 @@
+name: Build master
+
+on:
+  schedule:
+    - cron:  '13 2 10,23 * *'
+
+jobs:
+  test:
+    uses: openworm/owmeta/.github/workflows/test.yml@github-workflows
+    with:
+      ref: refs/heads/master
+    secrets:
+      pubmed_api_key: ${{secrets.PUBMED_API_KEY}}

.travis.yml

@@ -19,7 +19,7 @@ before_install:
 
 install:
 - pip install .
-- owm clone https://github.com/openworm/OpenWormData.git
+- owm clone https://github.com/openworm/OpenWormData.git --branch owmeta
 - |
   if [ $BUNDLE_TESTS ] ; then
     owm bundle register ./owmeta-schema-bundle.yml

README.md

@@ -1,8 +1,8 @@
-[![Build Status](https://travis-ci.org/openworm/owmeta.png?branch=dev)](https://travis-ci.org/openworm/owmeta/builds)
+[![Build Status](https://github.com/openworm/owmeta/actions/workflows/dev-test.yml/badge.svg)](https://github.com/openworm/owmeta/actions/workflows/dev-test.yml)
 [![Docs](https://readthedocs.org/projects/owmeta/badge/?version=latest)](https://owmeta.readthedocs.io/en/latest)
 [![Coverage Status](https://coveralls.io/repos/github/openworm/owmeta/badge.svg?branch=dev)](https://coveralls.io/github/openworm/owmeta?branch=dev)
 
-owmeta
+owmeta 
 ======
 <img width="1207" alt="pyow_in_overview" src="openworm-overview.png">
 
@@ -82,7 +82,7 @@ This project holds a working-copy of the database. You can retrieve it by
 executing the following command line after owmeta installation:
 
 ```bash
-owm clone https://github.com/openworm/OpenWormData.git
+owm clone https://github.com/openworm/OpenWormData.git --branch owmeta
 ```
 
 This command should create a directory `.owm` in your current working
@@ -170,16 +170,17 @@ data and models to corresponding articles from peer-reviewed literature:
 >>> evctx = conn(Context)(ident='http://example.org/evidence/context')
 
 # Make a context for defining domain knowledge
->>> dctx = conn(Context)(ident='http://example.org/data/context')
+>>> dctx = evctx(Context)(ident='http://example.org/data/context')
 >>> doc = evctx(Document)(key="Sulston83", author='Sulston et al.', date='1983')
 >>> e = evctx(Evidence)(key="Sulston83", reference=doc)
 >>> avdl = dctx(Neuron)(name="AVDL")
 >>> avdl.lineageName("AB alaaapalr")
 owmeta_core.statement.Statement(subj=Neuron(ident=rdflib.term.URIRef('http://data.openworm.org/sci/bio/Neuron#AVDL')), prop=owmeta.cell.Cell_lineageName(owner=Neuron(ident=rdflib.term.URIRef('http://data.openworm.org/sci/bio/Neuron#AVDL'))), obj=owmeta_core.dataobject_property.ContextualizedPropertyValue(rdflib.term.Literal('AB alaaapalr')), context=owmeta_core.context.Context(ident="http://example.org/data/context"))
 >>> e.supports(dctx.rdf_object)
 owmeta_core.statement.Statement(subj=Evidence(ident=rdflib.term.URIRef('http://data.openworm.org/Evidence#Sulston83')), prop=owmeta.evidence.Evidence_supports(owner=Evidence(ident=rdflib.term.URIRef('http://data.openworm.org/Evidence#Sulston83'))), obj=ContextDataObject(ident=rdflib.term.URIRef('http://example.org/data/context')), context=owmeta_core.context.Context(ident="http://example.org/evidence/context"))
->>> dctx.save_context()
->>> evctx.save_context()
+>>> with conn.transaction_manager:
+...     dctx.save_context()
+...     evctx.save_context()
 
 ```
 
@@ -267,9 +268,10 @@ Finally, when you're done accessing the database, be sure to disconnect from it:
 
 ```
 
-More examples can be found
-[here](http://owm-doc.readthedocs.org/en/latest/making_dataObjects.html) and
-[here](https://github.com/openworm/owmeta/tree/master/examples).
+More examples can be found [in the owmeta-core
+documentation](https://owmeta-core.readthedocs.io/en/latest/making_dataObjects.html)
+and [in the ./examples directory of the owmeta Git
+repository](https://github.com/openworm/owmeta/tree/master/examples).
 
 Documentation
 -------------

ci-install.sh

@@ -0,0 +1,25 @@
+#!/bin/sh -ex
+
+pip install -e .
+pip install coveralls
+pip install -r test-requirements.txt
+owm clone https://github.com/openworm/OpenWormData.git --branch owmeta
+if [ $BUNDLE_TESTS ] ; then
+    owm bundle register ./owmeta-schema-bundle.yml
+    owm bundle register ./owmeta-data-bundle.yml
+    owm bundle install openworm/owmeta-schema
+    owm bundle install openworm/owmeta-data
+fi
+if [ "$PYTHON_VERSION" = "3.6" ] ; then
+    # pyparsing 3.0 no longer allows setting the .name attribute for
+    # TokenConverter, but the fix for that only goes in RDFLib 6 which
+    # does not support Python 3.6. See
+    # https://github.com/RDFLib/rdflib/issues/1190 and
+    # https://github.com/rdflib/rdflib/issues/1370 for details on the issue.
+    #
+    # Since we're a library, it's up to the user how to resolve this
+    # (e.g., ignore and do not use SPARQL) so we do not impose any
+    # version constraints in package metadata, just adjust so the build
+    # passes
+    pip install 'pyparsing<3.0.0'
+fi

ci-test.sh

@@ -4,7 +4,6 @@ pt () {
     sh -c "pytest --cov=owmeta $*"
 }
 
-
 COVERAGES=""
 
 add_coverage () {

deploy.sh

@@ -0,0 +1,12 @@
+#!/bin/bash -xe
+HEAD_REV=$(git rev-parse HEAD)
+git ls-remote origin refs/heads/dev | grep -q -E "^$HEAD_REV" || \
+    ( echo "Not deploying since we aren't on the 'dev' branch" >&2 && \
+    exit 0 )
+
+git log --format=%s -n 1 "$HEAD_REV" | grep -E -q '(^MINOR:)|(\[skip-deploy\])' && exit 0
+
+date=$(date +"%Y%m%d%H%M%S")
+sed -i -r "s/__version__ = '([^']+)\\.dev0'/__version__ = '\\1.dev$date'/" owmeta/__init__.py
+python setup.py egg_info sdist bdist_wheel
+twine upload -c "Built by CI. Uploaded after $(date +"%Y-%m-%d %H:%M:%S")" dist/owmeta*tar.gz dist/owmeta*whl

docs/adding_data.rst

@@ -152,7 +152,7 @@ using contexts. The code below shows how to do that::
 
    >>> # Serialize the data in the nquads format so we can see that all of our
    >>> # statements are in the proper context
-   >>> print(g.serialize(format='nquads').decode('UTF-8'))
+   >>> print(g.serialize(format='nquads', encoding='UTF-8').decode('UTF-8'))
    <http://example.org/BDW/entities/Widget#12> <http...> <http://example.org/data/imports/BDW_Widgets_2017-2018> .
    <http://example.org/BDW/entities/Widget#12> <...
 

docs/conf.py

@@ -37,6 +37,7 @@
 intersphinx_mapping = {'python': ('https://docs.python.org/3', None),
                        'rdflib': ('https://rdflib.readthedocs.io/en/stable/', None),
                        'owmeta-core': ('https://owmeta-core.readthedocs.io/en/stable/', None),
+                       'requests': ('https://requests.readthedocs.io/en/stable/', None),
                        'bibtexparser': ('https://bibtexparser.readthedocs.io/en/master/', None)}
 # Add any paths that contain templates here, relative to this directory.
 templates_path = ['_templates']

examples/add_reference.py

@@ -10,21 +10,27 @@
 
 import owmeta_core as P
 from owmeta_core.data import Data
-from owmeta_core.context import Context
+from owmeta_core.dataobject import DataObject
+from owmeta_core.context import Context, IMPORTS_CONTEXT_KEY
 
 from owmeta.evidence import Evidence
 from owmeta.neuron import Neuron
 from owmeta.document import Document
 
 # Create dummy database configuration.
 d = Data()
+d[IMPORTS_CONTEXT_KEY] = 'http://example.org/imports'
 
 # Connect to database with dummy configuration
 conn = P.connect(conf=d)
 conn.mapper.add_class(Evidence)
 conn.mapper.add_class(Document)
-ctx = conn(Context)(ident='http://example.org/data')
 evctx = conn(Context)(ident='http://example.org/meta')
+ctx = evctx(Context)(ident='http://example.org/data')
+
+# Add the Context RDF class to the mapper -- normally you'd get these from the
+# openworm/owmeta-core bundle as a dependency
+conn.mapper.add_class(type(ctx.rdf_object))
 
 # Create a new Neuron object to work with
 n = ctx(Neuron)(name='AVAL')
@@ -46,9 +52,18 @@
 ctx.save_context()
 evctx.save_context()
 
+evctx.add_import(Document.definition_context)
+evctx.save_imports()
+
+# Add a couple contexts to the store so we can resolve needed types. normally you'd get
+# these from the openworm/owmeta-core bundle as a dependency, so they don't have to be
+# added to the database again.
+Document.definition_context.save_context(conn.rdf)
+DataObject.definition_context.save_context(conn.rdf)
+
 # What does my evidence object contain?
 for e_i in evctx.stored(Evidence)().load():
     print(e_i.reference(), e_i.supports())
 
 # Disconnect from the database.
-P.disconnect(conn)
+conn.disconnect()

examples/rmgr.py

@@ -113,7 +113,7 @@ def setup(sctx, ctx, name, type):
     return n
 
 
-with OWM('../.owm').connect() as conn:
+with OWM('../.owm').connect().transaction() as conn:
     ctx = conn(Context)('http://example.org/data')
     evctx = conn(Context)('http://example.org/evidence')
 

owmeta/__init__.py

@@ -13,7 +13,7 @@
 """
 
 from __future__ import print_function
-__version__ = '0.12.3.dev0'
+__version__ = '0.12.4.dev0'
 __author__ = 'OpenWorm.org authors and contributors'
 
 import logging

owmeta/bibtex.py

@@ -1,4 +1,3 @@
-import re
 import bibtexparser
 
 from .evidence import Evidence
@@ -15,7 +14,7 @@ def bibtex_to_document(bibtex_entry, context=None):
 
 
 def update_document_with_bibtex(document, bibtex_entry):
-    document.set_key(bibtex_entry['ID'])
+    document.key = bibtex_entry['ID']
     for ath in bibtex_entry.get('author', tuple()):
         document.author(ath)
 

owmeta/cell.py

@@ -56,7 +56,9 @@ class Cell(BiologyType):
 
     parentOf = ObjectProperty(value_type=This, multiple=True)
 
-    key_property = {'property': name, 'type': 'direct'}
+    key_property = 'name'
+
+    direct_key = True
 
     def __init__(self, name=None, lineageName=None, **kwargs):
         # NOTE: We name the `name` and `lineageName` as positional parameters for

owmeta/data_trans/bibtex.py

@@ -1,4 +1,3 @@
-from rdflib.namespace import Namespace
 from owmeta_core.datasource import Informational, DataTranslator, DataSource
 from owmeta_core.data_trans.local_file_ds import LocalFileDataSource
 from owmeta_core.data_trans.context_datasource import VariableIdentifierContext
@@ -23,7 +22,8 @@ def __init__(self, *args, **kwargs):
                                                                     imported=(CONTEXT,))
         self.context_property(self.evidence_context.rdf_object)
 
-    def commit_augment(self):
+    def after_transform(self):
+        super(EvidenceDataSource, self).after_transform()
         saved_contexts = set([])
         self.data_context.save_context(inline_imports=True, saved_contexts=saved_contexts)
         self.context.save_context(inline_imports=True, saved_contexts=saved_contexts)

owmeta/data_trans/connections.py

@@ -1,9 +1,6 @@
-import re
-import traceback
 import csv
 import logging
 
-from owmeta_core.context import Context
 from owmeta_core.datasource import GenericTranslation
 from owmeta_core.dataobject import ObjectProperty
 from owmeta_core.data_trans.csv_ds import CSVDataTranslator, CSVDataSource

owmeta/data_trans/data_with_evidence_ds.py

@@ -1,8 +1,7 @@
-from rdflib.namespace import Namespace
-
 from owmeta_core.context import Context
 from owmeta_core.datasource import Informational, DataSource
-from owmeta_core.data_trans.context_datasource import VariableIdentifierContext
+from owmeta_core.data_trans.context_datasource import (VariableIdentifierContext,
+        VariableIdentifierContextDataObject)
 
 from .. import CONTEXT, SCI_CTX
 
@@ -12,8 +11,10 @@
 class DataWithEvidenceDataSource(DSMixin, DataSource):
     '''
     A data source that has an "evidence context" containing statements which support those
-    in its "data context". The data source also has a combined context which  imports both
-    the data and evidence contexts.
+    in its "data context". The data source also has a combined context which imports both
+    the data and evidence contexts. The data and evidence contexts have identifiers based
+    on the data source's identifier and the combined context has the same identifier as
+    the data source.
     '''
 
     class_context = SCI_CTX
@@ -22,20 +23,23 @@ class DataWithEvidenceDataSource(DSMixin, DataSource):
                                               property_name='evidence_context',
                                               property_type='ObjectProperty',
                                               multiple=False,
+                                              value_type=VariableIdentifierContextDataObject,
                                               description='The context in which evidence'
                                                           ' for the "Data context" is defined')
 
     data_context_property = Informational(display_name='Data context',
                                           property_name='data_context',
                                           property_type='ObjectProperty',
                                           multiple=False,
+                                          value_type=VariableIdentifierContextDataObject,
                                           description='The context in which primary data'
                                                       ' for this data source is defined')
 
     combined_context_property = Informational(display_name='Combined context',
                                               property_name='combined_context',
                                               property_type='ObjectProperty',
                                               multiple=False,
+                                              value_type=VariableIdentifierContextDataObject,
                                               description='Context importing both the data and evidence contexts')
 
     def __init__(self, *args, **kwargs):
@@ -80,7 +84,8 @@ def context_for(self, ident=None, **kwargs):
             res = self.__ad_hoc_contexts[key]
         return res
 
-    def commit_augment(self):
+    def after_transform(self):
+        super(DataWithEvidenceDataSource, self).after_transform()
         for ctx in self.__ad_hoc_contexts.values():
             ctx.save()