Version 2.0

.gitignore

@@ -1,2 +1,3 @@
 *.pyc
 src/output_bioc.xml
+src/output_bioc.json

CHANGES.txt

@@ -1,3 +1,8 @@
+2.0
+---
+New reader and writer for BioC JSON, based on the converter by Don Comeau (https://github.com/ncbi-nlp/BioC-JSON).
+Renamed the existing, XML-based BioCReader and BioCWriter to BioCXMLReader and BioCXMLWriter, respectively.
+
 1.02.4
 ------
 New method `BioCWriter.iterfragments()`: Iterate over serialised XML fragments,

README.md

@@ -2,7 +2,7 @@
 
 **[PyBioC][1] is a native Python library for reading and writing BioC XML data.**
 
-More information about BioC is available [online][2].
+More information about BioC is available at [sourceforge][2].
 
 
 ## Installation
@@ -36,9 +36,9 @@ valid BioC XML format.
 ### Generate BioC object for export
 
 ```python
-from bioc import BioCWriter, BioCCollection, BioCDocument, BioCPassage
+from bioc import BioCXMLWriter, BioCCollection, BioCDocument, BioCPassage
 
-writer = BioCWriter()
+writer = BioCXMLWriter()
 writer.collection = BioCCollection()
 collection = writer.collection
 collection.date = '20150301'

setup.py

@@ -7,7 +7,7 @@
     readme = f.read()
 
 setup(name='PyBioC',
-      version='1.02.4',
+      version='2.0',
       author='Hernani Marques',
       author_email='h2m@access.uzh.ch',
       description='Python library for working with BioC XML data',

src/bioc/__init__.py

@@ -5,9 +5,9 @@
 __version__ = '1.02.4'
 
 __all__ = [
-    'BioCAnnotation', 'BioCCollection', 'BioCDocument',
-    'BioCLocation', 'BioCNode', 'BioCPassage', 'BioCRelation',
-    'BioCSentence', 'BioCReader', 'BioCWriter'
+    'BioCAnnotation', 'BioCCollection', 'BioCDocument', 'BioCLocation',
+    'BioCNode', 'BioCPassage', 'BioCRelation', 'BioCSentence',
+    'BioCXMLReader', 'BioCJSONReader', 'BioCXMLWriter', 'BioCJSONWriter'
     ]
 
 __author__ = 'Hernani Marques (h2m@access.uzh.ch)'
@@ -20,5 +20,7 @@
 from .bioc_passage import BioCPassage
 from .bioc_relation import BioCRelation
 from .bioc_sentence import BioCSentence
-from .bioc_reader import BioCReader
-from .bioc_writer import BioCWriter
+from .bioc_reader import BioCXMLReader
+from .bioc_reader import BioCJSONReader
+from .bioc_writer import BioCXMLWriter
+from .bioc_writer import BioCJSONWriter

src/bioc/bioc_reader.py

@@ -1,6 +1,7 @@
-__all__ = ['BioCReader']
+__all__ = ['BioCXMLReader', 'BioCJSONReader']
 
-from io import StringIO
+import io
+import json
 
 from lxml import etree
 
@@ -14,19 +15,16 @@
 from .bioc_relation import BioCRelation
 
 
-class BioCReader:
+class BioCXMLReader(object):
     """
-    This class can be used to store BioC XML files in PyBioC objects,
-    for further manipulation.
+    Reader for parsing BioC XML files into BioC* objects.
     """
-
     def __init__(self, source, dtd_valid_file=None):
         """
         source:             File path to a BioC XML input document.
         dtd_valid_file:     File path to a BioC.dtd file. Using this
                             optional argument ensures DTD validation.
         """
-
         self.source = source
         self.collection = BioCCollection()
         self.xml_tree = etree.parse(source)
@@ -88,7 +86,7 @@ def _read_passages(self, passage_elem_list, document_parent_elem):
                 # Is the (optional) text element available?
                 try:
                     passage.text = passage_elem.xpath('text')[0].text
-                except:
+                except (IndexError, AttributeError):
                     pass
                 self._read_annotations(passage_elem.xpath('annotation'),
                                        passage)
@@ -143,3 +141,35 @@ def _read_relations(self, relation_elem_list, relations_parent_elem):
                 relation.add_node(node)
 
             relations_parent_elem.add_relation(relation)
+
+
+
+class BioCJSONReader(object):
+    '''
+    Reader for parsing BioC JSON files into BioC* objects.
+    '''
+    def __init__(self, source):
+        self.source = source
+        self.collection = None
+
+    def read(self):
+        '''
+        Read self.source and save the result to self.collection.
+        '''
+        with io.open(self.source, encoding='utf-8') as f:
+            dict_ = json.load(f)
+        self.collection = self._read_dict(BioCCollection, dict_)
+
+    def _read_dict(self, class_, dict_):
+        obj = class_()
+        for label, value in dict_.items():
+            if label in ('offset', 'length'):
+                # The converter in Don Comeau's reference implementation
+                # converts offset and length to strings, too.
+                value = str(value)
+            elif isinstance(value, list):
+                # Get the class from the label (eg. "documents" -> BioCDocument).
+                class_ = globals()['BioC{}'.format(label.rstrip('s').title())]
+                value = [self._read_dict(class_, d) for d in value]
+            setattr(obj, label, value)
+        return obj

src/bioc/bioc_writer.py

@@ -1,37 +1,58 @@
-__all__ = ['BioCWriter']
+__all__ = ['BioCXMLWriter', 'BioCJSONWriter']
 
 import sys
+import json
 
 from lxml.builder import E
 from lxml.etree import tostring
 
 
-# Resolve Python 2/3 difference regarding the special method __str__().
+# Resolve Python 2/3 differences.
 if sys.version_info < (3,):
+    # In Py2, use codecs.open rather than io.open, because the write() method
+    # of the latter doesn't accept native str values (only unicode).
+    from codecs import open
+    # Since BioCXMLWriter.__str__ calls lxml.etree.tostring, it must actually
+    # encode the serialised dump to get native str.
     STR_ENCODING = 'ascii'
 else:
     STR_ENCODING = 'unicode'
 
 
-class BioCWriter:
-
+class _BioCWriter(object):
+    '''
+    Base for BioC serializers.
+    '''
     def __init__(self, filename=None, collection=None):
+        self.collection = collection
+        self.filename = filename
 
-        self.root_tree = None
+    def _check_for_data(self):
+        if self.collection is None:
+            raise Exception('No data available.')
+
+    def _resolve_filename(self, filename):
+        if filename is None:
+            if self.filename is None:
+                raise Exception('No output file path provided.')
+            filename = self.filename
+        return filename
 
-        self.collection = None
-        self.doctype = '''<?xml version='1.0' encoding='UTF-8'?>'''
-        self.doctype += '''<!DOCTYPE collection SYSTEM 'BioC.dtd'>'''
-        self.filename = filename
 
-        if collection is not None:
-            self.collection = collection
+class BioCXMLWriter(_BioCWriter):
+    '''
+    XML serializer for BioC objects.
+    '''
 
-        if filename is not None:
-            self.filename = filename
+    doctype = "<!DOCTYPE collection SYSTEM 'BioC.dtd'>"
+
+    def __init__(self, filename=None, collection=None):
+        super(BioCXMLWriter, self).__init__(filename, collection)
+        self.root_tree = None
 
     def __str__(self):
-        """ A BioCWriter object can be printed as string.
+        """
+        A BioCWriter object can be printed as string.
         """
         return self.tostring(encoding=STR_ENCODING)
 
@@ -43,32 +64,39 @@ def tostring(self, encoding='UTF-8'):
         unless encoding is "unicode", in which case a decoded
         string is returned (a unicode object in Python 2).
         '''
-        self._check_for_data()
-
         self.build()
 
+        xml_declaration = self._binary_encoding(encoding)
         s = tostring(self.root_tree,
                      encoding=encoding,
                      pretty_print=True,
+                     xml_declaration=xml_declaration,
                      doctype=self.doctype)
 
         return s
 
-    def _check_for_data(self):
-        if self.collection is None:
-            raise Exception('No data available.')
+    @staticmethod
+    def _binary_encoding(codec):
+        '''
+        Is this actually a binary encoding?
+
+        The etree.tostring method accepts an encoding
+        parameter value "unicode" or str/unicode,
+        in which case the returned serialisation is a
+        decoded unicode string rather than an encoded
+        byte string.
+        '''
+        return not callable(codec) and codec != 'unicode'
 
     def write(self, filename=None):
-        """ Use this method to write the data in the PyBioC objects
-            to disk.
+        """
+        Write the data in the PyBioC objects to disk.
 
-            filename:   Output file path (optional argument; filename
-                        provided by __init__ used otherwise.)
+        filename: Output file path (optional argument;
+                  filename provided through __init__ used
+                  otherwise.)
         """
-        if filename is None:
-            if self.filename is None:
-                raise Exception('No output file path provided.')
-            filename = self.filename
+        filename = self._resolve_filename(filename)
 
         with open(filename, 'wb') as f:
             f.write(self.tostring(encoding='UTF-8'))
@@ -80,17 +108,21 @@ def iterfragments(self, encoding='UTF-8'):
         Use this for large collections, as it avoids building
         the whole tree in memory.
         '''
-        # Temporarily remove the document nodes and reset the root tree.
+        self._check_for_data()
+
+        # Temporarily remove the document nodes and the root tree.
         documents = self.collection.documents
+        previous_tree = self.root_tree
         self.collection.documents = ()
         self.root_tree = None
 
         # Construct and serialise the collection-level nodes.
         # Split them into a head and tail portion.
         shell = self.tostring(encoding)
         tail = u'</collection>\n'
-        BOM = ''.encode(encoding)
-        if encoding != 'unicode':
+        BOM = ''
+        if self._binary_encoding(encoding):
+            BOM = ''.encode(encoding)
             tail = tail.encode(encoding).lstrip(BOM)
         head = shell[:-len(tail)]
 
@@ -109,11 +141,15 @@ def iterfragments(self, encoding='UTF-8'):
 
         yield tail
 
-        # Restore the collection object and reset the root tree again.
+        # Restore the collection object and the root tree.
         self.collection.documents = documents
-        self.root_tree = None
+        self.root_tree = previous_tree
 
     def build(self):
+        '''
+        Create an Element tree in memory.
+        '''
+        self._check_for_data()
         if self.root_tree is None:
             self._build_collection()
 
@@ -128,7 +164,8 @@ def _build_collection(self):
         # document+
         self._build_documents(self.collection.documents, collection_elem)
 
-    def _build_infons(self, infons_dict, infons_parent_elem):
+    @staticmethod
+    def _build_infons(infons_dict, infons_parent_elem):
         for infon_key, infon_val in infons_dict.items():
             infons_parent_elem.append(E('infon'))
             infon_elem = infons_parent_elem.xpath('infon')[-1]
@@ -227,3 +264,66 @@ def _build_sentences(self, sentences_list, passage_parent_elem):
             self._build_annotations(sentence.annotations, sentence_elem)
             # relation*
             self._build_relations(sentence.relations, sentence_elem)
+
+
+class BioCJSONWriter(_BioCWriter):
+    '''
+    JSON serializer for BioC objects.
+    '''
+    def __init__(self, filename=None, collection=None):
+        super(BioCJSONWriter, self).__init__(filename, collection)
+        self.root_dict = None
+
+    def __str__(self):
+        return str(self.tostring())
+
+    def tostring(self, **kwargs):
+        '''
+        Dump serialized BioC JSON to a string.
+        '''
+        self.build()
+        return json.dumps(self.root_dict, **kwargs)
+
+    def write(self, filename=None, **kwargs):
+        '''
+        Write serialised BioC JSON to disk.
+        '''
+        self.build()
+        filename = self._resolve_filename(filename)
+        with open(filename, 'w', encoding='utf-8') as f:
+            json.dump(self.root_dict, f, **kwargs)
+
+    def iterfragments(self, **kwargs):
+        '''
+        Iterate over chunks of serialised BioC JSON.
+
+        This method still creates an entire copy of the
+        structure in memory.
+        '''
+        self.build()
+        for chunk in json.JSONEncoder(**kwargs).iterencode(self.root_dict):
+            yield chunk
+
+    def build(self):
+        '''
+        Construct a nested dictionary in memory.
+        '''
+        self._check_for_data()
+        if self.root_dict is None:
+            self.root_dict = self._build_dict(self.collection)
+
+    def _build_dict(self, obj):
+        # Note:
+        # Unlike the DTD, Don Comeau's reference implementation of a BioC JSON
+        # converter does not enforce mutual exclusion of either sentences
+        # or text + annotations inside passage elements.
+        dict_ = {}
+        for label, value in obj.__dict__.items():
+            if label == 'text' and value is None:
+                value = ''  # avoid None/null
+            elif label in ('offset', 'length'):
+                value = int(value)
+            elif isinstance(value, list):
+                value = [self._build_dict(c) for c in value]
+            dict_[label] = value
+        return dict_