diff --git a/.gitignore b/.gitignore index d5770e0..6666947 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ *.pyc src/output_bioc.xml +src/output_bioc.json diff --git a/CHANGES.txt b/CHANGES.txt index 294af88..1df55b0 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,3 +1,8 @@ +2.0 +--- +New reader and writer for BioC JSON, based on the converter by Don Comeau (https://github.com/ncbi-nlp/BioC-JSON). +Renamed the existing, XML-based BioCReader and BioCWriter to BioCXMLReader and BioCXMLWriter, respectively. + 1.02.4 ------ New method `BioCWriter.iterfragments()`: Iterate over serialised XML fragments, diff --git a/README.md b/README.md index 5dd0150..e002d88 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ **[PyBioC][1] is a native Python library for reading and writing BioC XML data.** -More information about BioC is available [online][2]. +More information about BioC is available at [sourceforge][2]. ## Installation @@ -36,9 +36,9 @@ valid BioC XML format. ### Generate BioC object for export ```python -from bioc import BioCWriter, BioCCollection, BioCDocument, BioCPassage +from bioc import BioCXMLWriter, BioCCollection, BioCDocument, BioCPassage -writer = BioCWriter() +writer = BioCXMLWriter() writer.collection = BioCCollection() collection = writer.collection collection.date = '20150301' diff --git a/setup.py b/setup.py index 514b394..1f99754 100644 --- a/setup.py +++ b/setup.py @@ -7,7 +7,7 @@ readme = f.read() setup(name='PyBioC', - version='1.02.4', + version='2.0', author='Hernani Marques', author_email='h2m@access.uzh.ch', description='Python library for working with BioC XML data', diff --git a/src/bioc/__init__.py b/src/bioc/__init__.py index c240595..a3ab661 100644 --- a/src/bioc/__init__.py +++ b/src/bioc/__init__.py @@ -5,9 +5,9 @@ __version__ = '1.02.4' __all__ = [ - 'BioCAnnotation', 'BioCCollection', 'BioCDocument', - 'BioCLocation', 'BioCNode', 'BioCPassage', 'BioCRelation', - 'BioCSentence', 'BioCReader', 'BioCWriter' + 'BioCAnnotation', 'BioCCollection', 'BioCDocument', 'BioCLocation', + 'BioCNode', 'BioCPassage', 'BioCRelation', 'BioCSentence', + 'BioCXMLReader', 'BioCJSONReader', 'BioCXMLWriter', 'BioCJSONWriter' ] __author__ = 'Hernani Marques (h2m@access.uzh.ch)' @@ -20,5 +20,7 @@ from .bioc_passage import BioCPassage from .bioc_relation import BioCRelation from .bioc_sentence import BioCSentence -from .bioc_reader import BioCReader -from .bioc_writer import BioCWriter +from .bioc_reader import BioCXMLReader +from .bioc_reader import BioCJSONReader +from .bioc_writer import BioCXMLWriter +from .bioc_writer import BioCJSONWriter diff --git a/src/bioc/bioc_reader.py b/src/bioc/bioc_reader.py index 75bd86f..022b9c0 100644 --- a/src/bioc/bioc_reader.py +++ b/src/bioc/bioc_reader.py @@ -1,6 +1,7 @@ -__all__ = ['BioCReader'] +__all__ = ['BioCXMLReader', 'BioCJSONReader'] -from io import StringIO +import io +import json from lxml import etree @@ -14,19 +15,16 @@ from .bioc_relation import BioCRelation -class BioCReader: +class BioCXMLReader(object): """ - This class can be used to store BioC XML files in PyBioC objects, - for further manipulation. + Reader for parsing BioC XML files into BioC* objects. """ - def __init__(self, source, dtd_valid_file=None): """ source: File path to a BioC XML input document. dtd_valid_file: File path to a BioC.dtd file. Using this optional argument ensures DTD validation. """ - self.source = source self.collection = BioCCollection() self.xml_tree = etree.parse(source) @@ -88,7 +86,7 @@ def _read_passages(self, passage_elem_list, document_parent_elem): # Is the (optional) text element available? try: passage.text = passage_elem.xpath('text')[0].text - except: + except (IndexError, AttributeError): pass self._read_annotations(passage_elem.xpath('annotation'), passage) @@ -143,3 +141,35 @@ def _read_relations(self, relation_elem_list, relations_parent_elem): relation.add_node(node) relations_parent_elem.add_relation(relation) + + + +class BioCJSONReader(object): + ''' + Reader for parsing BioC JSON files into BioC* objects. + ''' + def __init__(self, source): + self.source = source + self.collection = None + + def read(self): + ''' + Read self.source and save the result to self.collection. + ''' + with io.open(self.source, encoding='utf-8') as f: + dict_ = json.load(f) + self.collection = self._read_dict(BioCCollection, dict_) + + def _read_dict(self, class_, dict_): + obj = class_() + for label, value in dict_.items(): + if label in ('offset', 'length'): + # The converter in Don Comeau's reference implementation + # converts offset and length to strings, too. + value = str(value) + elif isinstance(value, list): + # Get the class from the label (eg. "documents" -> BioCDocument). + class_ = globals()['BioC{}'.format(label.rstrip('s').title())] + value = [self._read_dict(class_, d) for d in value] + setattr(obj, label, value) + return obj diff --git a/src/bioc/bioc_writer.py b/src/bioc/bioc_writer.py index de9fd7e..aa34912 100644 --- a/src/bioc/bioc_writer.py +++ b/src/bioc/bioc_writer.py @@ -1,37 +1,58 @@ -__all__ = ['BioCWriter'] +__all__ = ['BioCXMLWriter', 'BioCJSONWriter'] import sys +import json from lxml.builder import E from lxml.etree import tostring -# Resolve Python 2/3 difference regarding the special method __str__(). +# Resolve Python 2/3 differences. if sys.version_info < (3,): + # In Py2, use codecs.open rather than io.open, because the write() method + # of the latter doesn't accept native str values (only unicode). + from codecs import open + # Since BioCXMLWriter.__str__ calls lxml.etree.tostring, it must actually + # encode the serialised dump to get native str. STR_ENCODING = 'ascii' else: STR_ENCODING = 'unicode' -class BioCWriter: - +class _BioCWriter(object): + ''' + Base for BioC serializers. + ''' def __init__(self, filename=None, collection=None): + self.collection = collection + self.filename = filename - self.root_tree = None + def _check_for_data(self): + if self.collection is None: + raise Exception('No data available.') + + def _resolve_filename(self, filename): + if filename is None: + if self.filename is None: + raise Exception('No output file path provided.') + filename = self.filename + return filename - self.collection = None - self.doctype = '''''' - self.doctype += '''''' - self.filename = filename - if collection is not None: - self.collection = collection +class BioCXMLWriter(_BioCWriter): + ''' + XML serializer for BioC objects. + ''' - if filename is not None: - self.filename = filename + doctype = "" + + def __init__(self, filename=None, collection=None): + super(BioCXMLWriter, self).__init__(filename, collection) + self.root_tree = None def __str__(self): - """ A BioCWriter object can be printed as string. + """ + A BioCWriter object can be printed as string. """ return self.tostring(encoding=STR_ENCODING) @@ -43,32 +64,39 @@ def tostring(self, encoding='UTF-8'): unless encoding is "unicode", in which case a decoded string is returned (a unicode object in Python 2). ''' - self._check_for_data() - self.build() + xml_declaration = self._binary_encoding(encoding) s = tostring(self.root_tree, encoding=encoding, pretty_print=True, + xml_declaration=xml_declaration, doctype=self.doctype) return s - def _check_for_data(self): - if self.collection is None: - raise Exception('No data available.') + @staticmethod + def _binary_encoding(codec): + ''' + Is this actually a binary encoding? + + The etree.tostring method accepts an encoding + parameter value "unicode" or str/unicode, + in which case the returned serialisation is a + decoded unicode string rather than an encoded + byte string. + ''' + return not callable(codec) and codec != 'unicode' def write(self, filename=None): - """ Use this method to write the data in the PyBioC objects - to disk. + """ + Write the data in the PyBioC objects to disk. - filename: Output file path (optional argument; filename - provided by __init__ used otherwise.) + filename: Output file path (optional argument; + filename provided through __init__ used + otherwise.) """ - if filename is None: - if self.filename is None: - raise Exception('No output file path provided.') - filename = self.filename + filename = self._resolve_filename(filename) with open(filename, 'wb') as f: f.write(self.tostring(encoding='UTF-8')) @@ -80,8 +108,11 @@ def iterfragments(self, encoding='UTF-8'): Use this for large collections, as it avoids building the whole tree in memory. ''' - # Temporarily remove the document nodes and reset the root tree. + self._check_for_data() + + # Temporarily remove the document nodes and the root tree. documents = self.collection.documents + previous_tree = self.root_tree self.collection.documents = () self.root_tree = None @@ -89,8 +120,9 @@ def iterfragments(self, encoding='UTF-8'): # Split them into a head and tail portion. shell = self.tostring(encoding) tail = u'\n' - BOM = ''.encode(encoding) - if encoding != 'unicode': + BOM = '' + if self._binary_encoding(encoding): + BOM = ''.encode(encoding) tail = tail.encode(encoding).lstrip(BOM) head = shell[:-len(tail)] @@ -109,11 +141,15 @@ def iterfragments(self, encoding='UTF-8'): yield tail - # Restore the collection object and reset the root tree again. + # Restore the collection object and the root tree. self.collection.documents = documents - self.root_tree = None + self.root_tree = previous_tree def build(self): + ''' + Create an Element tree in memory. + ''' + self._check_for_data() if self.root_tree is None: self._build_collection() @@ -128,7 +164,8 @@ def _build_collection(self): # document+ self._build_documents(self.collection.documents, collection_elem) - def _build_infons(self, infons_dict, infons_parent_elem): + @staticmethod + def _build_infons(infons_dict, infons_parent_elem): for infon_key, infon_val in infons_dict.items(): infons_parent_elem.append(E('infon')) infon_elem = infons_parent_elem.xpath('infon')[-1] @@ -227,3 +264,66 @@ def _build_sentences(self, sentences_list, passage_parent_elem): self._build_annotations(sentence.annotations, sentence_elem) # relation* self._build_relations(sentence.relations, sentence_elem) + + +class BioCJSONWriter(_BioCWriter): + ''' + JSON serializer for BioC objects. + ''' + def __init__(self, filename=None, collection=None): + super(BioCJSONWriter, self).__init__(filename, collection) + self.root_dict = None + + def __str__(self): + return str(self.tostring()) + + def tostring(self, **kwargs): + ''' + Dump serialized BioC JSON to a string. + ''' + self.build() + return json.dumps(self.root_dict, **kwargs) + + def write(self, filename=None, **kwargs): + ''' + Write serialised BioC JSON to disk. + ''' + self.build() + filename = self._resolve_filename(filename) + with open(filename, 'w', encoding='utf-8') as f: + json.dump(self.root_dict, f, **kwargs) + + def iterfragments(self, **kwargs): + ''' + Iterate over chunks of serialised BioC JSON. + + This method still creates an entire copy of the + structure in memory. + ''' + self.build() + for chunk in json.JSONEncoder(**kwargs).iterencode(self.root_dict): + yield chunk + + def build(self): + ''' + Construct a nested dictionary in memory. + ''' + self._check_for_data() + if self.root_dict is None: + self.root_dict = self._build_dict(self.collection) + + def _build_dict(self, obj): + # Note: + # Unlike the DTD, Don Comeau's reference implementation of a BioC JSON + # converter does not enforce mutual exclusion of either sentences + # or text + annotations inside passage elements. + dict_ = {} + for label, value in obj.__dict__.items(): + if label == 'text' and value is None: + value = '' # avoid None/null + elif label in ('offset', 'length'): + value = int(value) + elif isinstance(value, list): + value = [self._build_dict(c) for c in value] + dict_[label] = value + return dict_ diff --git a/src/stemmer.py b/src/stemmer.py index 6e14971..2ee21f6 100755 --- a/src/stemmer.py +++ b/src/stemmer.py @@ -9,8 +9,8 @@ from nltk import PorterStemmer from bioc import BioCAnnotation -from bioc import BioCReader -from bioc import BioCWriter +from bioc import BioCXMLReader +from bioc import BioCXMLWriter BIOC_IN = os.path.join('..', 'test_input', 'example_input.xml') BIOC_OUT = os.path.join('..', 'test_input', 'example_input_stemmed.xml') @@ -23,18 +23,18 @@ def main(): if len(sys.argv) >= 2: bioc_in = sys.argv[1] - # A BioCReader object is put in place to hold the example BioC XML + # A BioCXMLReader object is put in place to hold the example BioC XML # document - bioc_reader = BioCReader(bioc_in, dtd_valid_file=DTD_FILE) + bioc_reader = BioCXMLReader(bioc_in, dtd_valid_file=DTD_FILE) - # A BioCWRiter object is prepared to write out the annotated data - bioc_writer = BioCWriter(BIOC_OUT) + # A BioCXMLWRiter object is prepared to write out the annotated data + bioc_writer = BioCXMLWriter(BIOC_OUT) # The NLTK porter stemmer is used for stemming stemmer = PorterStemmer() # The example input file given above (by BIOC_IN) is fed into - # a BioCReader object; validation is done by the BioC DTD + # a BioCXMLReader object; validation is done by the BioC DTD bioc_reader.read() # Pass over basic data diff --git a/src/test_read+write.py b/src/test_read+write.py index c37ad7c..9d358c6 100755 --- a/src/test_read+write.py +++ b/src/test_read+write.py @@ -1,25 +1,33 @@ #!/usr/bin/env python -from bioc import BioCReader -from bioc import BioCWriter +from bioc import BioCXMLReader +from bioc import BioCJSONReader +from bioc import BioCXMLWriter +from bioc import BioCJSONWriter -test_file = '../test_input/example_input.xml' +test_xml = '../test_input/example_input.xml' +test_json = '../test_input/example_input.json' dtd_file = '../test_input/BioC.dtd' def main(): - bioc_reader = BioCReader(test_file, dtd_valid_file=dtd_file) - bioc_reader.read() - ''' - sentences = bioc_reader.collection.documents[0].passages[0].sentences - for sentence in sentences: - print sentence.offset - ''' - - bioc_writer = BioCWriter('output_bioc.xml') - bioc_writer.collection = bioc_reader.collection - bioc_writer.write() - print(bioc_writer) + # Read XML, write JSON. + xml_reader = BioCXMLReader(test_xml, dtd_valid_file=dtd_file) + xml_reader.read() + + json_writer = BioCJSONWriter() + json_writer.collection = xml_reader.collection + json_writer.write('output_bioc.json', indent=2) + print(json_writer) + + # Read JSON, write XML. + json_reader = BioCJSONReader(test_json) + json_reader.read() + + xml_writer = BioCXMLWriter('output_bioc.xml', xml_reader.collection) + xml_writer.write() + print(xml_writer) + if __name__ == '__main__': main() diff --git a/test_input/example_input.json b/test_input/example_input.json new file mode 100644 index 0000000..eb968fd --- /dev/null +++ b/test_input/example_input.json @@ -0,0 +1,35 @@ +{ + "key": "ctdBCIVLearningDataSet.key", + "source": "PUBMED", + "date": "20130422", + "documents": [ + { + "infons": {}, + "relations": [], + "passages": [ + { + "offset": 0, + "sentences": [], + "annotations": [], + "relations": [], + "text": "Possible role of valvular serotonin 5-HT(2B) receptors in the cardiopathy associated with fenfluramine.", + "infons": { + "type": "title" + } + }, + { + "offset": 104, + "sentences": [], + "annotations": [], + "relations": [], + "text": "Dexfenfluramine was approved in the United States for long-term use as an appetite suppressant until it was reported to be associated with valvular heart disease. The valvular changes (myofibroblast proliferation) are histopathologically indistinguishable from those observed in carcinoid disease or after long-term exposure to 5-hydroxytryptamine (5-HT)(2)-preferring ergot drugs (ergotamine, methysergide). 5-HT(2) receptor stimulation is known to cause fibroblast mitogenesis, which could contribute to this lesion. To elucidate the mechanism of \"fen-phen\"-associated valvular lesions, we examined the interaction of fenfluramine and its metabolite norfenfluramine with 5-HT(2) receptor subtypes and examined the expression of these receptors in human and porcine heart valves. Fenfluramine binds weakly to 5-HT(2A), 5-HT(2B), and 5-HT(2C) receptors. In contrast, norfenfluramine exhibited high affinity for 5-HT(2B) and 5-HT(2C) receptors and more moderate affinity for 5-HT(2A) receptors. In cells expressing recombinant 5-HT(2B) receptors, norfenfluramine potently stimulated the hydrolysis of inositol phosphates, increased intracellular Ca(2+), and activated the mitogen-activated protein kinase cascade, the latter of which has been linked to mitogenic actions of the 5-HT(2B) receptor. The level of 5-HT(2B) and 5-HT(2A) receptor transcripts in heart valves was at least 300-fold higher than the levels of 5-HT(2C) receptor transcript, which were barely detectable. We propose that preferential stimulation of valvular 5-HT(2B) receptors by norfenfluramine, ergot drugs, or 5-HT released from carcinoid tumors (with or without accompanying 5-HT(2A) receptor activation) may contribute to valvular fibroplasia in humans.", + "infons": { + "type": "abstract" + } + } + ], + "id": "10617681" + } + ], + "infons": {} +}