Skip to content

Issue55 double checking the revision #156

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
opsdep wants to merge 3 commits into
base: issue154_deleting_nansat_ingestor
Choose a base branch
from
Open

Issue55 double checking the revision #156

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
c0fa8c5
#55 converting strings into number in order to install them correctly…
Mar 4, 2021
daf8458
#55 remove update_vocabularies_related tests
Mar 4, 2021
473ecc7
#55 remove update vocabularies functions from manager of vocabulary
Mar 4, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions geospaas/catalog/fixtures/catalog.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,9 +29,9 @@
"summary": "This is a quite short summary about the test dataset.",
"source": ["AQUA", "MODIS"],
"geographic_location": 1,
"data_center": ["NERSC"],
"gcmd_location": ["VERTICAL LOCATION", "SEA SURFACE", "", "", ""],
"ISO_topic_category": ["Oceans"],
"data_center": 1391,
"gcmd_location": 503,
"ISO_topic_category": 14,
"access_constraints": null
}
},{
Expand All @@ -45,9 +45,9 @@
"summary": "This is a quite short summary about the test dataset.",
"source": ["AQUA", "MODIS"],
"geographic_location": 2,
"data_center": ["NERSC"],
"gcmd_location": ["VERTICAL LOCATION", "SEA SURFACE", "", "", ""],
"ISO_topic_category": ["Oceans"],
"data_center": 1391,
"gcmd_location": 503,
"ISO_topic_category": 14,
"access_constraints": null
}
},{
Expand Down
147 changes: 38 additions & 109 deletions geospaas/vocabularies/managers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#-------------------------------------------------------------------------------
# -------------------------------------------------------------------------------
# Name:
# Purpose:
#
Expand All @@ -9,70 +9,25 @@
# Last modified:
# Copyright: (c) NERSC
# License:
#-------------------------------------------------------------------------------
# -------------------------------------------------------------------------------
from __future__ import print_function

import os, json, warnings
import os
import json
import warnings
import pythesint as pti

from django.db import models


class VocabularyManager(models.Manager):
""" Base abstract class for all Managers here """

def update_and_get_list(self, get_list, update, force):
""" Get list of Pythesint entires after an update if needed

Parameters
----------
get_lilst : func
function to get lilst of Pythesint entries
update : func
function to update Pythesint

Returns
-------
pti_list : list
lilst of Pythesin entries

"""
pti_list = get_list()
if force==True or len(pti_list)==0:
update()
pti_list = [e for e in get_list() if not 'Revision' in e.keys()]
return pti_list

def create_instances(self, pti_list):
""" Create instances in database

Parameters
----------
pti_list : list with Pythesint entries

"""
num = 0
for entry in pti_list:
pp, created = self.get_or_create(entry)
if created: num+=1
print("Successfully added %d new entries" % num)

def get_or_create(self, entry, *args, **kwargs):
""" Get or create database instance from input pythesint entry """

params = {key : entry[self.mapping[key]] for key in self.mapping}
params = {key: entry[self.mapping[key]] for key in self.mapping}
return super(VocabularyManager, self).get_or_create(**params)

def create_from_vocabularies(self, force=False, **kwargs):
""" Get instances Pythesint and create instances in database.

Parameters
----------
force : bool
Force update of Vocabulary from Internet ?

"""
pti_list = self.update_and_get_list(self.get_list, self.update, force)
self.create_instances(pti_list)

class ParameterManager(VocabularyManager):
get_list = pti.get_wkv_variable_list
Expand All @@ -83,30 +38,6 @@ class ParameterManager(VocabularyManager):
short_name='short_name',
units='units')

def create_from_vocabularies(self, force=False, **kwargs):
""" Create parameter instances from the NERSC WKV list and CF-standard names.

Have to override VocabularyManager.create_from_vocabularies in order to allow
merge of CF-variable names and WKVs.

"""
warnings.warn(''' Since we do not yet have the mapping between the different
vocabularies, the GCMD science keywords are not linked to the catalog
parameter table ''')
pti_list1 = self.update_and_get_list(self.get_list, self.update, force)
pti_list2 = self.update_and_get_list(self.get_list2, self.update2, force)
# merge two lists
l1_std = [l['standard_name'] for l in pti_list1]
for l in pti_list2:
if 'standard_name' in l and l['standard_name'] not in pti_list1:
pti_list1.append(
dict(
standard_name=l['standard_name'],
short_name='',
units=l['canonical_units']))

self.create_instances(pti_list1)

def get_by_natural_key(self, standard_name):
return self.get(standard_name=standard_name)

Expand All @@ -115,53 +46,51 @@ class PlatformManager(VocabularyManager):
get_list = pti.get_gcmd_platform_list
update = pti.update_gcmd_platform
mapping = dict(category='Category',
series_entity='Series_Entity',
short_name='Short_Name',
long_name='Long_Name')

series_entity='Series_Entity',
short_name='Short_Name',
long_name='Long_Name')


class InstrumentManager(VocabularyManager):
get_list = pti.get_gcmd_instrument_list
update = pti.update_gcmd_instrument
mapping = dict(category='Category',
instrument_class='Class',
type='Type',
subtype='Subtype',
short_name='Short_Name',
long_name='Long_Name')

instrument_class='Class',
type='Type',
subtype='Subtype',
short_name='Short_Name',
long_name='Long_Name')


class ScienceKeywordManager(VocabularyManager):
get_list = pti.get_gcmd_science_keyword_list
update = pti.update_gcmd_science_keyword
mapping = dict(category='Category',
topic='Topic',
term='Term',
variable_level_1='Variable_Level_1',
variable_level_2='Variable_Level_2',
variable_level_3='Variable_Level_3',
detailed_variable='Detailed_Variable')
topic='Topic',
term='Term',
variable_level_1='Variable_Level_1',
variable_level_2='Variable_Level_2',
variable_level_3='Variable_Level_3',
detailed_variable='Detailed_Variable')

def get_by_natural_key(self, category, topic, term, variable_level_1,
variable_level_2, variable_level_3):
variable_level_2, variable_level_3):
return self.get(category=category, topic=topic, term=term,
variable_level_1=variable_level_1,
variable_level_2=variable_level_2,
variable_level_3=variable_level_3)
variable_level_1=variable_level_1,
variable_level_2=variable_level_2,
variable_level_3=variable_level_3)


class DataCenterManager(VocabularyManager):
get_list = pti.get_gcmd_provider_list
update = pti.update_gcmd_provider
mapping = dict(bucket_level0='Bucket_Level0',
bucket_level1='Bucket_Level1',
bucket_level2='Bucket_Level2',
bucket_level3='Bucket_Level3',
short_name='Short_Name',
long_name='Long_Name',
data_center_url='Data_Center_URL')
bucket_level1='Bucket_Level1',
bucket_level2='Bucket_Level2',
bucket_level3='Bucket_Level3',
short_name='Short_Name',
long_name='Long_Name',
data_center_url='Data_Center_URL')

def get_by_natural_key(self, sname):
return self.get(short_name=sname)
Expand Down Expand Up @@ -198,8 +127,8 @@ class ProjectManager(VocabularyManager):
get_list = pti.get_gcmd_project_list
update = pti.update_gcmd_project
mapping = dict(bucket='Bucket',
short_name='Short_Name',
long_name='Long_Name')
short_name='Short_Name',
long_name='Long_Name')

def get_by_natural_key(self, bucket, short_name):
return self.get(bucket=bucket, short_name=short_name)
Expand All @@ -218,11 +147,11 @@ class LocationManager(VocabularyManager):
get_list = pti.get_gcmd_location_list
update = pti.update_gcmd_location
mapping = dict(category='Location_Category',
type='Location_Type',
subregion1='Location_Subregion1',
subregion2='Location_Subregion2',
subregion3='Location_Subregion3')
type='Location_Type',
subregion1='Location_Subregion1',
subregion2='Location_Subregion2',
subregion3='Location_Subregion3')

def get_by_natural_key(self, category, type, subregion1, subregion2, subregion3):
return self.get(category=category, type=type, subregion1=subregion1,
subregion2=subregion2, subregion3=subregion3)
subregion2=subregion2, subregion3=subregion3)
61 changes: 0 additions & 61 deletions geospaas/vocabularies/tests.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,14 +26,6 @@ def setUp(self):
def tearDown(self):
self.patcher.stop()

def test_create_from_vocabularies(self):
""" Test shared with all vocabularies """
self.model.objects.create_from_vocabularies(force=True)
self.model.objects.create_from_vocabularies()
self.model.objects.get_list.assert_called()
self.model.objects.update.assert_called()
self.assertIn('Successfully added', self.mock_print.call_args[0][0])

def _insert_twice(self, attributes):
"""Test that an object of the given model class can't be inserted twice"""
object1 = self.model(**attributes)
Expand Down Expand Up @@ -415,56 +407,3 @@ def test_unique_constraint(self):
"""Check that the same VerticalDataResolution can't be inserted twice"""
with self.assertRaises(django.db.utils.IntegrityError):
self._insert_twice({'range': 'test'})


class CommandsTests(TestCase):
"""Unit tests for the custom commands of the vocabularies app"""

def setUp(self):
return_value = [{'Revision': '2019-02-13 08:48:55'}]
models = [
Parameter,
DataCenter,
HorizontalDataResolution,
Instrument,
ISOTopicCategory,
Location,
Platform,
Project,
ScienceKeyword,
TemporalDataResolution,
VerticalDataResolution,
]
# mock get_list in all managers
self.get_list_mocks = [
patch.object(model.objects, 'get_list', return_value=return_value).start()
for model in models]
# mock update in all managers
self.update_mocks = [
patch.object(model.objects, 'update', return_value=return_value).start()
for model in models]

def test_command_update_vocabularies(self):
"""Check that the command does not update the vocabularies if they are present"""

call_command('update_vocabularies')
# check that get_list was called only once
for mock in self.get_list_mocks:
mock.assert_called()
# check that update was never called
for mock in self.update_mocks:
mock.assert_not_called()

def test_command_update_vocabularies_force(self):
"""
Check that the command updates the vocabularies even if they are present when --force is
specified
"""

call_command('update_vocabularies', '--force')
# check that get_list was called only once
for mock in self.get_list_mocks:
mock.assert_called()
# check that update was called
for mock in self.update_mocks:
mock.assert_called()