diff --git a/Basics/Exploring_A_Data_Repo.ipynb b/Basics/Exploring_A_Data_Repo.ipynb
index d9e87c11..3a7b80d1 100644
--- a/Basics/Exploring_A_Data_Repo.ipynb
+++ b/Basics/Exploring_A_Data_Repo.ipynb
@@ -6,8 +6,8 @@
"source": [
"# Exploring a Data Repository\n",
"\n",
- "
Owner: **Phil Marshall** ([@drphilmarshall](https://github.com/LSSTScienceCollaborations/StackClub/issues/new?body=@drphilmarshall))\n",
- "
Last Verified to Run: **2018-09-07**\n",
+ "
Owner: **Rob Morgan** ([@rmorgan10](https://github.com/LSSTScienceCollaborations/StackClub/issues/new?body=@rmorgan10)), **Phil Marshall** ([@drphilmarshall](https://github.com/LSSTScienceCollaborations/StackClub/issues/new?body=@drphilmarshall))\n",
+ "
Last Verified to Run: **2018-11-02**\n",
"
Verified Stack Release: **16.0**\n",
"\n",
"This notebook shows how to find out what's in a data repository, and how to find out which inputs went into each component of it. \n",
@@ -60,9 +60,11 @@
"source": [
"from lsst.daf.persistence import Butler\n",
"\n",
- "# Instantiate the butler\n",
+ "# Instantiate the butler to bring us some HSC data.\n",
+ "\n",
"depth = 'WIDE' # WIDE, DEEP, UDEEP\n",
"field = 'SSP_WIDE' # SSP_WIDE, SSP_DEEP, SSP_UDEEP\n",
+ "\n",
"repo = '/datasets/hsc/repo/rerun/DM-13666/%s/'%(depth)\n",
"butler = Butler(repo)\n",
"\n",
@@ -97,14 +99,14 @@
"metadata": {},
"outputs": [],
"source": [
- "where_is('ingestImages.py', in_the='source', assuming_its_a='cmdlinetask')"
+ "# where_is('ingest.py', in_the='source', assuming_its_a='cmdlinetask')"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
- "The `HscMapper` class is defined in [HscMapper.py](https://github.com/lsst/obs_subaru/blob/master/python/lsst/obs/hsc/hscMapper.py). Let's instantiate one and read it."
+ "The `HscMapper` class is defined in [HscMapper.py](https://github.com/lsst/obs_subaru/blob/master/python/lsst/obs/hsc/hscMapper.py). Let's read about it."
]
},
{
@@ -122,7 +124,60 @@
"metadata": {},
"outputs": [],
"source": [
- "help(HscMapper)"
+ "# help(HscMapper)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "The mapper defines a (large) number of different dataset types. Some of these are specific to this particular dataset, others are more general. Even filtering out some intermediate dataset types, we are still left with a long list..."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "mapper = HscMapper(root=repo)\n",
+ "all_dataset_types = mapper.getDatasetTypes()\n",
+ "\n",
+ "remove = ['_config', '_filename', '_md', '_sub', '_len', '_schema', '_metadata']\n",
+ "\n",
+ "shortlist = []\n",
+ "for dataset_type in all_dataset_types:\n",
+ " keep = True\n",
+ " for word in remove:\n",
+ " if word in dataset_type:\n",
+ " keep = False\n",
+ " if keep:\n",
+ " shortlist.append(dataset_type)\n",
+ "\n",
+ "print(shortlist)"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "The `butler` purpoorts to be able to check whether a dataset actually exists or not, but needs a specific dataset ID to check it:"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "butler.datasetExists('calexp', dataId={})"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "Instead, one can try querying the metadata and checking for an error."
]
},
{
@@ -131,7 +186,15 @@
"metadata": {},
"outputs": [],
"source": [
- "help(Butler)"
+ "datasettype = 'calexp'\n",
+ "\n",
+ "try:\n",
+ " datasetkeys = butler.getKeys(datasettype)\n",
+ " onekey = list(datasetkeys.keys())[0]\n",
+ " metadata = butler.queryMetadata(datasettype, [onekey])\n",
+ " print(\"{} dataset exists.\".format(datasettype))\n",
+ "except:\n",
+ " print(\"{} dataset doesn't exist.\".format(datasettype))"
]
},
{
@@ -140,7 +203,7 @@
"metadata": {},
"outputs": [],
"source": [
- "butler.getKeys('deepCoadd_calexp')"
+ "# help(butler)"
]
},
{
@@ -157,10 +220,7 @@
"metadata": {},
"outputs": [],
"source": [
- "#This cell is temporary and just a way for me to have a concise reference while developing\n",
- "\n",
- "#Dataset types for the HSC_mapper dataset. \n",
- "#Need a better way of obtaining all of these names.\n",
+ "# Interesting dataset types for the HSC_mapper dataset. \n",
"datasettypes = ['calexp', 'calexpBackground', 'icSrc', \n",
" 'src', 'srcMatch', 'srcMatchFull', 'ossImage', \n",
" 'flattenedImage', 'wcs', 'fcr', 'photoCalib',\n",
@@ -184,14 +244,14 @@
"metadata": {},
"outputs": [],
"source": [
- "#This would be faster if only one query were issued\n",
+ "# This would be faster if only one query were issued\n",
"visits = butler.queryMetadata('calexp', ['visit'])\n",
"pointings = butler.queryMetadata('calexp', ['pointing'])\n",
"ccds = butler.queryMetadata('calexp', ['ccd'])\n",
"fields = butler.queryMetadata('calexp', ['field'])\n",
"filters = butler.queryMetadata('calexp', ['filter'])\n",
"\n",
- "#Collect number of objects from Source Catalog\n",
+ "# Collect number of objects from Source Catalog\n",
"sources = butler.queryMetadata('src', ['id'])"
]
},
@@ -221,12 +281,10 @@
]
},
{
- "cell_type": "code",
- "execution_count": null,
+ "cell_type": "markdown",
"metadata": {},
- "outputs": [],
"source": [
- "#Work on getting sky area estimate from tract info next"
+ "One key quantity for astronomers is the total sky area imaged. We can estimate this from the coadd tract info."
]
},
{
@@ -235,7 +293,7 @@
"metadata": {},
"outputs": [],
"source": [
- "#Collect tracts from files\n",
+ "# Collect tracts from files\n",
"import os, glob\n",
"tracts = sorted([int(os.path.basename(x)) for x in\n",
" glob.glob(os.path.join(repo, 'deepCoadd-results', 'merged', '*'))])\n",
@@ -251,7 +309,7 @@
"metadata": {},
"outputs": [],
"source": [
- "#calculate area from all tracts\n",
+ "# Calculate area from all tracts\n",
"skyMap = butler.get('deepCoadd_skyMap')\n",
"total_area = 0.0 #deg^2\n",
"plotting_vertices = []\n",
@@ -272,7 +330,7 @@
" #combine areas\n",
" total_area += area\n",
" \n",
- "print(total_area)\n",
+ "print(\"Total area imaged (sq deg): \",total_area)\n",
"\n",
"#round total area for table purposes\n",
"rounded_total_area = round(total_area, 2)\n"
@@ -293,17 +351,22 @@
"metadata": {},
"outputs": [],
"source": [
- "#print out a report of the metadata\n",
+ "# Print out a report of the metadata\n",
+ "\n",
+ "# dataset_name = 'HSC'\n",
+ "# display(Markdown('# Dataset: %s' %dataset_name))\n",
"\n",
+ "# A more automated version of the table title:\n",
"dataset_name = 'HSC'\n",
+ "display(Markdown('# %s' % repo))\n",
+ "\n",
+ "\n",
+ "# Make a table of the collected metadata\n",
"collected_data = [num_visits, num_pointings, num_ccds, num_fields, num_filters, num_sources, \n",
" num_tracts, rounded_total_area]\n",
"data_names = (\"Number of Visits\", \"Number of Pointings\", \"Number of CCDs\", \"Number of Fields\", \n",
" \"Number of Filters\", \"Number of Sources\", \"Number of Tracts\", \"Total Sky Area (deg$^2$)\")\n",
"\n",
- "display(Markdown('# Dataset: %s' %dataset_name))\n",
- "\n",
- "# Make a table of the collected metadata\n",
"output_table = \"| Metadata Characteristics | | \\n | :---: | --- | \\n \"\n",
"counter = 0\n",
"while counter < len(collected_data):\n",
@@ -311,6 +374,7 @@
" counter += 1\n",
"display(Markdown(output_table))\n",
"\n",
+ "# Show which fields and filters we're talking about:\n",
"display(Markdown('Fields: (%i total)' %num_fields))\n",
"print(fields)\n",
"display(Markdown('Filters: (%i total)' %num_filters))\n",
@@ -322,18 +386,28 @@
"execution_count": null,
"metadata": {},
"outputs": [],
- "source": [
- "butler.datasetExists('raw', dataId={})"
- ]
+ "source": []
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
- "source": [
- "butler.get"
- ]
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": []
},
{
"cell_type": "code",
@@ -346,7 +420,10 @@
"cell_type": "markdown",
"metadata": {},
"source": [
- "## Tracts and Patches, Coadd Images\n",
+ "## Sandbox: Initial Exploration of Tracts and Patches, Coadd Images\n",
+ "\n",
+ "_To be deleted eventually..._\n",
+ "\n",
"Let's try exploring the `hsc` dataset's coadd images, and the visits that went into them. Jim Bosch shows how to do this in [this community.lsst.org post](https://community.lsst.org/t/visualizing-source-images-in-a-coadd/441/2).\n",
"\n",
"We'll need a single coadd image to work from."