diff --git a/ImageProcessing/Re-RunHSC.ipynb b/ImageProcessing/Re-RunHSC.ipynb index 3d53533d..afdbfe02 100644 --- a/ImageProcessing/Re-RunHSC.ipynb +++ b/ImageProcessing/Re-RunHSC.ipynb @@ -12,17 +12,16 @@ "\n", "This project addresses issue [#63: HSC Re-run](https://github.com/LSSTScienceCollaborations/StackClub/issues/63)\n", "\n", - "This notebook demonstrates the [LSST Science Piplines data processing tutorial](https://pipelines.lsst.io/) with emphasis on how a given [obs package](https://github.com/lsst/obs_base) works under the hood of the command line tasks. It makes use of the [obs_subaru](https://github.com/lsst/obs_subaru) package to measure a forced photometry light curve for a small patch in the HSC sky in the [ci_hsc](https://github.com/lsst/ci_hsc/) repository. \n", + "This notebook demonstrates the pipeline described in the [LSST Science Piplines data processing tutorial](https://pipelines.lsst.io/), from ingesting images (using the [obs_subaru](https://github.com/lsst/obs_subaru) package) through image processing, coaddition, source detection and object measurement all the way through to measuring forced photometry light curves in a small patch of the HSC sky (in the [ci_hsc](https://github.com/lsst/ci_hsc/) repository). \n", "\n", "### Learning Objectives:\n", "After working through and studying this notebook you should be able to understand how to use the DRP pipeline from image visualization through to a forced photometry light curve. Specific learning objectives include: \n", - " 1. [configure command-line tasks](https://pipelines.lsst.io/v/w-2018-12/modules/lsst.pipe.base/command-line-task-config-howto.html) for your science case\n", - " 2. TODO\n", + " 1. [Configuring](https://pipelines.lsst.io/v/w-2018-12/modules/lsst.pipe.base/command-line-task-config-howto.html) and executing pipeline tasks in python as well as on the command line.\n", + " 2. The sequence of steps involved in the DRP pipeline.\n", " \n", - "Other techniques that are demonstrated, but not empasized, in this notebook are\n", - " 1. Use the `butler` to fetch data\n", - " 2. Visualize data with the LSST Stack\n", - " 3. TODO\n", + "Other techniques that are demonstrated, but not emphasized, in this notebook are\n", + " 1. Using the `butler` to fetch data\n", + " 2. Visualizing data with the LSST Stack\n", "\n", "### Logistics\n", "This notebook is intended to be runnable on `lsst-lspdev.ncsa.illinois.edu` from a local git clone of https://github.com/LSSTScienceCollaborations/StackClub.\n", @@ -33,7 +32,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -45,6 +44,31 @@ "import eups.setupcmd" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Pipeline Preview\n", + "\n", + "Before we unpack the pipeline described in the [LSST Science Piplines data processing tutorial](https://pipelines.lsst.io/), let's look at the complete set of command line tasks assembled into an end-to-end data reduction script." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "! cat Re-RunHSC.sh" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We'll come back to each step in turn throughout the rest of this notebook." + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -67,15 +91,14 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ - "USER = \"jmyles\"\n", - "# todo: $USER\n", - "datarepo = \"/home/{}/repositories/ci_hsc/\".format(USER)\n", - "datadir = \"/home/{}/DATA/\".format(USER)\n", - "os.system(\"mkdir -p {}\".format(datadir));" + "HOME = os.environ['HOME']\n", + "DATAREPO = \"{}/repositories/ci_hsc/\".format(HOME)\n", + "DATADIR = \"{}/DATA/\".format(HOME)\n", + "os.system(\"mkdir -p {}\".format(DATADIR));" ] }, { @@ -86,7 +109,7 @@ "source": [ "#!setup -j -r /home/jmyles/repositories/ci_hsc\n", "\n", - "setup = eups.setupcmd.EupsSetup([\"-j\",\"-r\", datarepo])\n", + "setup = eups.setupcmd.EupsSetup([\"-j\",\"-r\", DATAREPO])\n", "status = setup.run()\n", "print('setup exited with status {}'.format(status))" ] @@ -104,7 +127,7 @@ "metadata": {}, "outputs": [], "source": [ - "with open(datadir + \"_mapper\", \"w\") as f:\n", + "with open(DATADIR + \"_mapper\", \"w\") as f:\n", " f.write(\"lsst.obs.hsc.HscMapper\")" ] }, @@ -115,8 +138,7 @@ "outputs": [], "source": [ "# ingest script\n", - "# todo: $USER ()\n", - "!ingestImages.py /home/jmyles/DATA /home/jmyles/repositories/ci_hsc/raw/*.fits --mode=link" + "!ingestImages.py DATADIR /home/jmyles/repositories/ci_hsc/raw/*.fits --mode=link" ] }, { @@ -158,8 +180,8 @@ "os.system(\"ln -s {} {}\".format(datarepo + \"CALIB/\", datadir + \"CALIB\"))\n", "\n", "# ingest reference catalog into Butler repo\n", - "os.system(\"mkdir -p {}\".format(datadir + \"ref_cats\"))\n", - "os.system(\"ln -s {}ps1_pv3_3pi_20170110 {}ref_cats/ps1_pv3_3pi_20170110\".format(datarepo, datadir))" + "os.system(\"mkdir -p {}\".format(DATADIR + \"ref_cats\"))\n", + "os.system(\"ln -s {}ps1_pv3_3pi_20170110 {}ref_cats/ps1_pv3_3pi_20170110\".format(DATAREPO, DATADIR))" ] }, { @@ -316,14 +338,13 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import os\n", "import lsst.daf.persistence as dafPersist\n", - "datadir = \"/home/{}/DATA/\".format(os.environ['USER'])\n", - "butler = dafPersist.Butler(inputs=datadir + 'rerun/coaddForcedPhot/')" + "butler = dafPersist.Butler(inputs=DATADIR + 'rerun/coaddForcedPhot/')" ] }, { @@ -335,7 +356,7 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -352,7 +373,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ diff --git a/ImageProcessing/Re-RunHSC.sh b/ImageProcessing/Re-RunHSC.sh index e5155312..69a51201 100644 --- a/ImageProcessing/Re-RunHSC.sh +++ b/ImageProcessing/Re-RunHSC.sh @@ -20,10 +20,11 @@ source /opt/lsst/software/stack/loadLSST.bash eups list lsst_distrib setup lsst_distrib -# I. Setting up the Butler data repository +# I. Setting up the Butler data repository date -echo "setup Butler" +echo "Re-RunHSC INFO: set up the Butler" + setup -j -r /project/shared/data/ci_hsc DATADIR="/home/$USER/DATA" mkdir -p "$DATADIR" @@ -32,55 +33,65 @@ mkdir -p "$DATADIR" # We write this file to the data repository so that any instantiated Butler object knows which mapper to use. echo lsst.obs.hsc.HscMapper > $DATADIR/_mapper -# The injest script creates links in the instantiated butler repository to the original data files +# The ingest script creates links in the instantiated butler repository to the original data files +date +echo "Re-RunHSC INFO: ingest images with ingestImages.py" + ingestImages.py $DATADIR $CI_HSC_DIR/raw/*.fits --mode=link # Grab calibration files +date +echo "Re-RunHSC INFO: obtain calibration files with installTransmissionCurves.py" + installTransmissionCurves.py $DATADIR ln -s $CI_HSC_DIR/CALIB/ $DATADIR/CALIB mkdir -p $DATADIR/ref_cats ln -s $CI_HSC_DIR/ps1_pv3_3pi_20170110 $DATADIR/ref_cats/ps1_pv3_3pi_20170110 -date -echo "processCcd" + # II. Calibrate a single frame with processCcd.py +date +echo "Re-RunHSC INFO: process raw exposures with processCcd.py" + # Use calibration files to do CCD processing processCcd.py $DATADIR --rerun processCcdOutputs --id + # III. (omitted) Visualize images. + # IV. Make coadds -date -echo "make coadds" + # IV. A. Make skymap # A sky map is a tiling of the celestial sphere. It is composed of one or more tracts. # A tract is composed of one or more overlapping patches. Each tract has a WCS. # We define a skymap so that we can warp all of the exposure to fit on a single coordinate system # This is a necessary step for making coadds - date -echo "make skymap" +echo "Re-RunHSC INFO: make skymap with makeDiscreteSkyMap.py" + makeDiscreteSkyMap.py $DATADIR --id --rerun processCcdOutputs:coadd --config skyMap.projection="TAN" + # IV. B. Warp images onto skymap date -echo "warp images" +echo "Re-RunHSC INFO: warp images with makeCoaddTempExp.py" + makeCoaddTempExp.py $DATADIR --rerun coadd \ --selectId filter=HSC-R \ --id filter=HSC-R tract=0 patch=0,0^0,1^0,2^1,0^1,1^1,2^2,0^2,1^2,2 \ --config doApplyUberCal=False doApplySkyCorr=False - makeCoaddTempExp.py $DATADIR --rerun coadd \ --selectId filter=HSC-I \ --id filter=HSC-I tract=0 patch=0,0^0,1^0,2^1,0^1,1^1,2^2,0^2,1^2,2 \ --config doApplyUberCal=False doApplySkyCorr=False - # IV. C. Coadd warped images # Now that we have warped images, we can perform coaddition to get deeper images # The motivation for this is to have the deepest image possible for source detection date -echo "coadd warped images" +echo "Re-RunHSC INFO: coadd warped images with assembleCoadd.py" + assembleCoadd.py $DATADIR --rerun coadd \ --selectId filter=HSC-R \ --id filter=HSC-R tract=0 patch=0,0^0,1^0,2^1,0^1,1^1,2^2,0^2,1^2,2 @@ -89,12 +100,14 @@ assembleCoadd.py $DATADIR --rerun coadd \ --selectId filter=HSC-I \ --id filter=HSC-I tract=0 patch=0,0^0,1^0,2^1,0^1,1^1,2^2,0^2,1^2,2 -# V. Measuring sources -date -echo "measure sources" + +# V. Measuring Sources + # V. A. Source detection # As noted above, we do source detection on the deepest image possible. -echo "detect sources" +date +echo "Re-RunHSC INFO: detect objects in the coadd images with detectCoaddSources.py" + detectCoaddSources.py $DATADIR --rerun coadd:coaddPhot \ --id filter=HSC-R tract=0 patch=0,0^0,1^0,2^1,0^1,1^1,2^2,0^2,1^2,2 @@ -105,27 +118,32 @@ detectCoaddSources.py $DATADIR --rerun coaddPhot \ # Ultimately, for photometry, we will need to deblend objects. # In order to do this, we first merge the detected source catalogs. date -echo "merge detection cats" +echo "Re-RunHSC INFO: merge detection catalogs with mergeCoaddDetections.py" + mergeCoaddDetections.py $DATADIR --rerun coaddPhot --id filter=HSC-R^HSC-I -# V. C. Measure source catalogs on coadds -# Given a full source catalog, we can do regular photometry with implicit deblending. +# V. C. Measure objects in coadds +# Given a full coaddSource catalog, we can do regular photometry with implicit deblending. date -echo "measure source cats on coadds" +echo "Re-RunHSC INFO: measure objects in coadds with measureCoaddSources.py" + measureCoaddSources.py $DATADIR --rerun coaddPhot --id filter=HSC-R measureCoaddSources.py $DATADIR --rerun coaddPhot --id filter=HSC-I -# V. D. Merge multi-band source catalogs from coadds +# V. D. Merge multi-band catalogs from coadds date -echo "merge source cats from coadds" +echo "Re-RunHSC INFO: merge measurements from coadds with mergeCoaddMeasurements.py" + mergeCoaddMeasurements.py $DATADIR --rerun coaddPhot --id filter=HSC-R^HSC-I # V. E. Run forced photometry on coadds # Given a full source catalog, we can do forced photometry with implicit deblending. date -echo "run forcedphot on coadds" +echo "Re-RunHSC INFO: perform forced photometry on coadds with forcedPhotCoadd.py" + forcedPhotCoadd.py $DATADIR --rerun coaddPhot:coaddForcedPhot --id filter=HSC-R forcedPhotCoadd.py $DATADIR --rerun coaddForcedPhot --id filter=HSC-I + # VI. Multi-band catalog analysis # For analysis of the catalog, see part VI of StackClub/ImageProcessing/Re-RunHSC.ipynb