Skip to content

Commit 62fb1db

Browse files
mcduffiemcduffie
committed
Update L1A Dockerfile to work with Jupyterhub base image
1 parent a86a8db commit 62fb1db

File tree

5 files changed

+81
-57
lines changed

5 files changed

+81
-57
lines changed

‎.env‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,2 +1,2 @@
1-
DOCKER_GROUP=unity-sds
1+
DOCKER_GROUP=unity-ads
22
DOCKER_TAG=r0.1.0

‎Dockerfile.l1a‎

Lines changed: 72 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -1,81 +1,104 @@
1-
FROM unity-sds/sounder_sips_pge_common:r0.1.0 AS builder
1+
FROM unity-ads/sounder_sips_dev:r0.1.0 AS builder
22

3-
ARG BUILD_PATH /build
4-
ARG NUM_COMPILE_JOBS=4
3+
# Locations of paths for building source code
4+
ARG BUILDER_BASE_PATH=/build
5+
ARG BUILDER_SCRIPTS_PATH=$BUILDER_BASE_PATH/scripts
6+
ARG BUILDER_CODE_PATH=$BUILDER_BASE_PATH/src
7+
ARG BUILDER_CONFIG_PATH=$BUILDER_BASE_PATH/config
58

6-
RUN cd $BUILD_PATH/spss/src/sips_pge/l1a_atms/make && make -j $NUM_COMPILE_JOBS
9+
# Local file that is copied into the image then untarred
10+
ARG CODE_SOURCE_FILE
11+
ARG CODE_DEST_FILE=$BUILDER_BASE_PATH/sips_spss.tar.gz
712

8-
FROM centos:7
13+
USER root
14+
15+
# Create directories
16+
RUN mkdir $BUILDER_BASE_PATH && \
17+
mkdir -p $BUILDER_CODE_PATH && \
18+
mkdir -p $BUILDER_SCRIPTS_PATH
19+
20+
# Copy in source code and scripts
21+
COPY $CODE_SOURCE_FILE $CODE_DEST_FILE
22+
COPY install/build_common.sh install/build_l1a.sh install/patch_source.sh install/create_l1a_static_config.sh ${BUILDER_SCRIPTS_PATH}/
23+
24+
# Build source code
25+
RUN tar zfx $CODE_DEST_FILE --strip-components=1 -C $BUILDER_CODE_PATH && \
26+
$BUILDER_SCRIPTS_PATH/patch_source.sh $BUILDER_CODE_PATH && \
27+
$BUILDER_SCRIPTS_PATH/build_common.sh $BUILDER_CODE_PATH && \
28+
$BUILDER_SCRIPTS_PATH/build_l1a.sh $BUILDER_CODE_PATH && \
29+
$BUILDER_SCRIPTS_PATH/create_l1a_static_config.sh $BUILDER_CODE_PATH $BUILDER_CONFIG_PATH
30+
31+
# ---
32+
33+
FROM jupyter/base-notebook
34+
35+
# Renew usage of these values from the previous stage, these default values
36+
# need to mirror their values in the first stage
37+
ARG BUILDER_BASE_PATH=/build
38+
ARG BUILDER_CODE_PATH=$BUILDER_BASE_PATH/src
39+
ARG BUILDER_CONFIG_PATH=$BUILDER_BASE_PATH/config
40+
41+
# There are used by the notebook to override defaults
42+
ENV PGE_BINARY_PATH=/pge/bin
43+
ENV PGE_INTERFACE_PATH=/pge/interface
44+
ENV PGE_STATIC_DATA_PATH=/tmp/static
45+
ENV PGE_STATIC_CONFIG_PATH=/pge/config
946

1047
ENV PGSHOME=/opt/packages/sdptk5.2.20v1.01/TOOLKIT/
1148
ENV PGS_PC_INFO_FILE=/pge/static/SNDR.PGSToolkit_ProcessControlFile.pcf
1249
ENV PGS_LEAPSEC_FILE=${PGEHOME}/database/common/TD/leapsec.dat
1350

51+
# Switch to root for installs
52+
USER root
53+
1454
# Ensure using UTF-8
1555
RUN localedef -c -f UTF-8 -i en_US en_US.UTF-8
1656

1757
# Install packages required for this PGE
1858
# Packages in the second yum call come from EPEL
19-
RUN yum install -y libquadmath epel-release && \
20-
yum install -y python2-netcdf4 && \
21-
yum install -y python3-pip python3-netcdf4 && \
22-
pip3 install -U pip && \
23-
pip3 install notebook papermill && \
24-
yum clean all && \
25-
rm -rf /var/cache/yum
59+
ENV DEBIAN_FRONTEND noninteractive
60+
RUN apt-get update --yes && \
61+
apt-get install --yes --no-install-recommends \
62+
# libturbojpeg0-dev is used by hdf4 library instead of the installed jpeg-9b version
63+
# Even though boost is installed into /opt/packages, some of the SIPS PGEs use a system wide version
64+
libturbojpeg libboost-dev python3-netcdf4 \
65+
# For custom GCC
66+
libgmpxx4ldbl libmpfi0 libmpc3 \
67+
# Needed by netcdf libraries
68+
libcurl4 \
69+
# Needed by SDPTK library
70+
csh \
71+
# Needed by g2clib and SPSS software, contains the static library as well
72+
libpng-dev && \
73+
apt-get clean && rm -rf /var/lib/apt/lists/*
2674

2775
# Create directory structure for PGE files and necessary files
28-
RUN mkdir -p /pge/bin && \
29-
mkdir -p /pge/static && \
30-
mkdir -p /pge/interface && \
76+
RUN mkdir -p $PGE_BINARY_PATH && \
77+
mkdir -p $PGE_STATIC_CONFIG_PATH && \
78+
mkdir -p $PGE_INTERFACE_PATH && \
3179
mkdir -p /opt/packages
3280

3381
# Copy L1A PGE binary from build
34-
COPY --from=builder /build/spss/src/sips_pge/l1a_atms/pge/bin/L1AMw_main /pge/bin
82+
COPY --from=builder $BUILDER_CODE_PATH/src/sips_pge/l1a_atms/pge/bin/L1AMw_main $PGE_BINARY_PATH
3583

3684
# Copy MetExtractor binary from build
37-
COPY --from=builder /build/spss/src/scf_metextractors/main/bin/MetExtractor /pge/bin
85+
COPY --from=builder $BUILDER_CODE_PATH/src/scf_metextractors/main/bin/MetExtractor $PGE_BINARY_PATH
3886

3987
# Copy PGE interface notebook
40-
COPY interface/run_l1a_pge.ipynb /pge/interface
88+
COPY interface/run_l1a_pge.ipynb $PGE_INTERFACE_PATH
4189

4290
# Copy over SDP Toolkit for its database files
4391
COPY --from=builder /opt/packages/sdptk5.2.20v1.01 /opt/packages/sdptk5.2.20v1.01
4492

45-
# Update leap seconds file. This file needs to be periodically updated
46-
COPY ephemeris/leapsec.dat /opt/packages/sdptk5.2.20v1.01/TOOLKIT/database/common/TD/leapsec.dat
47-
COPY ephemeris/utcpole.dat /opt/packages/sdptk5.2.20v1.01/TOOLKIT/database/common/CSC/utcpole.dat
48-
49-
# Copy files from acceptance test directories, these are referenced by the input XML file
50-
COPY src/src/sips_pge/l1a_atms_snpp/acctest/in/pcf/SNDR.PGSToolkit_ProcessControlFile.pcf /pge/static
51-
COPY src/src/sips_pge/l1a_atms_snpp/acctest/in/SNDR.SNPP.ATMS.L1A.sfif_201214135000.xml /pge/static
52-
COPY src/src/sips_pge/l1a_atms_snpp/acctest/in/SNDR.SchemaParameterfile.060401120000.xsd /pge/static
53-
54-
# Modify SFIF file to point to Docker paths
55-
RUN sed -i 's|../../../static|/pge/static|' /pge/static/SNDR.SNPP.ATMS.L1A.sfif_201214135000.xml
56-
57-
# Modify to point to utcpole.dat in the database location
58-
RUN sed -i -e 's|utcpole_20201220.dat|utcpole.dat|' -e 's|../../in/pcf|~/database/common/CSC|' /pge/static/SNDR.PGSToolkit_ProcessControlFile.pcf
59-
60-
# Modify to point DEMs to a path controlled by Docker
61-
RUN sed -i -e 's|/peate/support/static/dem|/tmp/static/dem|' -e 's|/ref/devstable/STORE/mcf|/tmp/static/mcf|' /pge/static/SNDR.PGSToolkit_ProcessControlFile.pcf
62-
63-
# Static files referenced in SFIF file
64-
COPY src/src/sips_pge/l1a_atms_snpp/static/SNDR.SNPP.ATMS.L1A.template.v02_02_08_201214135000.nc /pge/static
65-
COPY src/src/sips_pge/l1a_atms_snpp/static/ATMS-SDR-CC_npp_20131201000000Z_20140101000000Z_ee00000000000000Z_PS-1-O-CCR-14-1487-JPSS-DPA-008-SIDEA-PE_noaa_all_all-_all.xml /pge/static
66-
COPY src/src/sips_pge/l1a_atms_snpp/static/SNDR.SNPP.ATMS.L1A.calibration_data_200204184500.csv /pge/static
67-
COPY src/src/sips_pge/l1a_atms_snpp/static/SNDR.SNPP.ATMS.APID_531_ENGHSKP_v11_200204184500.xml /pge/static
68-
COPY src/src/sips_pge/l1a_atms_snpp/static/SNDR.SNPP.ATMS.L1A.apf_180412120000.xml /pge/static
69-
COPY src/src/sips_pge/l1a_atms_snpp/static/SNDR.SIPS.SNPP.ATMS.L1A.SPDCMetConstants_170905120000.pev /pge/static
70-
COPY src/src/sips_pge/l1a_atms_snpp/static/SNDR.SIPS.ATMS.L1A.SPDCMetStructure_161130151628.xml /pge/static
71-
COPY src/src/sips_pge/l1a_atms_snpp/static/SNDR.SIPS.SNPP.ATMS.L1A.SPDCMetMappings_170905120000.xml /pge/static
72-
73-
# Template PGE configuration file modified by Jupyter notebook
74-
COPY src/src/sips_pge/l1a_atms_snpp/acctest/spdc_nominal2/in/SNDR.SNPP.ATMS.L1A.nominal2.config_201214135000.xml /pge/static/pge_config_template.xml
93+
# Copy over static config files
94+
COPY --from=builder $BUILDER_CONFIG_PATH $PGE_CONFIG_PATH
7595

7696
# Necessary for click framework used by papermill:
7797
# https://click.palletsprojects.com/en/8.0.x/unicode-support/
7898
ENV LANG=C.UTF-8
7999
ENV LC_ALL=en_US.UTF-8
80100

81-
ENTRYPOINT ["papermill", "/pge/interface/run_l1a_pge.ipynb"]
101+
# Switch back to jovyan to avoid accidental container runs as root
102+
USER ${NB_UID}
103+
104+
ENTRYPOINT ["papermill", "$PGE_INTERFACE_PATH/run_l1a_pge.ipynb"]

‎docker-compose.yml‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,11 +5,13 @@ services:
55
image: ${DOCKER_GROUP}/sounder_sips_l1a_pge:${DOCKER_TAG}
66
hostname: local
77
build:
8-
context: spss/
8+
context: ./
99
dockerfile: $PWD/Dockerfile.l1a
10+
args:
11+
CODE_SOURCE_FILE: $CODE_SOURCE_FILE
1012
l1b_pge:
1113
image: ${DOCKER_GROUP}/sounder_sips_l1b_pge:${DOCKER_TAG}
1214
hostname: local
1315
build:
14-
context: spss/
16+
context: ./
1517
dockerfile: $PWD/Dockerfile.l1b

‎install/create_l1a_static_config.sh‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,7 @@
11
#!/bin/bash
22

3+
set -e
4+
35
DEFAULT_STATIC_FILES_DIR="/unity/ads/sounder_sips/static_files/"
46

57
usage () {
@@ -30,10 +32,6 @@ if [ -z "$static_dir" ]; then
3032
static_dir=$DEFAULT_STATIC_FILES_DIR
3133
fi
3234

33-
# Copy SDPTK toolkit files
34-
cp $script_dir/../ephemeris/leapsec.dat $dst_dir
35-
cp $script_dir/../ephemeris/utcpole.dat $dst_dir
36-
3735
# Copy files from acceptance test directories, these are referenced by the input XML file
3836
cp $src_dir/src/sips_pge/l1a_atms_snpp/acctest/in/pcf/SNDR.PGSToolkit_ProcessControlFile.pcf $dst_dir
3937
cp $src_dir/src/sips_pge/l1a_atms_snpp/acctest/in/SNDR.SNPP.ATMS.L1A.sfif_201214135000.xml $dst_dir

‎install/patch_source.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,4 +26,5 @@ sed -i \
2626
-e 's/libpng15.a/libpng16.a/' \
2727
$modified_file
2828

29-
diff -u $bak_file $modified_file
29+
# The || true here prevents exiting with a non-zero exit code
30+
diff -u $bak_file $modified_file || true

0 commit comments

Comments
 (0)