diff --git a/docker/pulsar-all/Dockerfile b/docker/pulsar-all/Dockerfile index 42431fc94a067..11a67c9272f87 100644 --- a/docker/pulsar-all/Dockerfile +++ b/docker/pulsar-all/Dockerfile @@ -25,7 +25,19 @@ ARG PULSAR_OFFLOADER_TARBALL ADD ${PULSAR_IO_DIR} /connectors ADD ${PULSAR_OFFLOADER_TARBALL} / RUN mv /apache-pulsar-offloaders-*/offloaders /offloaders +RUN chmod -R g=u /connectors /offloaders FROM apachepulsar/pulsar:latest -COPY --from=pulsar-all /connectors /pulsar/connectors -COPY --from=pulsar-all /offloaders /pulsar/offloaders + +# Need permission to create directories and update file permissions +USER root + +RUN mkdir /pulsar/connectors /pulsar/offloaders && \ + chown pulsar:root /pulsar/connectors /pulsar/offloaders && \ + chmod g=u /pulsar/connectors /pulsar/offloaders + +# Return to pulsar (non root) user +USER pulsar + +COPY --from=pulsar-all --chown=pulsar:0 /connectors /pulsar/connectors +COPY --from=pulsar-all --chown=pulsar:0 /offloaders /pulsar/offloaders diff --git a/docker/pulsar-standalone/Dockerfile b/docker/pulsar-standalone/Dockerfile index 8555af0646611..55c1fd627e7fe 100644 --- a/docker/pulsar-standalone/Dockerfile +++ b/docker/pulsar-standalone/Dockerfile @@ -26,6 +26,9 @@ FROM apachepulsar/pulsar-dashboard:latest as dashboard # Restart from FROM openjdk:8-jdk +# Help to make these directories persist between container restarts +VOLUME ["/pulsar/conf", "/pulsar/data"] + # Note that the libpq-dev package is needed here in order to install # the required python psycopg2 package (for postgresql) later RUN apt-get update \ diff --git a/docker/pulsar/Dockerfile b/docker/pulsar/Dockerfile index 69b8b6768bc60..625895265aa1f 100644 --- a/docker/pulsar/Dockerfile +++ b/docker/pulsar/Dockerfile @@ -17,7 +17,8 @@ # under the License. # -# First create a stage with just the Pulsar tarball and scripts +# First create a stage with the Pulsar tarball, the scripts, the python client, +# the cpp client, and the data directory. Then ensure correct file permissions. FROM busybox as pulsar ARG PULSAR_TARBALL @@ -33,11 +34,25 @@ COPY scripts/pulsar-zookeeper-ruok.sh /pulsar/bin COPY scripts/watch-znode.py /pulsar/bin COPY scripts/install-pulsar-client-37.sh /pulsar/bin +COPY target/python-client/ /pulsar/pulsar-client +COPY target/cpp-client/ /pulsar/cpp-client + +RUN mkdir /pulsar/data + +# In order to support running this docker image as a container on OpenShift +# the final image needs to give the root group enough permission. +# The file permissions are maintained when copied into the target image. +RUN chmod -R g=u /pulsar + ### Create 2nd stage from OpenJDK image ### and add Python dependencies (for Pulsar functions) FROM openjdk:8-jdk-slim +# Create the pulsar group and user to make docker container run as a non root user by default +RUN groupadd -g 10001 pulsar +RUN adduser -u 10000 --gid 10001 --disabled-login --disabled-password --gecos '' pulsar + # Install some utilities RUN apt-get update \ && apt-get install -y netcat dnsutils less procps iputils-ping \ @@ -52,21 +67,25 @@ RUN python3.7 get-pip.py RUN update-alternatives --install /usr/bin/python python /usr/bin/python3 10 -ADD target/python-client/ /pulsar/pulsar-client -ADD target/cpp-client/ /pulsar/cpp-client +# The pulsar directory is staged correctly in the first stage, above. +# The chown and chmod ensure proper permissions for running as a non root user and non root group +# as well as running on OpenShift with a random user that is part of the root group +RUN mkdir /pulsar && chown pulsar:0 /pulsar && chmod g=u /pulsar +COPY --from=pulsar --chown=pulsar:0 /pulsar /pulsar + RUN echo networkaddress.cache.ttl=1 >> $JAVA_HOME/jre/lib/security/java.security RUN apt-get update \ && apt install -y /pulsar/cpp-client/*.deb \ && apt-get clean \ && rm -rf /var/lib/apt/lists/* -VOLUME ["/pulsar/conf", "/pulsar/data"] - ENV PULSAR_ROOT_LOGGER=INFO,CONSOLE - -COPY --from=pulsar /pulsar /pulsar WORKDIR /pulsar +# This script is intentionally run as the root user to make the dependencies +# available to the root user and the pulsar user RUN /pulsar/bin/install-pulsar-client-37.sh +# Switch to the pulsar user to ensure container defaults to run as a non root user +USER pulsar diff --git a/site2/docs/getting-started-docker.md b/site2/docs/getting-started-docker.md index 56cff28df57a9..f8f6a3888054d 100644 --- a/site2/docs/getting-started-docker.md +++ b/site2/docs/getting-started-docker.md @@ -27,6 +27,11 @@ and follow the instructions for your OS. A few things to note about this command: * The data, metadata, and configuration are persisted on Docker volumes in order to not start "fresh" every time the container is restarted. For details on the volumes you can use `docker volume inspect ` + * The pulsar docker image runs as user 10000, by default. In order for the pulsar process to + access the mounted volumes, the host volumes (`pulsardata` and `pulsarconf` in this example) + must give user 10000 read and write permissions. Alternatively, you can specify the user to run with and then make sure + that the host volume gives that user read and write permissions. Otherwise, the process will fail due to insufficient + permissions. * For Docker on Windows make sure to configure it to use Linux containers If you start Pulsar successfully, you will see `INFO`-level log messages like this: diff --git a/tests/docker-images/latest-version-image/Dockerfile b/tests/docker-images/latest-version-image/Dockerfile index eaa279a42cd11..579372278c201 100644 --- a/tests/docker-images/latest-version-image/Dockerfile +++ b/tests/docker-images/latest-version-image/Dockerfile @@ -19,6 +19,12 @@ FROM apachepulsar/pulsar-all:latest +# Switch to run as the root user to simplify building container and then running +# supervisord. Each of the pulsar components are spawned by supervisord and their +# process configuration files specify that the process will be run as the "pulsar" user +# However, any processes exec'ing into the containers will run as root, by default. +USER root + RUN rm -rf /var/lib/apt/lists/* && apt update RUN apt-get clean && apt-get update && apt-get install -y supervisor vim procps curl git diff --git a/tests/docker-images/latest-version-image/conf/bookie.conf b/tests/docker-images/latest-version-image/conf/bookie.conf index a71cb5c5d3756..f95e2314fbbb8 100644 --- a/tests/docker-images/latest-version-image/conf/bookie.conf +++ b/tests/docker-images/latest-version-image/conf/bookie.conf @@ -24,3 +24,4 @@ stdout_logfile=/var/log/pulsar/bookie.log directory=/pulsar environment=PULSAR_MEM="-Xmx128M -XX:MaxDirectMemorySize=512M",PULSAR_GC="-XX:+UseG1GC",dbStorage_writeCacheMaxSizeMb="16",dbStorage_readAheadCacheMaxSizeMb="16" command=/pulsar/bin/pulsar bookie +user=pulsar diff --git a/tests/docker-images/latest-version-image/conf/broker.conf b/tests/docker-images/latest-version-image/conf/broker.conf index f8bf8e761398f..bfcf6015a3b9c 100644 --- a/tests/docker-images/latest-version-image/conf/broker.conf +++ b/tests/docker-images/latest-version-image/conf/broker.conf @@ -24,4 +24,5 @@ stdout_logfile=/var/log/pulsar/broker.log directory=/pulsar environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseG1GC" command=/pulsar/bin/pulsar broker +user=pulsar diff --git a/tests/docker-images/latest-version-image/conf/functions_worker.conf b/tests/docker-images/latest-version-image/conf/functions_worker.conf index 3610b03251884..a023c1ebe297a 100644 --- a/tests/docker-images/latest-version-image/conf/functions_worker.conf +++ b/tests/docker-images/latest-version-image/conf/functions_worker.conf @@ -24,4 +24,5 @@ stdout_logfile=/var/log/pulsar/functions_worker.log directory=/pulsar environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseG1GC" command=/pulsar/bin/pulsar functions-worker +user=pulsar diff --git a/tests/docker-images/latest-version-image/conf/global-zk.conf b/tests/docker-images/latest-version-image/conf/global-zk.conf index bf56c5b62882a..6503f5f7c1ca5 100644 --- a/tests/docker-images/latest-version-image/conf/global-zk.conf +++ b/tests/docker-images/latest-version-image/conf/global-zk.conf @@ -24,4 +24,5 @@ stdout_logfile=/var/log/pulsar/global-zk.log directory=/pulsar environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseG1GC" command=/pulsar/bin/pulsar configuration-store +user=pulsar diff --git a/tests/docker-images/latest-version-image/conf/local-zk.conf b/tests/docker-images/latest-version-image/conf/local-zk.conf index 57681937791be..1c98a6bbf5695 100644 --- a/tests/docker-images/latest-version-image/conf/local-zk.conf +++ b/tests/docker-images/latest-version-image/conf/local-zk.conf @@ -24,4 +24,5 @@ stdout_logfile=/var/log/pulsar/local-zk.log directory=/pulsar environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseG1GC" command=/pulsar/bin/pulsar zookeeper +user=pulsar diff --git a/tests/docker-images/latest-version-image/conf/presto_worker.conf b/tests/docker-images/latest-version-image/conf/presto_worker.conf index 28e3c36661e09..6846ca2666233 100644 --- a/tests/docker-images/latest-version-image/conf/presto_worker.conf +++ b/tests/docker-images/latest-version-image/conf/presto_worker.conf @@ -23,4 +23,5 @@ redirect_stderr=true stdout_logfile=/var/log/pulsar/presto_worker.log directory=/pulsar environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseG1GC" -command=/pulsar/bin/pulsar sql-worker start \ No newline at end of file +command=/pulsar/bin/pulsar sql-worker start +user=pulsar \ No newline at end of file diff --git a/tests/docker-images/latest-version-image/conf/proxy.conf b/tests/docker-images/latest-version-image/conf/proxy.conf index 8bc1a53fe4fa8..1bed5a170a43a 100644 --- a/tests/docker-images/latest-version-image/conf/proxy.conf +++ b/tests/docker-images/latest-version-image/conf/proxy.conf @@ -24,4 +24,5 @@ stdout_logfile=/var/log/pulsar/proxy.log directory=/pulsar environment=PULSAR_MEM="-Xmx128M",PULSAR_GC="-XX:+UseG1GC" command=/pulsar/bin/pulsar proxy +user=pulsar