From 3832864475a41c32128585b461005da3bd0f5474 Mon Sep 17 00:00:00 2001 From: Mike Kipnis Date: Tue, 6 May 2025 21:13:17 +0000 Subject: [PATCH] updated docker --- .../src/main/resources/fixclient.cfg | 4 +-- ...buted_ats => Docker.Build_Distributed_ATS} | 6 ++-- docker/Docker.Crypto_CLOB | 12 +++++++ docker/Docker.UST_CLOB | 12 +++++++ docker/Docker.WebTrader | 5 +-- docker/build_distributed_ats.sh | 16 ++------- docker/crypto_clob_env.sh | 22 ++++++++++++ docker/docker-compose-crypto.yml | 31 ++++++++++++++++ docker/docker-compose-ust.yml | 31 ++++++++++++++++ docker/docker-compose.yml | 26 -------------- docker/dockerize_dats.sh | 12 +++++++ ...dockerize_it.sh => dockerize_webtrader.sh} | 6 ---- docker/logging.properties | 17 +++++++++ docker/start_ust_clob.sh | 35 +++++++++++++++++++ docker/ust_clob_env.sh | 21 +++++++++++ 15 files changed, 202 insertions(+), 54 deletions(-) rename docker/{Docker.build_distributed_ats => Docker.Build_Distributed_ATS} (74%) create mode 100644 docker/Docker.Crypto_CLOB create mode 100644 docker/Docker.UST_CLOB create mode 100644 docker/crypto_clob_env.sh create mode 100644 docker/docker-compose-crypto.yml create mode 100644 docker/docker-compose-ust.yml delete mode 100644 docker/docker-compose.yml create mode 100755 docker/dockerize_dats.sh rename docker/{dockerize_it.sh => dockerize_webtrader.sh} (57%) create mode 100644 docker/logging.properties create mode 100755 docker/start_ust_clob.sh create mode 100644 docker/ust_clob_env.sh diff --git a/MiscClients/spring_reactjs/src/main/resources/fixclient.cfg b/MiscClients/spring_reactjs/src/main/resources/fixclient.cfg index b6014d8..c6e2cbf 100644 --- a/MiscClients/spring_reactjs/src/main/resources/fixclient.cfg +++ b/MiscClients/spring_reactjs/src/main/resources/fixclient.cfg @@ -1,8 +1,8 @@ [DEFAULT] ConnectionType=initiator TargetCompID=FIX_GWY_1 -#SocketConnectHost=distributed_ats -SocketConnectHost=127.0.0.1 +SocketConnectHost=distributed_ats +#SocketConnectHost=127.0.0.1 #SocketConnectHost=165.22.179.176 StartTime=00:00:00 EndTime=24:00:00 diff --git a/docker/Docker.build_distributed_ats b/docker/Docker.Build_Distributed_ATS similarity index 74% rename from docker/Docker.build_distributed_ats rename to docker/Docker.Build_Distributed_ATS index 967465f..2a47d64 100644 --- a/docker/Docker.build_distributed_ats +++ b/docker/Docker.Build_Distributed_ATS @@ -1,10 +1,8 @@ FROM ubuntu:latest RUN apt-get update -RUN apt install -y vim cmake curl build-essential libapr1-dev libaprutil1-dev git libboost-all-dev sqlite3 libsqlite3-dev libtool libasio-dev libtinyxml2-dev - -ADD build_deps.sh / -RUN /build_deps.sh +RUN apt install -y vim cmake curl build-essential libapr1-dev libaprutil1-dev git libboost-all-dev sqlite3 libsqlite3-dev libtool libasio-dev libtinyxml2-dev zip python3-pip +RUN pip install psutil --break-system-packages ADD build_distributed_ats.sh / RUN /build_distributed_ats.sh diff --git a/docker/Docker.Crypto_CLOB b/docker/Docker.Crypto_CLOB new file mode 100644 index 0000000..02bd1d6 --- /dev/null +++ b/docker/Docker.Crypto_CLOB @@ -0,0 +1,12 @@ +FROM ghcr.io/mkipnis/distributed_ats:latest + +EXPOSE 15001 +EXPOSE 16001 +EXPOSE 17001 + +WORKDIR /opt/DistributedATS + +COPY crypto_clob_env.sh dats_env.sh +COPY start_crypto_clob.sh /opt/DistributedATS/MiscATS/CryptoCLOB/scripts/ + +RUN chmod +x dats_env.sh diff --git a/docker/Docker.UST_CLOB b/docker/Docker.UST_CLOB new file mode 100644 index 0000000..7770bd7 --- /dev/null +++ b/docker/Docker.UST_CLOB @@ -0,0 +1,12 @@ +FROM ghcr.io/mkipnis/distributed_ats:latest + +EXPOSE 15001 +EXPOSE 16001 +EXPOSE 17001 + +WORKDIR /opt/DistributedATS + +COPY ust_clob_env.sh dats_env.sh +COPY start_ust_clob.sh /opt/DistributedATS/MiscATS/USTreasuryCLOB/scripts/ + +RUN chmod +x dats_env.sh diff --git a/docker/Docker.WebTrader b/docker/Docker.WebTrader index ca0e122..d108ede 100644 --- a/docker/Docker.WebTrader +++ b/docker/Docker.WebTrader @@ -1,6 +1,7 @@ FROM tomcat:9.0 -ENV CATALINA_OUT /dev/stdout - ADD WebTraderRest-0.0.1-SNAPSHOT.war /usr/local/tomcat/webapps/ROOT.war + +COPY logging.properties /usr/local/tomcat/conf/logging.properties + EXPOSE 8080 diff --git a/docker/build_distributed_ats.sh b/docker/build_distributed_ats.sh index 0d28762..9df12f1 100755 --- a/docker/build_distributed_ats.sh +++ b/docker/build_distributed_ats.sh @@ -1,18 +1,6 @@ #!/bin/bash -[[ ! -f /opt/distributed_ats_src ]] && git clone -b FastDDS https://github.com/mkipnis/DistributedATS /opt/distributed_ats_src +[[ ! -f /opt/distributed_ats_src ]] && git clone -b master https://github.com/mkipnis/DistributedATS /opt/distributed_ats_src cd /opt/distributed_ats_src -rm -rf build -mkdir build -cd build - -export DEPS_INSTALL_DIR=/opt/distributed_ats_deps_build/ -cmake .. -Dfastcdr_DIR=$DEPS_INSTALL_DIR/lib/cmake/fastcdr/ -Dfastrtps_DIR=$DEPS_INSTALL_DIR/share/fastrtps/cmake/ -Dfoonathan_memory_DIR=$DEPS_INSTALL_DIR/lib/foonathan_memory/cmake/ -Dlog4cxx_DIR=$DEPS_INSTALL_DIR/lib/cmake/log4cxx -DCMAKE_INSTALL_PREFIX=/opt/DistributedATS -DLIQUIBOOK_HOME=$DEPS_INSTALL_DIR/liquibook/src -DQUICKFIX_INSTALL_PREFIX=$DEPS_INSTALL_DIR - -cmake --build . --target install --config Debug - -chmod +x /opt/DistributedATS/scripts/*.sh -chmod +x /opt/DistributedATS/MiscATS/MultiMatchingEngineATS/scripts/*.sh -chmod +x /opt/DistributedATS/MiscATS/USTreasuryCLOB/scripts/*.sh -chmod +x /opt/DistributedATS/MiscATS/CryptoCLOB/scripts/*.sh +HOME=/opt ./download_deps_and_build_all.sh diff --git a/docker/crypto_clob_env.sh b/docker/crypto_clob_env.sh new file mode 100644 index 0000000..fc6da81 --- /dev/null +++ b/docker/crypto_clob_env.sh @@ -0,0 +1,22 @@ +export DATS_HOME=/opt/DistributedATS +export DEPS_HOME=/opt/distributed_ats_deps_build +export LD_LIBRARY_PATH=$DEPS_HOME/lib:$DATS_HOME/lib:$LD_LIBRARY_PATH +export LOG4CXX_CONFIGURATION=/opt/DistributedATS/config/log4cxx.xml + +#echo Specific Specific + +#echo Crypto ATS +export BASEDIR_ATS=$DATS_HOME/MiscATS/CryptoCLOB +export DATS_LOG_HOME=$BASEDIR_ATS/logs +mkdir -p $BASEDIR_ATS/logs + +#echo US Treasuries +#export BASEDIR_ATS=$DATS_HOME/MiscATS/USTreasuryCLOB +#export DATS_LOG_HOME=$BASEDIR_ATS/logs +#mkdir -p $BASEDIR_ATS/logs + +#echo US Treasuries +#export BASEDIR_ATS=$DATS_HOME/MiscATS/MultiMatchingEngineATS +#export DATS_LOG_HOME=$BASEDIR_ATS/logs +#mkdir -p $BASEDIR_ATS/logs + diff --git a/docker/docker-compose-crypto.yml b/docker/docker-compose-crypto.yml new file mode 100644 index 0000000..c5ea1e0 --- /dev/null +++ b/docker/docker-compose-crypto.yml @@ -0,0 +1,31 @@ +version: '2' + +services: + # Core Services: Matching Engines, FIX Gateways + distributed_ats: + container_name: distributed_ats + image: ghcr.io/mkipnis/dats_crypto_clob:latest + command: > + bash -c "cd /opt/DistributedATS/ && + . ./dats_env.sh && + cd /opt/DistributedATS/MiscATS/CryptoCLOB/scripts && ./start_crypto_clob.sh" + volumes: + - ./data_ats:/opt/DistributedATS/DataService/sql/sqlite + - ./logs_ats:/opt/DistributedATS/MiscATS/CryptoCLOB/logs + ports: + - "15001:15001" + - "16001:16001" + - "17001:17001" + restart: unless-stopped + + # WebTrader Front-End + distributed_ats_webtrader: + container_name: distributed_ats_webtrader + image: ghcr.io/mkipnis/distributed_ats_webtrader:latest + depends_on: + - distributed_ats + volumes: + - ./webtrader_logs:/usr/local/tomcat/logs + ports: + - "8080:8080" + restart: unless-stopped diff --git a/docker/docker-compose-ust.yml b/docker/docker-compose-ust.yml new file mode 100644 index 0000000..44104d7 --- /dev/null +++ b/docker/docker-compose-ust.yml @@ -0,0 +1,31 @@ +version: '2' + +services: + # Core Services: Matching Engines, FIX Gateways + distributed_ats: + container_name: distributed_ats + image: ghcr.io/mkipnis/dats_ust_clob:latest + command: > + bash -c "cd /opt/DistributedATS/ && + . ./dats_env.sh && + cd /opt/DistributedATS/MiscATS/USTreasuryCLOB/scripts && ./start_ust_clob.sh" + volumes: + - ./data_ats:/opt/DistributedATS/DataService/sql/sqlite + - ./logs_ats:/opt/DistributedATS/MiscATS/USTreasuryCLOB/logs + ports: + - "15001:15001" + - "16001:16001" + - "17001:17001" + restart: unless-stopped + + # WebTrader Front-End + distributed_ats_webtrader: + container_name: distributed_ats_webtrader + image: ghcr.io/mkipnis/distributed_ats_webtrader:latest + depends_on: + - distributed_ats + volumes: + - ./webtrader_logs:/usr/local/tomcat/logs + ports: + - "8080:8080" + restart: unless-stopped diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml deleted file mode 100644 index 528d664..0000000 --- a/docker/docker-compose.yml +++ /dev/null @@ -1,26 +0,0 @@ -version: '2' -services: - # FastDDS Discovery - fastdds_discovery: - image: ghcr.io/mkipnis/distributed_ats_fast_dds - command: ["bash", "-c", "cd /opt/DistributedATS/; . ./dats_env.sh; cd /opt/distributed_ats_deps_build/bin; ./fast-discovery-server -i 0"] - ports: # UDP Discovery - - "11811:11811/udp" - - # Core: Matching Engines, Data Services, FIX Gateways - distributed_ats: - image: ghcr.io/mkipnis/distributed_ats_fast_dds - command: ["bash", "-c", "sleep 10; cd /opt/DistributedATS/; . ./dats_env.sh; cd /opt/DistributedATS/MiscATS/CryptoCLOB/scripts; ROS_DISCOVERY_SERVER='UDPv4:[0.0.0.0]:11811' & ./start_ats.sh"] - volumes: # Users and Instruments - # - "./data:/opt/DistributedATS/DataService/sql/sqlite" - - "./logs:/opt/DistributedATS/MiscATS/CryptoCLOB/logs" - ports: # FIX Gateways - - "15001:15001" - - "16001:16001" - - "17001:17001" - - # Front-End: Tomcat/Spring/React - distributed_ats_webtrader: - image: ghcr.io/mkipnis/distributed_ats_webtrader:latest - ports: - - "8080:8080" diff --git a/docker/dockerize_dats.sh b/docker/dockerize_dats.sh new file mode 100755 index 0000000..0175662 --- /dev/null +++ b/docker/dockerize_dats.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +set -x + +# Core +docker build -t ghcr.io/mkipnis/distributed_ats:latest -f Docker.Build_Distributed_ATS . +docker build --no-cache -t ghcr.io/mkipnis/dats_crypto_clob:latest -f Docker.Crypto_CLOB . +docker build --no-cache -t ghcr.io/mkipnis/dats_ust_clob:latest -f Docker.UST_CLOB . + +docker push ghcr.io/mkipnis/distributed_ats:latest +docker push ghcr.io/mkipnis/dats_crypto_clob:latest +docker push ghcr.io/mkipnis/dats_ust_clob:latest diff --git a/docker/dockerize_it.sh b/docker/dockerize_webtrader.sh similarity index 57% rename from docker/dockerize_it.sh rename to docker/dockerize_webtrader.sh index 74c2623..78f0cf9 100755 --- a/docker/dockerize_it.sh +++ b/docker/dockerize_webtrader.sh @@ -2,11 +2,6 @@ set -x -# Core -#docker build -t distributed_ats_fast_dds -f Docker.build_distributed_ats . -docker build -t ghcr.io/mkipnis/distributed_ats_fast_dds:latest -f Docker.build_distributed_ats . -docker push ghcr.io/mkipnis/distributed_ats_fast_dds:latest - # Front-end DOCKER_DIR=`pwd` ln -sf ../MiscClients/spring_reactjs/ WebTrader @@ -17,6 +12,5 @@ mvn install cp target/WebTraderRest-0.0.1-SNAPSHOT.war $DOCKER_DIR cd $DOCKER_DIR -#docker build -t distributed_ats_webtrader -f Docker.WebTrader . docker build -t ghcr.io/mkipnis/distributed_ats_webtrader:latest -f Docker.WebTrader . docker push ghcr.io/mkipnis/distributed_ats_webtrader:latest diff --git a/docker/logging.properties b/docker/logging.properties new file mode 100644 index 0000000..b65e90a --- /dev/null +++ b/docker/logging.properties @@ -0,0 +1,17 @@ +# Default global logging level. +.level = INFO + +# Console handler configuration +handlers = java.util.logging.ConsoleHandler + +# Console handler level and formatter +java.util.logging.ConsoleHandler.level = FINE +java.util.logging.ConsoleHandler.formatter = java.util.logging.SimpleFormatter + +# Set logging levels for different components +org.apache.catalina.level = INFO +org.apache.coyote.level = INFO +org.apache.tomcat.util.net.level = INFO + +# Optional: increase granularity +org.apache.level = FINE diff --git a/docker/start_ust_clob.sh b/docker/start_ust_clob.sh new file mode 100755 index 0000000..8642b2a --- /dev/null +++ b/docker/start_ust_clob.sh @@ -0,0 +1,35 @@ +#!/bin/bash + +trap cleanup 1 2 3 6 + +cleanup() +{ + echo "Caught Signal ... cleaning up..." + + $BASEDIR_ATS/scripts/stop_ats.sh + + echo "Done." + + exit 1; +} + +sleep 1 +$DATS_HOME/scripts/matchingengine.sh start matching_engine_MARKET_UST.ini + +sleep 1 +$DATS_HOME/scripts/dataservice.sh start data_service_a.ini +$DATS_HOME/scripts/dataservice.sh start data_service_b.ini + +sleep 1 +$DATS_HOME/scripts/fixgateway.sh start fix_gwy_1.cfg +$DATS_HOME/scripts/fixgateway.sh start fix_gwy_2.cfg + +while true; do + $DATS_HOME/scripts/dataservice.sh check data_service_a.ini + $DATS_HOME/scripts/dataservice.sh check data_service_b.ini + $DATS_HOME/scripts/matchingengine.sh check matching_engine_MARKET_UST.ini + $DATS_HOME/scripts/fixgateway.sh check fix_gwy_1.cfg + $DATS_HOME/scripts/fixgateway.sh check fix_gwy_2.cfg + + sleep 10; +done diff --git a/docker/ust_clob_env.sh b/docker/ust_clob_env.sh new file mode 100644 index 0000000..df7da7e --- /dev/null +++ b/docker/ust_clob_env.sh @@ -0,0 +1,21 @@ +export DATS_HOME=/opt/DistributedATS +export DEPS_HOME=/opt/distributed_ats_deps_build +export LD_LIBRARY_PATH=$DEPS_HOME/lib:$DATS_HOME/lib:$LD_LIBRARY_PATH +export LOG4CXX_CONFIGURATION=/opt/DistributedATS/config/log4cxx.xml + +#echo Specific Specific + +#echo Crypto ATS +#export BASEDIR_ATS=$DATS_HOME/MiscATS/CryptoCLOB +#export DATS_LOG_HOME=$BASEDIR_ATS/logs +#mkdir -p $BASEDIR_ATS/logs + +#echo US Treasuries +export BASEDIR_ATS=$DATS_HOME/MiscATS/USTreasuryCLOB +export DATS_LOG_HOME=$BASEDIR_ATS/logs +mkdir -p $BASEDIR_ATS/logs + +#echo US Treasuries +#export BASEDIR_ATS=$DATS_HOME/MiscATS/MultiMatchingEngineATS +#export DATS_LOG_HOME=$BASEDIR_ATS/logs +#mkdir -p $BASEDIR_ATS/logs