v7.2 release (#29)

* changes for v7.2 open source release

Author: xuyan-ru

.devcontainer/Dockerfile

@@ -0,0 +1,12 @@
+FROM mcr.microsoft.com/devcontainers/cpp:1.0.4-debian-12
+# Install the xz-utils package
+RUN apt-get update && apt-get install -y xz-utils
+
+RUN apt-get install npm -y && \
+    npm install -g @bazel/bazelisk && \
+    apt-get install libblas-dev -y && \
+    apt-get install liblapack-dev -y 
+
+RUN apt-get install python3-minimal -y && \
+    apt-get install python3-pandas python3-scipy python3-absl python3-pybind11 python3-protobuf awscli -y && \
+    rm -f /usr/bin/python && ln -s /usr/bin/python3 /usr/bin/python

.devcontainer/devcontainer.json

@@ -0,0 +1,7 @@
+{
+    "image": "mcr.microsoft.com/devcontainers/cpp:1.0.4-debian-12",
+    "build": {
+       // Path is relative to the devcontainer.json file
+       "dockerfile": "Dockerfile"
+    },
+}

.dockerignore

@@ -0,0 +1,4 @@
+# Exclude all unnecessary folders from docker
+docs/
+example/
+test/

.github/workflows/ci_process.yml

@@ -0,0 +1,25 @@
+name: ci-testing
+
+# Controls when the workflow will run .
+on:
+  # Triggers the workflow on  pull request events for master
+ pull_request:
+   branches: [ master, dev ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - name: Build the project
+      run: make build_docker
+    - name: Run the project
+      run: |
+        docker-compose up -d
+        sleep 30
+    - name: Run unit tests
+      run: |
+        set -o pipefail
+        make test_lillymol | tee ./unit_test_results.txt
+    - name: Stop running
+      run: docker-compose down

.github/workflows/docker_publish.yml

@@ -0,0 +1,78 @@
+name: Docker
+
+# This workflow uses actions that are not certified by GitHub.
+# They are provided by a third-party and are governed by
+# separate terms of service, privacy policy, and support
+# documentation.
+
+on:
+  push:
+    branches: [ "master" ]
+
+env:
+  # Use docker.io for Docker Hub if empty
+  REGISTRY: ghcr.io
+  # github.repository as <account>/<repo>
+  IMAGE_NAME: ${{ github.repository }}
+
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+      # This is used to complete the identity challenge
+      # with sigstore/fulcio when running outside of PRs.
+      id-token: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v3
+
+      # Install the cosign tool except on PR
+      # https://github.com/sigstore/cosign-installer
+      - name: Install cosign
+        if: github.event_name != 'pull_request'
+        uses: sigstore/cosign-installer@f3c664df7af409cb4873aa5068053ba9d61a57b6 #v2.6.0
+        with:
+          cosign-release: 'v1.11.0'
+
+
+      # Workaround: https://github.com/docker/build-push-action/issues/461
+      - name: Setup Docker buildx
+        uses: docker/setup-buildx-action@79abd3f86f79a9d68a23c75a09a9a85889262adf
+
+      # Login against a Docker registry except on PR
+      # https://github.com/docker/login-action
+      - name: Log into registry ${{ env.REGISTRY }}
+        if: github.event_name != 'pull_request'
+        uses: docker/login-action@28218f9b04b4f3f62068d7b6ce6ca5b26e35336c
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      # Extract metadata (tags, labels) for Docker
+      # https://github.com/docker/metadata-action
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@98669ae865ea3cffbcbaa878cf57c20bbf1c6c38
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+          tags: |
+            type=sha
+
+      # Build and push Docker image with Buildx (don't push on PR)
+      # https://github.com/docker/build-push-action
+      - name: Build and push Docker image
+        id: build-and-push
+        uses: docker/build-push-action@ac9327eae2b366085ac7f6a2d02df8aa8ead720a
+        with:
+          context: .
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max

Dockerfile

@@ -1,34 +1,41 @@
-FROM gcc:13.2
+# builder stage
+FROM gcc:13.2 AS build
 
 RUN apt-get update && \ 
     apt-get upgrade -y
 
 RUN apt-get install npm -y && \
-    npm install -g @bazel/bazelisk
-
-RUN apt-get install libblas-dev -y && \
+    npm install -g @bazel/bazelisk && \
+    apt-get install libblas-dev -y && \
     apt-get install liblapack-dev -y 
 
 RUN apt-get install python3-minimal -y && \
-    apt-get install python3-pandas python3-scipy python3-absl python3-pybind11 python3-protobuf -y
-
-RUN rm -f /usr/bin/python && ln -s /usr/bin/python3 /usr/bin/python
+    apt-get install python3-pandas python3-scipy python3-absl python3-pybind11 python3-protobuf -y && \
+    rm -f /usr/bin/python && ln -s /usr/bin/python3 /usr/bin/python
 
 COPY . ./LillyMol
 
+WORKDIR /LillyMol/src
+
 ENV LILLYMOL_HOME=/LillyMol \
-    BUILD_DIR=Linux
+    BUILD_DIR=Linux \
+    BUILD_BDB=1 \
+    BUILD_PYTHON=1
 
-# This step probably not necessary since now, third party
-# dependencies are all linked static. Maybe that will change.
-ENV LD_LIBRARY_PATH=/LillyMol/third_party/lib
+RUN ./update_bazel_configs.sh && ./build_third_party.sh && ./build_from_src.sh
 
-WORKDIR /LillyMol/src
+# final stage
+FROM ubuntu:mantic AS final
+
+RUN apt-get update && \ 
+    apt-get upgrade -y && \
+    apt-get install python3-minimal -y && \
+    apt-get install python3-pandas python3-scipy python3-absl python3-pybind11 python3-protobuf -y && \
+    rm -f /usr/bin/python && ln -s /usr/bin/python3 /usr/bin/python
 
-ENV BUILD_BDB=1
-ENV BUILD_PYTHON=1
+COPY --from=build /LillyMol /LillyMol
 
-RUN ./build_third_party.sh
-RUN ./build_from_src.sh
+ENV LILLYMOL_HOME=/LillyMol \
+    BUILD_DIR=Linux
 
 WORKDIR /LillyMol

Makefile

@@ -22,27 +22,45 @@
 default:
 	bash -c 'if [[ -z "$$(type -p bazelisk)" && -z "$$(type -p bazel)" ]] ; then echo "No bazel/bazelisk, see README.md" && exit 1 ; fi'
 	echo "Default build does not build targets 'berkeleydb' and 'python'"
+	cd src && ./update_bazel_configs.sh
 	cd src && ./build_third_party.sh
 	cd src && ./build_from_src.sh
 
 all:
 	bash -c 'if [[ -z "$$(type -p bazelisk)" && -z "$$(type -p bazel)" ]] ; then echo "No bazel/bazelisk, see README.md" && exit 1 ; fi'
-	cd src && BUILD_BDB=1 BUILD_PYTHON=1 ./build_third_party.sh
-	cd src && BUILD_BDB=1 BUILD_PYTHON=1 ./build_from_src.sh
+	cd src && BUILD_BDB=1 BUILD_PYTHON=1 BUILD_VENDOR=1 ./update_bazel_configs.sh
+	cd src && BUILD_BDB=1 BUILD_PYTHON=1 BUILD_VENDOR=1 ./build_third_party.sh
+	cd src && BUILD_BDB=1 BUILD_PYTHON=1 BUILD_VENDOR=1 ./build_from_src.sh
 
 berkeleydb:
 	bash -c 'if [[ -z "$$(type -p bazelisk)" && -z "$$(type -p bazel)" ]] ; then echo "No bazel/bazelisk, see README.md" && exit 1 ; fi'
+	cd src && BUILD_BDB=1 BUILD_PYTHON=1 BUILD_VENDOR=1 ./update_bazel_configs.sh
 	cd src && BUILD_BDB=1 ./build_third_party.sh
 	cd src && BUILD_BDB=1 ./build_from_src.sh
 
 python:
 	bash -c 'if [[ -z "$$(type -p bazelisk)" && -z "$$(type -p bazel)" ]] ; then echo "No bazel/bazelisk, see README.md" && exit 1 ; fi'
+	cd src && BUILD_BDB=1 BUILD_PYTHON=1 BUILD_VENDOR=1 ./update_bazel_configs.sh
 	cd src && BUILD_PYTHON=1 ./build_third_party.sh
 	cd src && BUILD_PYTHON=1 ./build_from_src.sh
 
+vendor:
+	bash -c 'if [[ -z "$$(type -p bazelisk)" && -z "$$(type -p bazel)" ]] ; then echo "No bazel/bazelisk, see README.md" && exit 1 ; fi'
+	cd src && BUILD_BDB=1 BUILD_PYTHON=1 BUILD_VENDOR=1 ./update_bazel_configs.sh
+	cd src && BUILD_VENDOR=1 ./build_third_party.sh
+	cd src && BUILD_VENDOR=1 ./build_from_src.sh
+
 build_docker:
-	docker build -f Dockerfile -t lillymol .
+	docker build -f Dockerfile -t lillymolprivate .
 
 test_lillymol:
+	docker container exec lilly_mol bash -c 'cd bin/Linux/ && echo "Number of executables built: `ls -1 | wc -l`" && ls -1'
 	docker container exec lilly_mol bash -c "cd test/ && ./run_all_test.sh"
 	docker container exec lilly_mol bash -c "cd src/ && ./run_python_unit_tests.sh 2>&1"
+
+start_s3:
+	docker-compose up -d
+	sleep 30
+	echo 'run s3 commannd with: aws s3 --endpoint "http://localhost:4566" <s3 command>'
+stop_s3:
+	docker-compose down

README.md

@@ -1,8 +1,5 @@
 # Welcome to the Eli Lilly LillyMol implementation.
 
-![Static Badge](https://img.shields.io/badge/tested_docker_gcc_version-10.2%7C12.3%7C13.2-blue)
-![Static Badge](https://img.shields.io/badge/supported_platform-Linux-green)
-
 ## Background
 LillyMol is a C++ library for Cheminformatics. This repo also contains a variety of useful
 command line tools that have been built with LillyMol.
@@ -75,6 +72,14 @@ module load gcc10
 module load bazelisk
 module load git
 ```
+
+Other system components that are needed, which may or may not be
+available.
+
+* wget
+* unzip
+* libz-dev
+
 ### Python
 If you wish to build the python bindings, you will need a recent version of
 python. Development was done with python3.11 and has not been
@@ -89,11 +94,15 @@ Note that with the default build (below) Python bindings are not built.
 If you have bazelisk and gcc installed, there is a reasonable possibility that
 issuing `make` in the top level directory will work (but see note below
 about NFS filesystems).
+
 ```
-cd /path/to/Lillymol
+# Inside Lilly use the private repo
+git clone https://github.com/EliLillyCo/LillyMol
+
+cd /path/to/LillyMol
 make
 ```
-Executables will be in bin/$(uname) and libraries in lib. More details
+Executables will be in `bin/$(uname)` and libraries in `lib`. More details
 below. There is no concept of installation prefix, everything remains
 in the repo.
 
@@ -103,10 +112,36 @@ are built. If you wish to build either of those
 make python
 make berkeleydb
 ```
+or
+```
+make all
+```
+
+If you look at [Makefile](Makefile) you will see that all it is doing
+is sequentially invoking the three scripts discussed below, possibly with
+
+### Configuring for bazel
+Within the src directory, the file `WORKSPACE` configures the build environment
+for `bazel`. If you are building python bindings, this file needs to be updated
+to reflect the location of your local python. The script `update_bazel_configs.sh`
+does this automatically from the Makefile.
 
-If you look at (Makefile)[Makefile] you will see that all it is doing
-is sequentially invoking the two scripts discussed below, possibly with
-various shell variables set, which enable the optional builds.
+### Installation Directory
+There is an 'install' target in the BUILD files, and defined in
+[build_deps/install.bzl](src/build_deps/install.bzl).
+This is where LillyMol executables will be installed when
+the 'install' run target is run. Again `update_bazel_configs.sh`
+will update this to '/path/to/LillyMol/bin/$(uname)'.
+Check to see that the update has
+been done correctly and adjust if not, or to set another location.
+
+```
+tail build_deps/install.bzl
+```
+That file contains other mechanisms for specifying the install directory.
+But remember, every time `make` is run, that file will be automatically
+updated again. Remove the calls to `update_bazel_configs.sh` from
+the Makefile if needed.
 
 ### C++ Dependencies.
 There are several dependencies which could be installed on the system,
@@ -134,12 +169,9 @@ directory (next to src) and then download, build and install the following depen
 
 - **BerkeleyDb**: used for key/value databases
 - **f2c/libf2c**: there is some fortran in LillyMol.
-- **HighwayHash**: hash function from Google
 
 Running 'build_third_party.sh' needs to be done once.
 
-Currently HighwayHash is impeding deployment on Mac's.
-
 Note that BerkeleyDB and Python bindings are only built if requested. 
 In [Makefile](/Makefile) you will see use of the shell variables
 'BUILD_PYTHON' and 'BUILD_BDB' which if set, enables building of
@@ -152,39 +184,15 @@ re-run the script and all dependencies are downloaded and rebuilt. If there is
 an individual dependency that you would like to rebuild, just remove it from
 the `third_party` directory, run the script again and it will be rebuilt.
 
-Note that [.bazelrc](/src/.bazelrc) contains a hardware restriction to quite old
-Intel hardware. You should update update `--cxxopt` to reflect
-your hardware. Using `--cxxopt=-march=native --cxxopt=-mtune=native` is likely
-what you want. Build for the local hardware.
-
-`WORKSPACE` configures the build environment for `bazel`. The path
-to the external dependencies must be configured in that file. They
-can be either full path names, or path names relative to the main
-WORKSPACE file. In the WORKSPACE file in this distribution, there is a mixture, with
-many of the dependencies in the `../third_party` directory, and
-others with Lilly specific full path names. You may need to
-update WORKSPACE for local needs.
-
-Note that installing these external dependencies and running bazel may require
+Note too that installing these external dependencies and running bazel may require
 considerable amounts of disk space. For example at the time of writing my
 'third_party' directory contains 1.2GB and my bazel temporary area contains 2.2GB.
 
-### Installation Directory
-There is an 'install' target in the BUILD files, and defined in
-[build_deps/install.bzl](src/build_deps/install.bzl).
-This is where LillyMol executables will be installed  when
-the 'install' run target is run.
 
-'build_third_party.sh' changes the default location for executables
-to be '/path/to/LillyMol/bin/$(uname)'. Check to see that the update has
-been done correctly and adjust if not, or to set another location.
-
-```
-tail build_deps/install.bzl
-```
-That file contains other mechanisms for specifying the install directory.
-But remember, every time `build_third_party.sh` runs it will change
-this file.
+Note that [.bazelrc](/src/.bazelrc) contains a hardware restriction to quite old
+Intel hardware. You should update update `--cxxopt` to reflect
+your hardware. Using `--cxxopt=-march=native --cxxopt=-mtune=native` is likely
+what you want. Build for the local hardware.
 
 ### Python Bindings
 During building of external dependencies (with build_third_party.sh