diff --git a/.github/workflows/rerun-ci.yml b/.github/workflows/rerun-ci.yml index d2282e0092..3fdd4c0f56 100644 --- a/.github/workflows/rerun-ci.yml +++ b/.github/workflows/rerun-ci.yml @@ -18,6 +18,7 @@ env: jobs: decide-rerun-action: + if: ${{ github.event.workflow_run.conclusion == 'failure' }} runs-on: ubuntu-latest outputs: action: ${{ steps.decision.outputs.action }} @@ -28,19 +29,16 @@ jobs: WORKFLOW_NAME: ${{ github.event.workflow_run.name }} RUN_ID: ${{ github.event.workflow_run.id }} RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} - CONCLUSION: ${{ github.event.workflow_run.conclusion }} EVENT_NAME: ${{ github.event.workflow_run.event }} HEAD_BRANCH: ${{ github.event.workflow_run.head_branch }} run: | set -euo pipefail action="skip" - reason="non-failure" + reason="unsupported event: $EVENT_NAME" - if [[ "$CONCLUSION" == "failure" ]]; then - if [[ "$EVENT_NAME" != "push" && "$EVENT_NAME" != "pull_request" ]]; then - reason="unsupported event: $EVENT_NAME" - elif (( RUN_ATTEMPT > MAX_RERUNS )); then + if [[ "$EVENT_NAME" == "push" || "$EVENT_NAME" == "pull_request" ]]; then + if (( RUN_ATTEMPT > MAX_RERUNS )); then reason="retry limit reached" else action="rerun" diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index f91bd809a3..266e70feb1 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -100,3 +100,76 @@ jobs: with: token: ${{ secrets.CODECOV_TOKEN }} file: ${{ env.REPORT_DIR }}/*.xml + + build-server-macos-rocksdb: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + include: + - os: macos-15-intel + server_java_options: '' + - os: macos-15 + server_java_options: '-Xms512m -Xmx2g' + env: + USE_STAGE: 'false' # Whether to include the stage repository. + TRAVIS_DIR: hugegraph-server/hugegraph-dist/src/assembly/travis + REPORT_DIR: target/site/jacoco + BACKEND: rocksdb + JAVA_VERSION: '11' + SERVER_JAVA_OPTIONS: ${{ matrix.server_java_options }} + + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + fetch-depth: 5 + + - name: Install Java ${{ env.JAVA_VERSION }} + uses: actions/setup-java@v4 + with: + java-version: ${{ env.JAVA_VERSION }} + distribution: 'zulu' + + - name: Cache Maven packages + uses: actions/cache@v4 + with: + path: ~/.m2 + key: ${{ runner.os }}-m2-${{ hashFiles('**/pom.xml') }} + restore-keys: ${{ runner.os }}-m2 + + - name: Use staged maven repo settings + if: ${{ env.USE_STAGE == 'true' }} + run: | + cp $HOME/.m2/settings.xml /tmp/settings.xml + cp -vf .github/configs/settings.xml $HOME/.m2/settings.xml && cat $HOME/.m2/settings.xml + + - name: Compile + run: | + mvn clean compile -pl hugegraph-server/hugegraph-test -am -U -Dmaven.javadoc.skip=true -ntp + + - name: Run RocksDB core test + run: | + $TRAVIS_DIR/run-core-test.sh $BACKEND + + - name: Run RocksDB API test + run: | + $TRAVIS_DIR/run-api-test.sh $BACKEND $REPORT_DIR + + - name: Show server log on failure + if: failure() + run: | + VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) + SERVER_DIR=hugegraph-server/apache-hugegraph-server-$VERSION/ + if [ -f "$SERVER_DIR/logs/hugegraph-server.log" ]; then + tail -n 200 "$SERVER_DIR/logs/hugegraph-server.log" + fi + + - name: Stop RocksDB server + if: always() + run: | + VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) + SERVER_DIR=hugegraph-server/apache-hugegraph-server-$VERSION/ + if [ -f "$SERVER_DIR/bin/pid" ]; then + $TRAVIS_DIR/stop-server.sh $SERVER_DIR || true + fi diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/build-report.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/build-report.sh index 0e17491dd5..920914267d 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/build-report.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/build-report.sh @@ -21,22 +21,53 @@ BACKEND=$1 JACOCO_PORT=$2 JACOCO_REPORT_FILE=$3 +TRAVIS_DIR=$(cd "$(dirname "$0")" && pwd) +REPO_ROOT=$(cd "$TRAVIS_DIR/../../../../.." && pwd) + +function command_available() { + local cmd=$1 + [[ -x "$(command -v "$cmd")" ]] +} + +function download_to_dir() { + local dir=$1 + local url=$2 + local file="$dir/$(basename "$url")" + + mkdir -p "$dir" + if command_available "curl"; then + curl -fL "$url" -o "$file" + elif command_available "wget"; then + wget -P "$dir" "$url" + else + echo "Required curl or wget but they are unavailable" + exit 1 + fi +} + OPTION_CLASS_FILES_BACKEND="--classfiles hugegraph-$BACKEND/target/classes/org/apache/hugegraph" if [ "$BACKEND" == "memory" ]; then # hugegraph-memory is the same as hugegraph-core OPTION_CLASS_FILES_BACKEND="" fi -cd hugegraph-server/hugegraph-test +case "$JACOCO_REPORT_FILE" in + /*) REPORT_FILE=$JACOCO_REPORT_FILE ;; + *) REPORT_FILE="$REPO_ROOT/$JACOCO_REPORT_FILE" ;; +esac +mkdir -p "$(dirname "$REPORT_FILE")" + +cd "$REPO_ROOT/hugegraph-server/hugegraph-test" mvn jacoco:dump@pull-test-data -Dapp.host=localhost -Dapp.port=$JACOCO_PORT -Dskip.dump=false -cd ../ +cd "$REPO_ROOT/hugegraph-server" if [[ ! -e "${TRAVIS_DIR}/jacococli.jar" ]]; then - wget -P "${TRAVIS_DIR}" https://github.com/apache/hugegraph-doc/raw/binary-1.0/dist/server/jacococli.jar + download_to_dir "${TRAVIS_DIR}" \ + "https://github.com/apache/hugegraph-doc/raw/binary-1.0/dist/server/jacococli.jar" fi java -jar $TRAVIS_DIR/jacococli.jar report hugegraph-test/target/jacoco-it.exec \ --classfiles hugegraph-dist/target/classes/org/apache/hugegraph \ --classfiles hugegraph-api/target/classes/org/apache/hugegraph \ --classfiles hugegraph-core/target/classes/org/apache/hugegraph \ - ${OPTION_CLASS_FILES_BACKEND} --xml "${JACOCO_REPORT_FILE}" + ${OPTION_CLASS_FILES_BACKEND} --xml "${REPORT_FILE}" diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test.sh index 14ea659527..2edeb9ce6c 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/run-api-test.sh @@ -21,7 +21,42 @@ BACKEND=$1 REPORT_DIR=$2 REPORT_FILE=$REPORT_DIR/jacoco-api-test-for-raft.xml -TRAVIS_DIR=$(dirname $0) +TRAVIS_DIR=$(cd "$(dirname "$0")" && pwd) +REPO_ROOT=$(cd "$TRAVIS_DIR/../../../../.." && pwd) + +function command_available() { + local cmd=$1 + [[ -x "$(command -v "$cmd")" ]] +} + +function sed_in_place() { + local expression=$1 + local file=$2 + + case "$(uname)" in + Darwin) sed -i '' "$expression" "$file" ;; + *) sed -i "$expression" "$file" ;; + esac +} + +function download_to_dir() { + local dir=$1 + local url=$2 + local file="$dir/$(basename "$url")" + + mkdir -p "$dir" + if command_available "curl"; then + curl -fL "$url" -o "$file" + elif command_available "wget"; then + wget -P "$dir" "$url" + else + echo "Required curl or wget but they are unavailable" + exit 1 + fi +} + +cd "$REPO_ROOT" + VERSION=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout) SERVER_DIR=hugegraph-server/apache-hugegraph-server-$VERSION/ CONF=$SERVER_DIR/conf/graphs/hugegraph.properties @@ -32,20 +67,22 @@ JACOCO_PORT=36320 mvn package -Dmaven.test.skip=true -ntp # add mysql dependency -wget -P $SERVER_DIR/lib/ https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.28/mysql-connector-java-8.0.28.jar +download_to_dir "$SERVER_DIR/lib/" \ + "https://repo1.maven.org/maven2/mysql/mysql-connector-java/8.0.28/mysql-connector-java-8.0.28.jar" if [[ ! -e "$SERVER_DIR/lib/ikanalyzer-2012_u6.jar" ]]; then - wget -P $SERVER_DIR/lib/ https://raw.githubusercontent.com/apache/hugegraph-doc/ik_binary/dist/server/ikanalyzer-2012_u6.jar + download_to_dir "$SERVER_DIR/lib/" \ + "https://raw.githubusercontent.com/apache/hugegraph-doc/ik_binary/dist/server/ikanalyzer-2012_u6.jar" fi # config rest-server -sed -i '/^#*auth\.authenticator=/d' $REST_SERVER_CONF -sed -i '/^#*auth\.admin_token=/d' $REST_SERVER_CONF +sed_in_place '/^#*auth\.authenticator=/d' "$REST_SERVER_CONF" +sed_in_place '/^#*auth\.admin_token=/d' "$REST_SERVER_CONF" echo "auth.authenticator=org.apache.hugegraph.auth.StandardAuthenticator" >> $REST_SERVER_CONF echo "auth.admin_token=pa" >> $REST_SERVER_CONF # config hugegraph.properties -sed -i 's/gremlin.graph=.*/gremlin.graph=org.apache.hugegraph.auth.HugeFactoryAuthProxy/' $CONF +sed_in_place 's/gremlin.graph=.*/gremlin.graph=org.apache.hugegraph.auth.HugeFactoryAuthProxy/' "$CONF" # config gremlin-server echo " diff --git a/hugegraph-server/hugegraph-dist/src/assembly/travis/start-server.sh b/hugegraph-server/hugegraph-dist/src/assembly/travis/start-server.sh index 7846b04512..b7b70e8cc9 100755 --- a/hugegraph-server/hugegraph-dist/src/assembly/travis/start-server.sh +++ b/hugegraph-server/hugegraph-dist/src/assembly/travis/start-server.sh @@ -17,13 +17,12 @@ # set -ev -HOME_DIR=$(pwd) -TRAVIS_DIR=$(dirname $0) +TRAVIS_DIR=$(cd "$(dirname "$0")" && pwd) BASE_DIR=$1 BACKEND=$2 JACOCO_PORT=$3 -JACOCO_DIR=${HOME_DIR}/${TRAVIS_DIR} +JACOCO_DIR=${TRAVIS_DIR} JACOCO_JAR=${JACOCO_DIR}/jacocoagent.jar BIN=$BASE_DIR/bin @@ -33,27 +32,46 @@ GREMLIN_CONF=$BASE_DIR/conf/gremlin-server.yaml . "${BIN}"/util.sh -declare -A backend_serializer_map=(["memory"]="text" \ - ["hbase"]="hbase" \ - ["rocksdb"]="binary" \ - ["hstore"]="binary") +function sed_in_place() { + local expression=$1 + local file=$2 -SERIALIZER=${backend_serializer_map[$BACKEND]} + case "$(uname)" in + Darwin) sed -i '' "$expression" "$file" ;; + *) sed -i "$expression" "$file" ;; + esac +} + +case "$BACKEND" in + memory) + SERIALIZER=text + ;; + hbase) + SERIALIZER=hbase + ;; + rocksdb|hstore) + SERIALIZER=binary + ;; + *) + echo "Unsupported backend: $BACKEND" + exit 1 + ;; +esac # Set backend and serializer -sed -i "s/backend=.*/backend=$BACKEND/" $CONF -sed -i "s/serializer=.*/serializer=$SERIALIZER/" $CONF +write_property "$CONF" "backend" "$BACKEND" +write_property "$CONF" "serializer" "$SERIALIZER" # Set timeout for hbase if [ "$BACKEND" == "hbase" ]; then - sed -i '$arestserver.request_timeout=200' $REST_CONF - sed -i '$agremlinserver.timeout=200' $REST_CONF - sed -i 's/evaluationTimeout.*/evaluationTimeout: 200000/' $GREMLIN_CONF + echo "restserver.request_timeout=200" >> $REST_CONF + echo "gremlinserver.timeout=200" >> $REST_CONF + sed_in_place 's/evaluationTimeout.*/evaluationTimeout: 200000/' "$GREMLIN_CONF" fi # Set usePD=true for hstore if [ "$BACKEND" == "hstore" ]; then - sed -i '$ausePD=true' $REST_CONF + echo "usePD=true" >> $REST_CONF fi # Append schema.sync_deletion=true to config file @@ -67,5 +85,14 @@ if [ -n "$JACOCO_PORT" ]; then JACOCO_OPTION="-javaagent:${JACOCO_JAR}=includes=*,port=${JACOCO_PORT},destfile=jacoco-it.exec,output=tcpserver" fi +SERVER_JAVA_OPTIONS="${SERVER_JAVA_OPTIONS:-}" +if [ -n "$SERVER_JAVA_OPTIONS" ]; then + if [ -n "$JACOCO_OPTION" ]; then + JACOCO_OPTION="${JACOCO_OPTION} ${SERVER_JAVA_OPTIONS}" + else + JACOCO_OPTION="${SERVER_JAVA_OPTIONS}" + fi +fi + echo -e "pa" | $BIN/init-store.sh $BIN/start-hugegraph.sh -j "$JACOCO_OPTION" -t 60