diff --git a/.gitignore b/.gitignore index 416a92f..48503cb 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,11 @@ replay_pid* # Build output target/ +build/ + +# Gradle +.gradle/ +!gradle/wrapper/gradle-wrapper.jar # IDE .idea/ diff --git a/agenteval-core/build.gradle.kts b/agenteval-core/build.gradle.kts new file mode 100644 index 0000000..5a739ee --- /dev/null +++ b/agenteval-core/build.gradle.kts @@ -0,0 +1,5 @@ +dependencies { + api(libs.jackson.databind) + api(libs.slf4j.api) + implementation(libs.jackson.dataformat.yaml) +} diff --git a/agenteval-datasets/build.gradle.kts b/agenteval-datasets/build.gradle.kts new file mode 100644 index 0000000..cac4a72 --- /dev/null +++ b/agenteval-datasets/build.gradle.kts @@ -0,0 +1,8 @@ +dependencies { + api(project(":agenteval-core")) + implementation(project(":agenteval-judge")) + implementation(libs.jackson.databind) + implementation(libs.jackson.datatype.jsr310) + implementation(libs.slf4j.api) + testImplementation(libs.mockito.core) +} diff --git a/agenteval-embeddings/build.gradle.kts b/agenteval-embeddings/build.gradle.kts new file mode 100644 index 0000000..307f586 --- /dev/null +++ b/agenteval-embeddings/build.gradle.kts @@ -0,0 +1,6 @@ +dependencies { + api(project(":agenteval-core")) + implementation(libs.jackson.databind) + implementation(libs.slf4j.api) + testImplementation(libs.mockito.core) +} diff --git a/agenteval-github-actions/build.gradle.kts b/agenteval-github-actions/build.gradle.kts new file mode 100644 index 0000000..b314a74 --- /dev/null +++ b/agenteval-github-actions/build.gradle.kts @@ -0,0 +1,17 @@ +plugins { + alias(libs.plugins.shadow) +} + +dependencies { + implementation(project(":agenteval-core")) + implementation(project(":agenteval-reporting")) + implementation(libs.jackson.databind) + implementation(libs.slf4j.api) + testImplementation(libs.mockito.core) +} + +tasks.shadowJar { + manifest { + attributes("Main-Class" to "com.agenteval.github.GitHubActionRunner") + } +} diff --git a/agenteval-gradle-plugin/build.gradle.kts b/agenteval-gradle-plugin/build.gradle.kts new file mode 100644 index 0000000..a649ae8 --- /dev/null +++ b/agenteval-gradle-plugin/build.gradle.kts @@ -0,0 +1,25 @@ +plugins { + `java-gradle-plugin` +} + +dependencies { + implementation(project(":agenteval-core")) + implementation(project(":agenteval-judge")) + implementation(project(":agenteval-metrics")) + implementation(project(":agenteval-datasets")) + implementation(project(":agenteval-reporting")) + implementation(libs.slf4j.api) +} + +gradlePlugin { + plugins { + create("agenteval") { + id = "com.agenteval.evaluate" + implementationClass = "com.agenteval.gradle.AgentEvalPlugin" + } + } +} + +tasks.processResources { + duplicatesStrategy = DuplicatesStrategy.EXCLUDE +} diff --git a/agenteval-intellij/build.gradle.kts b/agenteval-intellij/build.gradle.kts new file mode 100644 index 0000000..affec3b --- /dev/null +++ b/agenteval-intellij/build.gradle.kts @@ -0,0 +1,18 @@ +dependencies { + compileOnly(libs.jackson.databind) + testImplementation(libs.jackson.databind) +} + +sourceSets { + main { + java { + exclude( + "com/agenteval/intellij/AgentEvalIcons.java", + "com/agenteval/intellij/AgentEvalToolWindowFactory.java", + "com/agenteval/intellij/AgentEvalToolWindow.java", + "com/agenteval/intellij/ReportFileWatcher.java", + "com/agenteval/intellij/MetricGutterIconProvider.java" + ) + } + } +} diff --git a/agenteval-judge/build.gradle.kts b/agenteval-judge/build.gradle.kts new file mode 100644 index 0000000..307f586 --- /dev/null +++ b/agenteval-judge/build.gradle.kts @@ -0,0 +1,6 @@ +dependencies { + api(project(":agenteval-core")) + implementation(libs.jackson.databind) + implementation(libs.slf4j.api) + testImplementation(libs.mockito.core) +} diff --git a/agenteval-junit5/build.gradle.kts b/agenteval-junit5/build.gradle.kts new file mode 100644 index 0000000..6904e02 --- /dev/null +++ b/agenteval-junit5/build.gradle.kts @@ -0,0 +1,9 @@ +dependencies { + api(project(":agenteval-core")) + api(project(":agenteval-datasets")) + api(libs.junit.jupiter.api) + api(libs.junit.jupiter.params) + implementation(libs.slf4j.api) + testImplementation(libs.mockito.core) + testImplementation(libs.junit.platform.testkit) +} diff --git a/agenteval-langchain4j/build.gradle.kts b/agenteval-langchain4j/build.gradle.kts new file mode 100644 index 0000000..6a13e86 --- /dev/null +++ b/agenteval-langchain4j/build.gradle.kts @@ -0,0 +1,5 @@ +dependencies { + api(project(":agenteval-core")) + compileOnly(libs.langchain4j.core) + implementation(libs.slf4j.api) +} diff --git a/agenteval-langgraph4j/build.gradle.kts b/agenteval-langgraph4j/build.gradle.kts new file mode 100644 index 0000000..261277a --- /dev/null +++ b/agenteval-langgraph4j/build.gradle.kts @@ -0,0 +1,5 @@ +dependencies { + api(project(":agenteval-core")) + compileOnly(libs.langgraph4j.core) + implementation(libs.slf4j.api) +} diff --git a/agenteval-mcp/build.gradle.kts b/agenteval-mcp/build.gradle.kts new file mode 100644 index 0000000..e105371 --- /dev/null +++ b/agenteval-mcp/build.gradle.kts @@ -0,0 +1,5 @@ +dependencies { + api(project(":agenteval-core")) + implementation(libs.slf4j.api) + implementation(libs.jackson.databind) +} diff --git a/agenteval-metrics/build.gradle.kts b/agenteval-metrics/build.gradle.kts new file mode 100644 index 0000000..d37231c --- /dev/null +++ b/agenteval-metrics/build.gradle.kts @@ -0,0 +1,7 @@ +dependencies { + api(project(":agenteval-core")) + api(project(":agenteval-judge")) + implementation(libs.slf4j.api) + implementation(project(":agenteval-embeddings")) + testImplementation(libs.mockito.core) +} diff --git a/agenteval-redteam/build.gradle.kts b/agenteval-redteam/build.gradle.kts new file mode 100644 index 0000000..f777739 --- /dev/null +++ b/agenteval-redteam/build.gradle.kts @@ -0,0 +1,7 @@ +dependencies { + api(project(":agenteval-core")) + implementation(project(":agenteval-judge")) + implementation(libs.jackson.databind) + implementation(libs.slf4j.api) + testImplementation(libs.mockito.core) +} diff --git a/agenteval-reporting/build.gradle.kts b/agenteval-reporting/build.gradle.kts new file mode 100644 index 0000000..28f480e --- /dev/null +++ b/agenteval-reporting/build.gradle.kts @@ -0,0 +1,7 @@ +dependencies { + api(project(":agenteval-core")) + implementation(libs.slf4j.api) + implementation(libs.jackson.databind) + implementation(libs.jackson.datatype.jsr310) + testImplementation(libs.mockito.core) +} diff --git a/agenteval-spring-ai/build.gradle.kts b/agenteval-spring-ai/build.gradle.kts new file mode 100644 index 0000000..5186b7e --- /dev/null +++ b/agenteval-spring-ai/build.gradle.kts @@ -0,0 +1,8 @@ +dependencies { + api(project(":agenteval-core")) + compileOnly(libs.spring.ai.model) + compileOnly(libs.spring.ai.client.chat) + compileOnly(libs.spring.ai.commons) + implementation(libs.spring.boot.autoconfigure) + implementation(libs.slf4j.api) +} diff --git a/build.gradle.kts b/build.gradle.kts new file mode 100644 index 0000000..5304789 --- /dev/null +++ b/build.gradle.kts @@ -0,0 +1,56 @@ +plugins { + alias(libs.plugins.spotbugs) apply false + alias(libs.plugins.shadow) apply false +} + +subprojects { + apply(plugin = "java-library") + apply(plugin = "checkstyle") + + group = "com.agenteval" + version = "0.1.0-SNAPSHOT" + + repositories { + mavenCentral() + } + + configure { + toolchain { + languageVersion = JavaLanguageVersion.of(21) + } + } + + tasks.withType { + options.compilerArgs.addAll(listOf("-Xlint:all", "-Werror", "-parameters")) + } + + dependencies { + "implementation"(platform("org.junit:junit-bom:5.11.4")) + "implementation"(platform("com.fasterxml.jackson:jackson-bom:2.18.3")) + + "testImplementation"("org.junit.jupiter:junit-jupiter") + "testImplementation"("org.assertj:assertj-core:3.27.7") + "testRuntimeOnly"("ch.qos.logback:logback-classic:1.5.25") + } + + configure { + toolVersion = "10.21.4" + configFile = rootProject.file("checkstyle.xml") + } + + if (name != "agenteval-gradle-plugin" && name != "agenteval-intellij") { + apply(plugin = "com.github.spotbugs") + + configure { + excludeFilter = rootProject.file("spotbugs-exclude.xml") + } + + tasks.matching { it.name == "spotbugsTest" }.configureEach { + enabled = false + } + } + + tasks.withType { + useJUnitPlatform() + } +} diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml new file mode 100644 index 0000000..d28894f --- /dev/null +++ b/gradle/libs.versions.toml @@ -0,0 +1,50 @@ +[versions] +junit-bom = "5.11.4" +jackson-bom = "2.18.3" +slf4j = "2.0.16" +assertj = "3.27.7" +logback = "1.5.25" +mockito = "5.14.2" +checkstyle = "10.21.4" +spring-ai = "1.0.0" +spring-boot = "3.4.2" +langchain4j = "0.36.2" +langgraph4j = "1.0.0" +spotbugs-plugin = "6.0.27" +shadow = "8.1.1" + +[libraries] +# BOMs +junit-bom = { module = "org.junit:junit-bom", version.ref = "junit-bom" } +jackson-bom = { module = "com.fasterxml.jackson:jackson-bom", version.ref = "jackson-bom" } + +# Jackson (versions managed by BOM) +jackson-databind = { module = "com.fasterxml.jackson.core:jackson-databind" } +jackson-dataformat-yaml = { module = "com.fasterxml.jackson.dataformat:jackson-dataformat-yaml" } +jackson-datatype-jsr310 = { module = "com.fasterxml.jackson.datatype:jackson-datatype-jsr310" } + +# Logging +slf4j-api = { module = "org.slf4j:slf4j-api", version.ref = "slf4j" } +logback-classic = { module = "ch.qos.logback:logback-classic", version.ref = "logback" } + +# Testing +assertj-core = { module = "org.assertj:assertj-core", version.ref = "assertj" } +mockito-core = { module = "org.mockito:mockito-core", version.ref = "mockito" } +junit-jupiter = { module = "org.junit.jupiter:junit-jupiter" } +junit-jupiter-api = { module = "org.junit.jupiter:junit-jupiter-api" } +junit-jupiter-params = { module = "org.junit.jupiter:junit-jupiter-params" } +junit-platform-testkit = { module = "org.junit.platform:junit-platform-testkit" } + +# Spring +spring-ai-model = { module = "org.springframework.ai:spring-ai-model", version.ref = "spring-ai" } +spring-ai-client-chat = { module = "org.springframework.ai:spring-ai-client-chat", version.ref = "spring-ai" } +spring-ai-commons = { module = "org.springframework.ai:spring-ai-commons", version.ref = "spring-ai" } +spring-boot-autoconfigure = { module = "org.springframework.boot:spring-boot-autoconfigure", version.ref = "spring-boot" } + +# LangChain4j / LangGraph4j +langchain4j-core = { module = "dev.langchain4j:langchain4j-core", version.ref = "langchain4j" } +langgraph4j-core = { module = "org.bsc.langgraph4j:langgraph4j-core-jdk8", version.ref = "langgraph4j" } + +[plugins] +spotbugs = { id = "com.github.spotbugs", version.ref = "spotbugs-plugin" } +shadow = { id = "com.github.johnrengelman.shadow", version.ref = "shadow" } diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar new file mode 100644 index 0000000..b498d24 Binary files /dev/null and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties new file mode 100644 index 0000000..d6e308a --- /dev/null +++ b/gradle/wrapper/gradle-wrapper.properties @@ -0,0 +1,5 @@ +distributionBase=GRADLE_USER_HOME +distributionPath=wrapper/dists +distributionUrl=https\://services.gradle.org/distributions/gradle-8.12-bin.zip +zipStoreBase=GRADLE_USER_HOME +zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew new file mode 100755 index 0000000..17a9170 --- /dev/null +++ b/gradlew @@ -0,0 +1,176 @@ +#!/usr/bin/env sh + +############################################################################## +## +## Gradle start up script for UN*X +## +############################################################################## + +# Attempt to set APP_HOME +# Resolve links: $0 may be a link +PRG="$0" +# Need this for relative symlinks. +while [ -h "$PRG" ] ; do + ls=`ls -ld "$PRG"` + link=`expr "$ls" : '.*-> \(.*\)$'` + if expr "$link" : '/.*' > /dev/null; then + PRG="$link" + else + PRG=`dirname "$PRG"`"/$link" + fi +done +SAVED="`pwd`" +cd "`dirname \"$PRG\"`/" >/dev/null +APP_HOME="`pwd -P`" +cd "$SAVED" >/dev/null + +APP_NAME="Gradle" +APP_BASE_NAME=`basename "$0"` + +# Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +DEFAULT_JVM_OPTS="" + +# Use the maximum available, or set MAX_FD != -1 to use that value. +MAX_FD="maximum" + +warn () { + echo "$*" +} + +die () { + echo + echo "$*" + echo + exit 1 +} + +# OS specific support (must be 'true' or 'false'). +cygwin=false +msys=false +darwin=false +nonstop=false +case "`uname`" in + CYGWIN* ) + cygwin=true + ;; + Darwin* ) + darwin=true + ;; + MINGW* ) + msys=true + ;; + NONSTOP* ) + nonstop=true + ;; +esac + +CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar + +# Determine the Java command to use to start the JVM. +if [ -n "$JAVA_HOME" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + if [ ! -x "$JAVACMD" ] ; then + die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." + fi +else + JAVACMD="java" + which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. + +Please set the JAVA_HOME variable in your environment to match the +location of your Java installation." +fi + +# Increase the maximum file descriptors if we can. +if [ "$cygwin" = "false" -a "$darwin" = "false" -a "$nonstop" = "false" ] ; then + MAX_FD_LIMIT=`ulimit -H -n` + if [ $? -eq 0 ] ; then + if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then + MAX_FD="$MAX_FD_LIMIT" + fi + ulimit -n $MAX_FD + if [ $? -ne 0 ] ; then + warn "Could not set maximum file descriptor limit: $MAX_FD" + fi + else + warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" + fi +fi + +# For Darwin, add options to specify how the application appears in the dock +if $darwin; then + GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" +fi + +# For Cygwin, switch paths to Windows format before running java +if $cygwin ; then + APP_HOME=`cygpath --path --mixed "$APP_HOME"` + CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` + JAVACMD=`cygpath --unix "$JAVACMD"` + + # We build the pattern for arguments to be converted via cygpath + ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` + SEP="" + for dir in $ROOTDIRSRAW ; do + ROOTDIRS="$ROOTDIRS$SEP$dir" + SEP="|" + done + OURCYGPATTERN="(^($ROOTDIRS))" + # Add a user-defined pattern to the cygpath arguments + if [ "$GRADLE_CYGPATTERN" != "" ] ; then + OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" + fi + # Now convert the arguments - kludge to limit ourselves to /bin/sh + i=0 + for arg in "$@" ; do + CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` + CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option + + if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition + eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` + else + eval `echo args$i`="\"$arg\"" + fi + i=$((i+1)) + done + case $i in + (0) set -- ;; + (1) set -- "$args0" ;; + (2) set -- "$args0" "$args1" ;; + (3) set -- "$args0" "$args1" "$args2" ;; + (4) set -- "$args0" "$args1" "$args2" "$args3" ;; + (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; + (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; + (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; + (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; + (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; + esac +fi + +# Escape application args +save () { + for i do printf %s\\n "$i" | sed "s/'/'\\\\''/g;1s/^/'/;\$s/\$/' \\\\/" ; done + echo " " +} +APP_ARGS=$(save "$@") + +# Collect all arguments for the java command, following the shell quoting and substitution rules +if $JAVACMD --add-opens java.base/java.lang=ALL-UNNAMED -version ; then + DEFAULT_JVM_OPTS="--add-opens java.base/java.lang=ALL-UNNAMED $DEFAULT_JVM_OPTS" +fi + +eval set -- $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS "\"-Dorg.gradle.appname=$APP_BASE_NAME\"" -classpath "\"$CLASSPATH\"" org.gradle.wrapper.GradleWrapperMain "$APP_ARGS" + +# by default we should be in the correct project dir, but when run from Finder on Mac, the cwd is wrong +if [ "$(uname)" = "Darwin" ] && [ "$HOME" = "$PWD" ]; then + cd "$(dirname "$0")" +fi + +exec "$JAVACMD" "$@" diff --git a/gradlew.bat b/gradlew.bat new file mode 100644 index 0000000..e95643d --- /dev/null +++ b/gradlew.bat @@ -0,0 +1,84 @@ +@if "%DEBUG%" == "" @echo off +@rem ########################################################################## +@rem +@rem Gradle startup script for Windows +@rem +@rem ########################################################################## + +@rem Set local scope for the variables with windows NT shell +if "%OS%"=="Windows_NT" setlocal + +set DIRNAME=%~dp0 +if "%DIRNAME%" == "" set DIRNAME=. +set APP_BASE_NAME=%~n0 +set APP_HOME=%DIRNAME% + +@rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. +set DEFAULT_JVM_OPTS= + +@rem Find java.exe +if defined JAVA_HOME goto findJavaFromJavaHome + +set JAVA_EXE=java.exe +%JAVA_EXE% -version >NUL 2>&1 +if "%ERRORLEVEL%" == "0" goto init + +echo. +echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:findJavaFromJavaHome +set JAVA_HOME=%JAVA_HOME:"=% +set JAVA_EXE=%JAVA_HOME%/bin/java.exe + +if exist "%JAVA_EXE%" goto init + +echo. +echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% +echo. +echo Please set the JAVA_HOME variable in your environment to match the +echo location of your Java installation. + +goto fail + +:init +@rem Get command-line arguments, handling Windows variants + +if not "%OS%" == "Windows_NT" goto win9xME_args + +:win9xME_args +@rem Slurp the command line arguments. +set CMD_LINE_ARGS= +set _SKIP=2 + +:win9xME_args_slurp +if "x%~1" == "x" goto execute + +set CMD_LINE_ARGS=%* + +:execute +@rem Setup the command line + +set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar + +@rem Execute Gradle +"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% + +:end +@rem End local scope for the variables with windows NT shell +if "%ERRORLEVEL%"=="0" goto mainEnd + +:fail +rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of +rem the _cmd.exe /c_ return code! +if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 +exit /b 1 + +:mainEnd +if "%OS%"=="Windows_NT" endlocal + +:omega diff --git a/settings.gradle.kts b/settings.gradle.kts new file mode 100644 index 0000000..5132f33 --- /dev/null +++ b/settings.gradle.kts @@ -0,0 +1,19 @@ +rootProject.name = "agenteval" + +include( + "agenteval-core", + "agenteval-judge", + "agenteval-embeddings", + "agenteval-metrics", + "agenteval-datasets", + "agenteval-reporting", + "agenteval-junit5", + "agenteval-spring-ai", + "agenteval-langchain4j", + "agenteval-langgraph4j", + "agenteval-mcp", + "agenteval-redteam", + "agenteval-github-actions", + "agenteval-gradle-plugin", + "agenteval-intellij" +)