diff --git a/AGENTS.mk b/AGENTS.mk deleted file mode 100644 index 020f246..0000000 --- a/AGENTS.mk +++ /dev/null @@ -1,146 +0,0 @@ -# -# Copyright (C) 2026 Intel Corporation -# -# SPDX-License-Identifier: Apache-2.0 -# - -MAKEFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) -MAKEFILE_DIR := $(realpath $(dir $(MAKEFILE_PATH))) - -AGENT_PROVIDER ?= openrouter -AGENT_MODEL ?= anthropic/claude-sonnet-4.6 -AGENT_BASE_URL ?= "https://openrouter.ai/api/v1" - -# Docker configuration -DOCKER_HOSTDIR := ${MAKEFILE_DIR}/docker -DOCKER_WORKDIR := /home/user/IDEAS -DOCKER_RUN ?= docker run --rm \ - --init \ - -it \ - -v $(MAKEFILE_DIR):$(DOCKER_WORKDIR) \ - -w $(DOCKER_WORKDIR)/$(patsubst $(MAKEFILE_DIR)/%,%,$(CURDIR)) \ - -e OPENROUTER_API_KEY \ - -e TRANSLATION_DIR \ - -e AGENT_PROVIDER \ - -e AGENT_MODEL \ - -e AGENT_BASE_URL \ - -e RUSTFLAGS \ - ideas-$(shell id -u) - -ifdef DOCKER_RUN - RUN_PREFIX = \ - mkdir -p $(DOCKER_HOSTDIR)/.venv && \ - $(DOCKER_RUN) \ - /bin/sh -c 'set -e; - RUN_SUFFIX = ' -else - RUN_PREFIX = - RUN_SUFFIX = -endif - -TARGETS_LIB ?= $(shell [ -d build-ninja ] && find build-ninja -maxdepth 1 -type f -executable -exec basename {} \; | cut -d. -f1 | grep -E "^lib" | sed -e "s/^lib//gi") -TARGETS_BIN ?= $(shell [ -d build-ninja ] && find build-ninja -maxdepth 1 -type f -executable -exec basename {} \; | cut -d. -f1 | grep -vE "^lib") -TARGETS ?= $(TARGETS_BIN) $(TARGETS_LIB) -ifeq (${TARGETS},) -ifeq ($(filter cmake clean,$(MAKECMDGOALS)),) -$(error No TARGETS found! You need to run cmake!) -endif -endif - - -.PRECIOUS: test_crates/%/Cargo.toml -.PRECIOUS: test_crates/%/src/lib.c -.PRECIOUS: test_crates/%/src/main.c -.PRECIOUS: test_crates/%/build.rs - -test_crates/%/Cargo.toml \ -test_crates/%/src/lib.c \ -test_crates/%/build.rs: | build-ninja/lib%.so.sources - uv run python -m ideas.init.crate cargo_toml=test_crates/$*/Cargo.toml \ - template=lib \ - reexport_lib=false \ - hydra.output_subdir=null \ - hydra.run.dir=test_crates/$* - uv run python -m ideas.init.consolidate compile_commands=build-ninja/compile_commands.json \ - cargo_toml=test_crates/$*/Cargo.toml \ - source_priority=build-ninja/lib$*.so.sources \ - hydra.output_subdir=null \ - hydra.run.dir=test_crates/$* - uv run python -m ideas.agents.build instrumentation=coverage \ - hydra.output_subdir=null \ - hydra.job.name=init.build \ - hydra.run.dir=test_crates/$* - -test_crates/%/Cargo.toml \ -test_crates/%/src/main.c \ -test_crates/%/build.rs: | build-ninja/%.sources - uv run python -m ideas.init.crate cargo_toml=test_crates/$*/Cargo.toml \ - template=bin \ - hydra.output_subdir=null \ - hydra.run.dir=test_crates/$* - uv run python -m ideas.init.consolidate compile_commands=build-ninja/compile_commands.json \ - cargo_toml=test_crates/$*/Cargo.toml \ - source_priority=build-ninja/$*.sources \ - hydra.output_subdir=null \ - hydra.run.dir=test_crates/$* - uv run python -m ideas.agents.build instrumentation=coverage \ - hydra.output_subdir=null \ - hydra.job.name=init.build \ - hydra.run.dir=test_crates/$* - - -.PHONY: testgen_agent -testgen_agent: $(patsubst %,test_crates/%/tests/test_assert.rs,${TARGETS}) ; - -.PRECIOUS: test_crates/%/tests/test_assert.rs -test_crates/%/tests/test_assert.rs: test_crates/%/Cargo.toml test_crates/%/src/lib.c | build-ninja/lib%.so.sources - $(RUN_PREFIX) \ - uv run python -m ideas.agents.testgen model=$(if $(AGENT_PROVIDER),${AGENT_PROVIDER}/,)${AGENT_MODEL} \ - cargo_toml=test_crates/$*/Cargo.toml \ - c_code=test_crates/$*/src/lib.c \ - project_name=$* \ - test_crate_out=test_crates/$* \ - hydra.output_subdir=null \ - hydra.job.name=testgen \ - hydra.run.dir=test_crates/$* \ - $(RUN_SUFFIX) - $(RUN_PREFIX) \ - uv run python -m ideas.agents.testgen model=$(if $(AGENT_PROVIDER),${AGENT_PROVIDER}/,)${AGENT_MODEL} \ - guarantee_assert_tests=true \ - collect_to_assert=true \ - cargo_toml=test_crates/$*/Cargo.toml \ - c_code=test_crates/$*/src/lib.c \ - project_name=$* \ - test_crate_out=test_crates/$* \ - hydra.output_subdir=null \ - hydra.job.name=assert_writer \ - hydra.run.dir=test_crates/$* \ - $(RUN_SUFFIX) - # Agents are not guaranteed to produce the file - [ -f test_crates/$*/tests/test_assert.rs ] || { echo "ERROR: Agent failed to generate test_crates/$*/tests/test_assert.rs"; exit 1; } - -test_crates/%/tests/test_assert.rs: test_crates/%/Cargo.toml test_crates/%/src/main.c | build-ninja/%.sources - $(RUN_PREFIX) \ - uv run python -m ideas.agents.testgen_bin model=$(if $(AGENT_PROVIDER),${AGENT_PROVIDER}/,)${AGENT_MODEL} \ - cargo_toml=test_crates/$*/Cargo.toml \ - c_code=test_crates/$*/src/main.c \ - project_name=$* \ - test_crate_out=test_crates/$* \ - hydra.output_subdir=null \ - hydra.job.name=testgen \ - hydra.run.dir=test_crates/$* \ - $(RUN_SUFFIX) - $(RUN_PREFIX) \ - uv run python -m ideas.agents.testgen_bin model=$(if $(AGENT_PROVIDER),${AGENT_PROVIDER}/,)${AGENT_MODEL} \ - guarantee_assert_tests=true \ - collect_to_assert=true \ - cargo_toml=test_crates/$*/Cargo.toml \ - c_code=test_crates/$*/src/main.c \ - project_name=$* \ - test_crate_out=test_crates/$* \ - hydra.output_subdir=null \ - hydra.job.name=assert_writer \ - hydra.run.dir=test_crates/$* \ - $(RUN_SUFFIX) - # Agents are not guaranteed to produce the file - [ -f test_crates/$*/tests/test_assert.rs ] || { echo "ERROR: Agent failed to generate test_crates/$*/tests/test_assert.rs"; exit 1; } diff --git a/IDEAS.mk b/IDEAS.mk index 0f1d5e7..c407314 100644 --- a/IDEAS.mk +++ b/IDEAS.mk @@ -4,230 +4,252 @@ # SPDX-License-Identifier: Apache-2.0 # -MAKEFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) -MAKEFILE_DIR := $(realpath $(dir $(MAKEFILE_PATH))) -EXTRACT_INFO_CMAKE := ${MAKEFILE_DIR}/extract_info.cmake -AGENTS_MAKEFILE := $(MAKEFILE_DIR)/AGENTS.mk - -PROVIDER ?= hosted_vllm -MODEL ?= Qwen/Qwen3.5-397B-A17B -HOST ?= localhost -PORT ?= 8000 -BASE_URL ?= http://${HOST}:${PORT}/v1 -TRANSLATION_DIR ?= translation.$(shell git --git-dir=${MAKEFILE_DIR}/.git rev-parse HEAD) -ifeq (${PROVIDER},hosted_vllm) -override TRANSLATE_ARGS += model.base_url=${BASE_URL} -endif -RUSTFLAGS ?= -Awarnings## Ignore Rust compiler warnings -CARGO_NET_OFFLINE ?= true## Cargo offline mode -CFLAGS ?= -w## Ignore C compiler warnings -export EXTRACT_INFO_CMAKE CFLAGS - -VCS ?= git -GIT_AUTHOR_NAME ?= ideas -GIT_AUTHOR_EMAIL ?= ideas@localhost -export GIT_AUTHOR_NAME GIT_AUTHOR_EMAIL - -TRANSLATION_TEST ?= smoke -EVALUATION_TEST ?= test_cases +.NOTPARALLEL: +MAKEFILE_DIR := $(realpath $(dir $(abspath $(lastword $(MAKEFILE_LIST))))) +include ${MAKEFILE_DIR}/VARIABLES.mk + TEST_FILES := $(wildcard test_vectors/*.json) -TARGETS_LIB ?= $(shell [ -d build-ninja ] && find build-ninja -maxdepth 1 -name 'lib*.so.sources' -exec basename {} .so.sources \; | sed -e "s/^lib//gi") -TARGETS_BIN ?= $(shell [ -d build-ninja ] && find build-ninja -maxdepth 1 -name '*.sources' ! -name 'lib*.so.sources' -exec basename {} .sources \; ) -TARGETS ?= $(TARGETS_BIN) $(TARGETS_LIB) +GIT := @git -C ${TRANSLATION_DIR} +CARGO := cargo -q +PYTHON := uv run python + ifeq (${TARGETS},) -ifeq ($(filter cmake clean,$(MAKECMDGOALS)),) -$(error No TARGETS found! You need to run cmake!) +ifeq ($(filter bear clean,$(MAKECMDGOALS)),) +$(error No TARGETS found! You need to run bear!) endif endif +# TRANSLATION_TEST=null (or empty) disables translation-time testing: no test file is +# required from the -sys crate and RecurrentTranslator only wraps global symbols. Empty +# is normalized to null because hydra parses a bare `tests=` as "" rather than None. +ifeq ($(filter-out null,$(strip ${TRANSLATION_TEST})),) +override TRANSLATION_TEST := null +TRANSLATION_TEST_DEP := +else +# Recursive `=` so `%` stays literal until the pattern rule substitutes the stem +TRANSLATION_TEST_DEP = ${TRANSLATION_DIR}/%-sys/tests/${TRANSLATION_TEST}.rs +endif + +# bear +.PHONY: bear +bear: build-ninja/events.jsonl + @[ -s build-ninja/events.jsonl ] || (echo "BROKEN ${CURDIR}/${TRANSLATION_DIR}") + @: # empty rule to suppress "Nothing to be done for" + +ifneq ($(wildcard CMakePresets.json),) +build-ninja/events.jsonl: CMakeLists.txt CMakePresets.json + rm -rf $(@D) + cmake -S $($@,${WORKSPACE_CARGO_TOML}) + ${GIT} add $(@F) + ${GIT} commit -qm "Created cargo workspace" + @echo "" -build-ninja/CMakeCache.txt: build-ninja/cmake.log -build-ninja/compile_commands.json: build-ninja/cmake.log -build-ninja/build.log: build-ninja/cmake.log # init .PHONY: init -init: $(patsubst %,${TRANSLATION_DIR}/%/init,${TARGETS}) ; -${TRANSLATION_DIR}/%/init: ${TRANSLATION_DIR}/%/build.rs - touch ${TRANSLATION_DIR}/$*/Cargo.toml - touch ${TRANSLATION_DIR}/$*/build.rs +.PRECIOUS: ${TRANSLATION_DIR}/%-sys/Cargo.toml ${TRANSLATION_DIR}/%-sys/src/lib.c +init: $(patsubst %,${TRANSLATION_DIR}/%-sys/Cargo.toml,${TARGETS}) + touch $(patsubst %,${TRANSLATION_DIR}/%-sys/Cargo.toml,${TARGETS}) \ + $(patsubst %,${TRANSLATION_DIR}/%-sys/src/lib.c,${TARGETS}) +${TRANSLATION_DIR}/%-sys/src/lib.c \ +${TRANSLATION_DIR}/%-sys/Cargo.toml &: | ${TRANSLATION_DIR}/Cargo.toml build-ninja/%.so.d/compile_commands.json + ${PYTHON} -m ideas.consolidate cargo_toml=${TRANSLATION_DIR}/$*-sys/Cargo.toml \ + template=lib \ + compile_commands=build-ninja/$*.so.d/compile_commands.json \ + links=build-ninja/$*.so.d/links.json \ + vcs=${VCS} \ + hydra.output_subdir=.consolidate \ + hydra.job.name=consolidate \ + hydra.run.dir=${TRANSLATION_DIR}/$*-sys + @touch ${TRANSLATION_DIR}/$*-sys/Cargo.toml ${TRANSLATION_DIR}/$*-sys/src/lib.c + @echo "" +${TRANSLATION_DIR}/%-sys/src/lib.c \ +${TRANSLATION_DIR}/%-sys/Cargo.toml &: | ${TRANSLATION_DIR}/Cargo.toml build-ninja/%.d/compile_commands.json + ${PYTHON} -m ideas.consolidate cargo_toml=${TRANSLATION_DIR}/$*-sys/Cargo.toml \ + compile_commands=build-ninja/$*.d/compile_commands.json \ + links=build-ninja/$*.d/links.json \ + vcs=${VCS} \ + hydra.output_subdir=.consolidate \ + hydra.job.name=consolidate \ + hydra.run.dir=${TRANSLATION_DIR}/$*-sys + @touch ${TRANSLATION_DIR}/$*-sys/Cargo.toml ${TRANSLATION_DIR}/$*-sys/src/lib.c + @echo "" -# initialize workspace -.PRECIOUS: ${TRANSLATION_DIR}/Cargo.toml -${TRANSLATION_DIR}/Cargo.toml: - @mkdir -p ${TRANSLATION_DIR} - uv run python -m ideas.init.workspace cargo_toml=$@ vcs=${VCS} - -# initialize translated crate for each C target -# consolidate each C target -# generate build scripts -.PRECIOUS: ${TRANSLATION_DIR}/%/Cargo.toml -.PRECIOUS: ${TRANSLATION_DIR}/%/src/lib.c -.PRECIOUS: ${TRANSLATION_DIR}/%/src/main.c -.PRECIOUS: ${TRANSLATION_DIR}/%/build.rs - -${TRANSLATION_DIR}/%/Cargo.toml \ -${TRANSLATION_DIR}/%/src/lib.c \ -${TRANSLATION_DIR}/%/build.rs: | ${TRANSLATION_DIR}/Cargo.toml build-ninja/compile_commands.json build-ninja/lib%.so.sources - uv run python -m ideas.init.crate cargo_toml=${TRANSLATION_DIR}/$*/Cargo.toml \ - template=lib \ - vcs=${VCS} \ - hydra.output_subdir=.init.crate \ - hydra.run.dir=${TRANSLATION_DIR}/$* - uv run python -m ideas.init.consolidate compile_commands=build-ninja/compile_commands.json \ - vcs=${VCS} \ - cargo_toml=${TRANSLATION_DIR}/$*/Cargo.toml \ - source_priority=build-ninja/lib$*.so.sources \ - hydra.output_subdir=.init.consolidate \ - hydra.run.dir=${TRANSLATION_DIR}/$* - uv run python -m ideas.init.build cargo_toml=${TRANSLATION_DIR}/$*/Cargo.toml \ - vcs=${VCS} \ - hydra.output_subdir=.init.build \ - hydra.job.name=init.build \ - hydra.run.dir=${TRANSLATION_DIR}/$* - -${TRANSLATION_DIR}/%/Cargo.toml \ -${TRANSLATION_DIR}/%/src/main.c \ -${TRANSLATION_DIR}/%/build.rs: | ${TRANSLATION_DIR}/Cargo.toml build-ninja/compile_commands.json build-ninja/%.sources - uv run python -m ideas.init.crate cargo_toml=${TRANSLATION_DIR}/$*/Cargo.toml \ - template=bin \ - vcs=${VCS} \ - hydra.output_subdir=.init.crate \ - hydra.run.dir=${TRANSLATION_DIR}/$* - uv run python -m ideas.init.consolidate compile_commands=build-ninja/compile_commands.json \ - vcs=${VCS} \ - cargo_toml=${TRANSLATION_DIR}/$*/Cargo.toml \ - source_priority=build-ninja/$*.sources \ - hydra.output_subdir=.init.consolidate \ - hydra.run.dir=${TRANSLATION_DIR}/$* - uv run python -m ideas.init.build cargo_toml=${TRANSLATION_DIR}/$*/Cargo.toml \ - vcs=${VCS} \ - hydra.output_subdir=.init.build \ - hydra.job.name=init.build \ - hydra.run.dir=${TRANSLATION_DIR}/$* # translate .PHONY: translate -translate: $(patsubst %,${TRANSLATION_DIR}/%/translate,${TARGETS}) ; -${TRANSLATION_DIR}/%/translate: ${TRANSLATION_DIR}/%/src/lib.rs | build-ninja/lib%.so.sources ; -${TRANSLATION_DIR}/%/translate: ${TRANSLATION_DIR}/%/src/main.rs | build-ninja/%.sources ; - -.PRECIOUS: ${TRANSLATION_DIR}/%/src/lib.rs -${TRANSLATION_DIR}/%/src/lib.rs: ${TRANSLATION_DIR}/%/src/lib.c | ${TRANSLATION_DIR}/%/Cargo.toml ${TRANSLATION_DIR}/%/tests/${TRANSLATION_TEST}.rs - -uv run python -m ideas.translate model.name=${PROVIDER}/${MODEL} \ - filename=${TRANSLATION_DIR}/$*/src/lib.c \ - cargo_toml=${TRANSLATION_DIR}/$*/Cargo.toml \ - tests=${TRANSLATION_TEST} \ +.PRECIOUS: ${TRANSLATION_DIR}/%/Cargo.toml ${TRANSLATION_DIR}/%/src/lib.rs ${TRANSLATION_DIR}/%/src/main.rs +translate: $(patsubst %,${TRANSLATION_DIR}/%/Cargo.toml,${TARGETS}) + touch $(patsubst %,${TRANSLATION_DIR}/%/Cargo.toml,${TARGETS}) \ + $(patsubst %,${TRANSLATION_DIR}/%/src/lib.rs,${TARGETS_LIB}) $(patsubst %,${TRANSLATION_DIR}/%/src/main.rs,${TARGETS_BIN}) +${TRANSLATION_DIR}/%/src/lib.rs \ +${TRANSLATION_DIR}/%/Cargo.toml &: ${TRANSLATION_DIR}/%-sys/src/lib.c | ${TRANSLATION_DIR}/%-sys/Cargo.toml ${TRANSLATION_TEST_DEP} build-ninja/%.so.d/compile_commands.json + -${PYTHON} -m ideas.translate cargo_toml=${TRANSLATION_DIR}/$*/Cargo.toml \ + bindings_cargo_toml=${TRANSLATION_DIR}/$*-sys/Cargo.toml \ + template=lib \ vcs=${VCS} \ + tests=${TRANSLATION_TEST} \ + 'deps=[libc,openssl,flate2,regex]' \ + model.name=${PROVIDER}/${MODEL} \ hydra.output_subdir=.translate \ hydra.job.name=translate \ hydra.run.dir=${TRANSLATION_DIR}/$* ${TRANSLATE_ARGS} - @touch ${TRANSLATION_DIR}/$*/build.rs - @touch $@ - -.PRECIOUS: ${TRANSLATION_DIR}/%/src/main.rs -${TRANSLATION_DIR}/%/src/main.rs: ${TRANSLATION_DIR}/%/src/main.c | ${TRANSLATION_DIR}/%/Cargo.toml ${TRANSLATION_DIR}/%/tests/${TRANSLATION_TEST}.rs - -uv run python -m ideas.translate model.name=${PROVIDER}/${MODEL} \ - filename=${TRANSLATION_DIR}/$*/src/main.c \ - cargo_toml=${TRANSLATION_DIR}/$*/Cargo.toml \ - tests=${TRANSLATION_TEST} \ + @touch ${TRANSLATION_DIR}/$*/Cargo.toml ${TRANSLATION_DIR}/$*/src/lib.rs + @echo "" +${TRANSLATION_DIR}/%/src/main.rs \ +${TRANSLATION_DIR}/%/Cargo.toml &: ${TRANSLATION_DIR}/%-sys/src/lib.c | ${TRANSLATION_DIR}/%-sys/Cargo.toml ${TRANSLATION_TEST_DEP} build-ninja/%.d/compile_commands.json + -${PYTHON} -m ideas.translate cargo_toml=${TRANSLATION_DIR}/$*/Cargo.toml \ + bindings_cargo_toml=${TRANSLATION_DIR}/$*-sys/Cargo.toml \ vcs=${VCS} \ + tests=${TRANSLATION_TEST} \ + 'deps=[libc,openssl,flate2,regex]' \ + model.name=${PROVIDER}/${MODEL} \ hydra.output_subdir=.translate \ hydra.job.name=translate \ hydra.run.dir=${TRANSLATION_DIR}/$* ${TRANSLATE_ARGS} - @touch ${TRANSLATION_DIR}/$*/build.rs - @touch $@ + @touch ${TRANSLATION_DIR}/$*/Cargo.toml ${TRANSLATION_DIR}/$*/src/main.rs + @echo "" + + +# cost +.PHONY: cost +cost: ${TRANSLATION_DIR}/cost.tsv +ifneq (${VERBOSE},0) + @echo "# ${CURDIR}/${TRANSLATION_DIR}" + @cat $^ | tr -d '$$,' | datamash -g1 sum 3 sum 4 sum 5 sum 6 | awk '{printf "%28s $$%10.4f %12\047d tok (%12\047d in / %12\047d out)\n",$$1,$$2,$$3,$$4,$$5}' + @echo "" +else + @: +endif + +${TRANSLATION_DIR}/cost.tsv: $(patsubst %,${TRANSLATION_DIR}/%/cost.tsv,${TARGETS}) + @cat $^ | sort -k1 > $@ + +${TRANSLATION_DIR}/%/cost.tsv: ${TRANSLATION_DIR}/%/translate.log + @cat $^ | awk 'match($$0,/^\[[^]]+\]\[([^]]+)\]\[[^]]+\].*`([^`]+)`[^$$]*([$$][0-9.]+), ([0-9,]+) tok \(([0-9,]+) in \/ ([0-9,]+) out\)/,m){printf "%s\t%s\t%s\t%s\t%s\t%s\n",m[1],m[2],m[3],m[4],m[5],m[6]}' | sort -k1,2 > $@ + # build .PHONY: build -build: ${TRANSLATION_DIR}/build.log ; - -.PRECIOUS: ${TRANSLATION_DIR}/build.log +.PRECIOUS: ${TRANSLATION_DIR}/build.log ${TRANSLATION_DIR}/%/build.log +build: ${TRANSLATION_DIR}/build.log + @: ${TRANSLATION_DIR}/build.log: $(patsubst %,${TRANSLATION_DIR}/%/build.log,${TARGETS}) ; - cat $^ > $@ - -.PRECIOUS: ${TRANSLATION_DIR}/%/build.log -${TRANSLATION_DIR}/%/build.log: ${TRANSLATION_DIR}/%/src/lib.rs - -export RUSTFLAGS=${RUSTFLAGS} && cargo build --quiet --manifest-path ${TRANSLATION_DIR}/$*/Cargo.toml 2> ${TRANSLATION_DIR}/$*/build.log + @cat $^ > $@ +${TRANSLATION_DIR}/%/build.log: | ${TRANSLATION_DIR}/%/Cargo.toml + -export RUSTFLAGS=${RUSTFLAGS} && cd ${TRANSLATION_DIR} && ${CARGO} build -p $* 2> $*/build.log @cat ${TRANSLATION_DIR}/$*/build.log -${TRANSLATION_DIR}/%/build.log: ${TRANSLATION_DIR}/%/src/main.rs - -export RUSTFLAGS=${RUSTFLAGS} && cargo build --quiet --manifest-path ${TRANSLATION_DIR}/$*/Cargo.toml 2> ${TRANSLATION_DIR}/$*/build.log - @cat ${TRANSLATION_DIR}/$*/build.log # test .PHONY: test -test: ${TRANSLATION_DIR}/cargo_${EVALUATION_TEST}.log ; - -.PRECIOUS: ${TRANSLATION_DIR}/cargo_${EVALUATION_TEST}.log -${TRANSLATION_DIR}/cargo_${EVALUATION_TEST}.log: ${TRANSLATION_DIR}/build.log $(patsubst %,${TRANSLATION_DIR}/%/cargo_${EVALUATION_TEST}.log,${TARGETS}) - cat $(filter-out $<,$^) > $@ - -.PRECIOUS: ${TRANSLATION_DIR}/%/cargo_${EVALUATION_TEST}.log -${TRANSLATION_DIR}/%/cargo_${EVALUATION_TEST}.log: ${TRANSLATION_DIR}/%/build.log ${TRANSLATION_DIR}/%/tests/${EVALUATION_TEST}.rs | ${TRANSLATION_DIR}/%/Cargo.toml - uv run python -m ideas.evaluate manifest=${TRANSLATION_DIR}/$*/Cargo.toml \ - test_cases=${EVALUATION_TEST} \ - output_file=$@ +.PRECIOUS: ${TRANSLATION_DIR}/cargo_${EVALUATION_TEST}.log ${TRANSLATION_DIR}/%/cargo_${EVALUATION_TEST}.log +.PRECIOUS: ${TRANSLATION_DIR}/cargo_${EVALUATION_TEST}.jsonl ${TRANSLATION_DIR}/%/cargo_${EVALUATION_TEST}.jsonl +test: ${TRANSLATION_DIR}/cargo_${EVALUATION_TEST}.log ${TRANSLATION_DIR}/cargo_${EVALUATION_TEST}.jsonl ${TRANSLATION_DIR}/build.log + @: +${TRANSLATION_DIR}/cargo_${EVALUATION_TEST}.log: $(patsubst %,${TRANSLATION_DIR}/%/cargo_${EVALUATION_TEST}.log,${TARGETS}) + @cat $^ > $@ +${TRANSLATION_DIR}/cargo_${EVALUATION_TEST}.jsonl: $(patsubst %,${TRANSLATION_DIR}/%/cargo_${EVALUATION_TEST}.jsonl,${TARGETS}) + @cat $^ > $@ +${TRANSLATION_DIR}/%/cargo_${EVALUATION_TEST}.log \ +${TRANSLATION_DIR}/%/cargo_${EVALUATION_TEST}.jsonl &: ${TRANSLATION_DIR}/%/build.log ${TRANSLATION_DIR}/%/tests/${EVALUATION_TEST}.rs ${TRANSLATION_DIR}/%-sys/tests/${EVALUATION_TEST}.rs + -export RUSTFLAGS=${RUSTFLAGS} NEXTEST_EXPERIMENTAL_LIBTEST_JSON=1 && \ + cd ${TRANSLATION_DIR} && ${CARGO} nextest run -p $* --test ${EVALUATION_TEST} --message-format=libtest-json --no-fail-fast --cargo-quiet --color=always --success-output=never --failure-output=never --status-level=none --final-status-level=all --no-tests=pass > $*/cargo_${EVALUATION_TEST}.jsonl 2> $*/cargo_${EVALUATION_TEST}.log # convert cando tests -.PRECIOUS: ${TRANSLATION_DIR}/%/tests/test_cases.rs -${TRANSLATION_DIR}/%/tests/test_cases.rs: | ${TEST_FILES} ${TRANSLATION_DIR}/%/Cargo.toml runner/Cargo.toml build-ninja/lib%.so.sources - uv run python -m ideas.convert_tests runner_manifest=runner/Cargo.toml \ +# FIXME: We should just copy tests and deps from the -sys crate! +.PRECIOUS: ${TRANSLATION_DIR}/%/tests/test_cases.rs ${TRANSLATION_DIR}/%-sys/tests/test_cases.rs +${TRANSLATION_DIR}/%/tests/test_cases.rs: | ${TEST_FILES} ${TRANSLATION_DIR}/%/Cargo.toml runner/Cargo.toml build-ninja/%.so.d/compile_commands.json + ${PYTHON} -m ideas.convert_tests runner_manifest=runner/Cargo.toml \ vcs=${VCS} \ template=${MAKEFILE_DIR}/tools/rust_tests/lib_testing.rs \ output=tests/test_cases.rs \ 'test_vectors=[$(shell echo "$(TEST_FILES)" | tr ' ' ',')]' \ hydra.output_subdir=.convert_tests \ hydra.run.dir=${TRANSLATION_DIR}/$* - -${TRANSLATION_DIR}/%/tests/test_cases.rs: | ${TEST_FILES} ${TRANSLATION_DIR}/%/Cargo.toml build-ninja/%.sources - uv run python -m ideas.convert_tests vcs=${VCS} \ +${TRANSLATION_DIR}/%/tests/test_cases.rs: | ${TEST_FILES} ${TRANSLATION_DIR}/%/Cargo.toml build-ninja/%.d/compile_commands.json + ${PYTHON} -m ideas.convert_tests vcs=${VCS} \ output=tests/test_cases.rs \ 'test_vectors=[$(shell echo "$(TEST_FILES)" | tr ' ' ',')]' \ hydra.output_subdir=.convert_tests \ hydra.run.dir=${TRANSLATION_DIR}/$* +${TRANSLATION_DIR}/%-sys/tests/test_cases.rs: | ${TEST_FILES} ${TRANSLATION_DIR}/%-sys/Cargo.toml runner/Cargo.toml build-ninja/%.so.d/compile_commands.json + ${PYTHON} -m ideas.convert_tests runner_manifest=runner/Cargo.toml \ + vcs=${VCS} \ + template=${MAKEFILE_DIR}/tools/rust_tests/lib_testing.rs \ + output=tests/test_cases.rs \ + 'test_vectors=[$(shell echo "$(TEST_FILES)" | tr ' ' ',')]' \ + hydra.output_subdir=.convert_tests \ + hydra.run.dir=${TRANSLATION_DIR}/$*-sys +${TRANSLATION_DIR}/%-sys/tests/test_cases.rs: | ${TEST_FILES} ${TRANSLATION_DIR}/%-sys/Cargo.toml build-ninja/%.d/compile_commands.json + ${PYTHON} -m ideas.convert_tests vcs=${VCS} \ + output=tests/test_cases.rs \ + 'test_vectors=[$(shell echo "$(TEST_FILES)" | tr ' ' ',')]' \ + hydra.output_subdir=.convert_tests \ + hydra.run.dir=${TRANSLATION_DIR}/$*-sys + # can't rely on test vectors without explicit targets .PRECIOUS: test_vectors/%.json test_vectors/%.json: $(error $@ not found) -.PRECIOUS: test_vectors/%/%.json -test_vectors/%/%.json: - $(error $@ not found) - - -# testgen for each C target -.PRECIOUS: test_crates/%/tests/test_assert.rs -test_crates/%/tests/test_assert.rs: | build-ninja/lib%.so.sources - -@$(MAKE) -j1 -f $(AGENTS_MAKEFILE) $@ - -test_crates/%/tests/test_assert.rs: | build-ninja/%.sources - -@$(MAKE) -j1 -f $(AGENTS_MAKEFILE) $@ - -.PRECIOUS: ${TRANSLATION_DIR}/%/tests/test_assert.rs -${TRANSLATION_DIR}/%/tests/test_assert.rs: test_crates/%/tests/test_assert.rs - mkdir -p $(dir $@) - cp $< $@ - -# test wrappers instead of bindings -.PRECIOUS: ${TRANSLATION_DIR}/%/tests/test_assert_wrapper.rs -${TRANSLATION_DIR}/%/tests/test_assert_wrapper.rs: test_crates/%/tests/test_assert.rs - cat $< | sed 's/$*::binding::/$*::wrapper::/g' > $@ - -# smoke test -.PRECIOUS: ${TRANSLATION_DIR}/%/tests/smoke.rs -${TRANSLATION_DIR}/%/tests/smoke.rs: - mkdir -p $(dir $@) - printf '#[test]\nfn smoke() {\n assert_eq!(1, 1);\n}\n' > $@ +# generate I/O equivalence tests +.PHONY: testgen +.PRECIOUS: ${TRANSLATION_DIR}/%-sys/tests/io.rs +${TRANSLATION_DIR}/%-sys/tests/io.rs: +testgen: $(patsubst %,${TRANSLATION_DIR}/%-sys/tests/io.rs,${TARGETS}) + touch $(patsubst %,${TRANSLATION_DIR}/%-sys/tests/io.rs,${TARGETS}) + +${TRANSLATION_DIR}/%-sys/tests/io.rs: | ${TRANSLATION_DIR}/%-sys/Cargo.toml + ${PYTHON} -m ideas.agents.generate_io_tests model=${PROVIDER}/${MODEL} \ + manifest=${TRANSLATION_DIR}/$*-sys/Cargo.toml \ + budget=${TESTGEN_BUDGET} \ + coverage=${TESTGEN_COVERAGE} \ + hydra.output_subdir=.generate_io_tests \ + hydra.job.name=generate_io_tests \ + hydra.run.dir=${TRANSLATION_DIR}/$*-sys + + +# write smoke tests +.PRECIOUS: ${TRANSLATION_DIR}/%-sys/tests/smoke.rs +${TRANSLATION_DIR}/%-sys/tests/smoke.rs: + @mkdir -p $(@D) + @printf '#[test]\nfn smoke() {\n assert_eq!(1, 1);\n}\n' > $@ + ${GIT} add $*-sys/tests/smoke.rs + ${GIT} commit -qm "Generated smoke tests" # clean diff --git a/Makefile b/Makefile index 107a267..82bf5e4 100644 --- a/Makefile +++ b/Makefile @@ -4,32 +4,31 @@ # SPDX-License-Identifier: Apache-2.0 # -MAKEFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) -MAKEFILE_DIR := $(realpath $(dir $(MAKEFILE_PATH))) +MAKEFILE_DIR := $(realpath $(dir $(abspath $(lastword $(MAKEFILE_LIST))))) +include ${MAKEFILE_DIR}/VARIABLES.mk + +BEAR_VERSION = 4.1.5## bear version to install + +HF_TOKEN = ## Hugging Face token (optional, recommended for faster download speed) +HF_CACHE = ${HOME}/.cache/huggingface## Hugging Face cache dir on host +VLLM_NAME = local-model## Name of the vLLM container +VLLM_LAUNCH = docker run -it --rm --name ${VLLM_NAME} \ + --runtime nvidia --gpus all --ipc=host \ + -v ${HF_CACHE}:/root/.cache/huggingface \ + -e HF_TOKEN \ + -p ${PORT}:${PORT}## vLLM launch preamble +VLLM_IMAGE = vllm/vllm-openai@sha256:251eba5cc7c12fed0b75da22a9240e582b1c9e39f6fbc064f86781b963bd814f## vLLM image +VLLM_RECIPE = zai-org/GLM-5.2-FP8 \ + --revision ba978f7d347eaf65d22f1a86833408afdb953541 \ + --tensor-parallel-size 8 \ + --kv-cache-dtype fp8 \ + --reasoning-parser glm45 \ + --max-model-len auto## See https://recipes.vllm.ai/ + EXAMPLES_DIR := examples -IDEAS_MAKEFILE := $(MAKEFILE_DIR)/IDEAS.mk -AGENTS_MAKEFILE := $(MAKEFILE_DIR)/AGENTS.mk - -PROVIDER ?= openrouter## Provider to use with DSPy/LiteLLM -MODEL ?= anthropic/claude-sonnet-4.6## Model to use to translate -REVISION ?= None## Revision of model to load in vLLM -HOST ?= localhost -PORT ?= 8000## Port to use for vLLM -BASE_URL ?= http://${HOST}:${PORT}/v1## Base URL of vLLM server -VLLM_VERSION ?= 0.17.1 -VLLM_ARGS ?= --tensor-parallel-size 8 --enable-expert-parallel --max-num-seqs 16 --max-model-len 128k## Args to pass to vllm serve -TRANSLATION_DIR ?= translation.$(shell git rev-parse HEAD)## Directory to put IDEAS translation -TRANSLATE_ARGS ?= ## Args to pass to IDEAS translation -RUSTFLAGS ?= -Awarnings## Flags to build Rust translation -VERBOSE ?= 0## Whether to output failed/partial projects in summaries -VCS ?= git## Whether to use version control during translation. Options: ['git', 'none'] -CC ?= clang## C compiler to use for translation and building -EVALUATION_TEST ?= test_cases## Evaluation test directory/name to run; defaults to `test_cases` - -# Pass these variables to other Makefiles -export PROVIDER MODEL BASE_URL TRANSLATION_DIR RUSTFLAGS CC EVALUATION_TEST - -EXAMPLES ?= $(sort $(patsubst %/test_case,%,$(shell find ${EXAMPLES_DIR} -maxdepth 3 -name test_case -type d)))## List of examples to run on +ALL_EXAMPLES := $(sort $(patsubst %/test_case,%,$(shell find ${EXAMPLES_DIR} -maxdepth 3 -name test_case -type d))) +EXAMPLES ?= ${ALL_EXAMPLES}## List of examples to run on + ifeq ($(EXAMPLES),) $(warning No projects found in ${EXAMPLES_DIR}. You may need to re-run commands!) endif @@ -42,23 +41,47 @@ docker/build:## Build translation Docker image docker/build: docker/docker_build.log .PRECIOUS: docker/docker_build.log -docker/docker_build.log: docker/ideas.Dockerfile - rm -rf docker/.venv - cd docker && \ - docker build --build-arg USER_UID=$(shell id -u) \ - --build-arg USER_GID=$(shell id -g) \ - -f ideas.Dockerfile -t ideas-$(shell id -u) . && \ - docker images --quiet ideas:latest 2>&1 | tee $(notdir $@) - -.PHONY: docker -docker:## Mount translation Docker image -docker: docker/docker_build.log - mkdir -p docker/.venv - docker run -it --rm -v ".:/home/user/IDEAS" \ - -v "./docker/.venv:/home/user/IDEAS/.venv" \ - -e OPENROUTER_API_KEY \ - ideas-$(shell id -u) bash - +docker/docker_build.log: docker/ideas.Dockerfile uv.lock pyproject.toml + cp uv.lock pyproject.toml docker/ + cd docker && docker build --build-arg USER_UID=$(shell id -u) \ + --build-arg USER_GID=$(shell id -g) \ + -f ideas.Dockerfile -t ideas-$(shell id -u) . + rm docker/uv.lock docker/pyproject.toml + docker images --quiet ideas-$(shell id -u):latest > $@ + +.PHONY: examples/docker +examples/docker:## Mount all examples to the translation Docker image +examples/docker: docker/docker_build.log + mkdir -p $(foreach ex,${EXAMPLES},${MAKEFILE_DIR}/${ex}/${TRANSLATION_DIR}) + ${DOCKER_RUN} \ + --mount type=tmpfs,dst=${MAKEFILE_DIR}/examples \ + $(foreach ex,${EXAMPLES},\ + --mount type=tmpfs,dst=${MAKEFILE_DIR}/${ex} \ + --mount type=bind,src=${MAKEFILE_DIR}/${ex}/test_case,dst=${MAKEFILE_DIR}/${ex}/test_case \ + --mount type=bind,src=${MAKEFILE_DIR}/${ex}/${TRANSLATION_DIR},dst=${MAKEFILE_DIR}/${ex}/${TRANSLATION_DIR}) \ + --env TRANSLATION_DIR \ + --env "EXAMPLES=${EXAMPLES}" \ + -it ${DOCKER_IMAGE} bash + +examples/%/docker:##Mount specific example to translation Docker image +examples/%/docker: docker/docker_build.log + mkdir -p ${MAKEFILE_DIR}/$(@D)/${TRANSLATION_DIR} + ${DOCKER_RUN} \ + --mount type=tmpfs,dst=${MAKEFILE_DIR}/examples \ + --mount type=tmpfs,dst=${MAKEFILE_DIR}/$(@D) \ + --mount type=bind,src=${MAKEFILE_DIR}/$(@D)/test_case,dst=${MAKEFILE_DIR}/$(@D)/test_case \ + --mount type=bind,src=${MAKEFILE_DIR}/$(@D)/${TRANSLATION_DIR},dst=${MAKEFILE_DIR}/$(@D)/${TRANSLATION_DIR} \ + --env TRANSLATION_DIR \ + --env EXAMPLES=$(@D) \ + -it ${DOCKER_IMAGE} bash + +.PHONY: vllm/serve +vllm/serve:## Start vLLM server + ${VLLM_LAUNCH} ${VLLM_IMAGE} ${VLLM_RECIPE} + +.PHONY: vllm/kill +vllm/kill:## Gracefully stop the running vLLM server + docker stop ${VLLM_NAME} .PHONY: install install: install-uv install-rust ## Install uv and Rust @@ -87,12 +110,12 @@ install-clang:## Install Clang-21, must be sudo install-sys-deps:## Install system dependencies, must be sudo apt install libpcre3-dev libpcre2-dev -.PHONY: serve -serve:## Start vLLM server - uv run --no-project --python 3.11 --with vllm==${VLLM_VERSION} vllm serve ${MODEL} --revision ${REVISION} --host ${HOST} --port ${PORT} --dtype auto ${VLLM_ARGS} +.PHONY: install-bear +install-bear:## Install bear ${BEAR_VERSION} from source (requires Rust) + git clone --branch ${BEAR_VERSION} --depth 1 https://github.com/rizsotto/Bear /tmp/bear + cd /tmp/bear && cargo build --release && ./scripts/install.sh + rm -rf /tmp/bear -kill:## Kill all vLLM servers - pkill -f "^uv run --no-project --python 3.11 --with vllm==${VLLM_VERSION} vllm serve" -u ${USER} .PHONY: FORCE FORCE: @@ -109,97 +132,110 @@ examples/init: $(addsuffix /init,${EXAMPLES}) ; @echo "# ${TRANSLATION_DIR}" examples/%/init:## Initialize specific example examples/%/init: FORCE - -@$(MAKE) -j1 -f $(IDEAS_MAKEFILE) -C $(@D) cmake - -@$(MAKE) -j1 -f $(IDEAS_MAKEFILE) -C $(@D) init + -@$(MAKE) --no-print-directory -f $(IDEAS_MAKEFILE) -C $(@D) bear + -@$(MAKE) --no-print-directory -f $(IDEAS_MAKEFILE) -C $(@D) init -.PHONY: examples/cmake -examples/cmake:## CMake generate and build all examples -examples/cmake: $(addsuffix /cmake,${EXAMPLES}) ; - @echo "# cmake" - @echo "\`\`\`" - @find ${EXAMPLES} -path "*/build-ninja/build.log" -size 0 -exec echo SUCCEEDED \; | uniq -c - @find ${EXAMPLES} -path "*/build-ninja/build.log" -size +0 -exec echo FAILED \; | uniq -c - @echo "\`\`\`" -examples/%/cmake:## CMake generate and build specific example -examples/%/cmake: FORCE - -@$(MAKE) -j1 -f $(IDEAS_MAKEFILE) -C $(@D) cmake +.PHONY: examples/bear +examples/bear:## Use bear to intercept compile and linker commands +examples/bear: $(addsuffix /bear,${EXAMPLES}) ; +ifneq (${VERBOSE},0) + @echo "" +endif + @echo "--- Bear ---" + @find ${EXAMPLES} -maxdepth 2 -path "*/build-ninja/events.jsonl" \( -size +0 -printf 'SUCCEEDED\n' -o -printf 'FAILED\n' \) | sort -r | uniq -c +examples/%/bear:## Bear intercept and build specific example +examples/%/bear: FORCE + -@$(MAKE) --no-print-directory -f $(IDEAS_MAKEFILE) -C $(@D) bear -.PHONY: examples/testgen_agent -examples/testgen_agent:## Generate test vectors for all targets in all C examples with an agent -examples/testgen_agent: $(addsuffix /testgen_agent,${EXAMPLES}) -examples/%/testgen_agent:## Generate test vectors for all targets in a specific C example with an agent -examples/%/testgen_agent: FORCE - -@$(MAKE) -j1 -f $(IDEAS_MAKEFILE) -C $(@D) cmake - -@$(MAKE) -j1 -f $(AGENTS_MAKEFILE) -C $(@D) testgen_agent +.PHONY: examples/testgen +examples/testgen:## Generate I/O test vectors for all targets in all C examples with an agent +examples/testgen: $(addsuffix /testgen,${EXAMPLES}) +examples/%/testgen:## Generate I/O test vectors for all targets in a specific C example with an agent +examples/%/testgen: FORCE + -@$(MAKE) --no-print-directory -f $(IDEAS_MAKEFILE) -C $(@D) bear + -@$(MAKE) --no-print-directory -f $(IDEAS_MAKEFILE) -C $(@D) testgen .PHONY: examples/translate examples/translate:## Translate all examples examples/translate: $(addsuffix /translate,${EXAMPLES}) - @echo "# ${TRANSLATION_DIR}" - @echo "\`\`\`" - @echo "--- Translation Count ---" - @find ${EXAMPLES} -path "*/${TRANSLATION_DIR}/translate.log" | wc -l - @echo "\`\`\`" +ifneq (${VERBOSE},0) + @echo "" +endif + @echo "--- Translation Count for ${TRANSLATION_DIR} ---" + @find ${EXAMPLES} -maxdepth 2 -path "*/${TRANSLATION_DIR}/translate.log" | wc -l examples/%/translate:## Translate specific example examples/%/translate: FORCE - -@$(MAKE) -j1 -f $(IDEAS_MAKEFILE) -C $(@D) cmake - -@$(MAKE) -j1 -f $(IDEAS_MAKEFILE) -C $(@D) translate + -@$(MAKE) --no-print-directory -f $(IDEAS_MAKEFILE) -C $(@D) bear + -@$(MAKE) --no-print-directory -f $(IDEAS_MAKEFILE) -C $(@D) translate .PHONY: examples/build examples/build:## Build all translated examples examples/build: $(addsuffix /build,${EXAMPLES}) - @echo "# ${TRANSLATION_DIR}" - @echo "\`\`\`" - @echo "--- Project Builds ---" - @find ${EXAMPLES} -path "*/${TRANSLATION_DIR}/build.log" -size 0 -exec echo SUCCEEDED \; | uniq -c - @find ${EXAMPLES} -path "*/${TRANSLATION_DIR}/build.log" -size +0 -exec echo FAILED \; | uniq -c ifneq (${VERBOSE},0) @echo "" - @find ${EXAMPLES} -path "*/${TRANSLATION_DIR}/build.log" -size +0 | sort | sed -e "s/${TRANSLATION_DIR}.*//gi" | sed -e "s/^/ FAILED /gi" +endif + @echo "--- Project Builds for ${TRANSLATION_DIR}/build.log ---" + @find ${EXAMPLES} -maxdepth 2 -path "*/${TRANSLATION_DIR}/build.log" \( -size 0 -printf 'builds\n' -o -printf 'BROKEN\n' \) | sort -r | uniq -c +ifneq (${VERBOSE},0) @echo "" + @find ${EXAMPLES} -maxdepth 2 -path "*/${TRANSLATION_DIR}/build.log" -size +0 -printf 'BROKEN %h\n' | sed 's|/${TRANSLATION_DIR}$$||' | sort endif - @echo "\`\`\`" examples/%/build:## Build specific translated example examples/%/build: FORCE - -@$(MAKE) -j1 -f $(IDEAS_MAKEFILE) -C $(@D) cmake - -@$(MAKE) -j1 -f $(IDEAS_MAKEFILE) -C $(@D) build + -@$(MAKE) --no-print-directory -f $(IDEAS_MAKEFILE) -C $(@D) bear + -@$(MAKE) --no-print-directory -f $(IDEAS_MAKEFILE) -C $(@D) build .PHONY: examples/test examples/test:## Test all translated examples examples/test: $(addsuffix /test,${EXAMPLES}) - @echo "# ${TRANSLATION_DIR}" - @echo "\`\`\`" - @echo "--- Project Builds ---" - @find ${EXAMPLES} -path "*/${TRANSLATION_DIR}/build.log" -size 0 -exec echo SUCCEEDED \; | uniq -c - @find ${EXAMPLES} -path "*/${TRANSLATION_DIR}/build.log" -size +0 -exec echo FAILED \; | uniq -c - @echo "" ifneq (${VERBOSE},0) - @find ${EXAMPLES} -path "*/${TRANSLATION_DIR}/build.log" -size +0 | sort | sed -e "s/${TRANSLATION_DIR}.*//gi" | sed -e "s/^/ FAILED /gi" @echo "" endif - @echo "--- Project Completion Count ---" - @find ${EXAMPLES} -path '*/${TRANSLATION_DIR}/cargo_${EVALUATION_TEST}.log' -exec ./scripts/test_log_stats.sh {} \; | cut -d" " -f1 | sort | uniq -c - @echo "" + @echo "--- Project Completion Count for ${TRANSLATION_DIR}/cargo_${EVALUATION_TEST}.jsonl ---" + @find ${EXAMPLES} -maxdepth 2 -path '*/${TRANSLATION_DIR}/cargo_${EVALUATION_TEST}.jsonl' -exec ./scripts/test_log_stats.sh {} + | cut -d" " -f1 | sort | uniq -c ifneq (${VERBOSE},0) - @find ${EXAMPLES} -path '*/${TRANSLATION_DIR}/cargo_${EVALUATION_TEST}.log' -exec ./scripts/test_log_stats.sh {} \; | egrep "PARTIAL" | sort | sed -e "s/${TRANSLATION_DIR}.*//gi" | sed -e 's/^/ /' - @echo "" - @find ${EXAMPLES} -path '*/${TRANSLATION_DIR}/cargo_${EVALUATION_TEST}.log' -exec ./scripts/test_log_stats.sh {} \; | egrep "MISSING" | sort | sed -e "s/${TRANSLATION_DIR}.*//gi" | sed -e 's/^/ /' - @echo "" - @find ${EXAMPLES} -path '*/${TRANSLATION_DIR}/cargo_${EVALUATION_TEST}.log' -exec ./scripts/test_log_stats.sh {} \; | egrep "FAILED" | sort | sed -e "s/${TRANSLATION_DIR}.*//gi" | sed -e 's/^/ /' @echo "" + @find ${EXAMPLES} -maxdepth 2 -path '*/${TRANSLATION_DIR}/cargo_${EVALUATION_TEST}.jsonl' -exec ./scripts/test_log_stats.sh {} + | egrep -v "^complete" | sort | sed 's|/${TRANSLATION_DIR}/cargo_${EVALUATION_TEST}.jsonl$$||' endif - @echo "--- Aggregated Test Count ---" - @find ${EXAMPLES} -path '*/${TRANSLATION_DIR}/cargo_${EVALUATION_TEST}.log' | xargs cat | grep -aE "^test \S+ ... \S+$$" | cut -d" " -f4 | sort | uniq -c - @echo "\`\`\`" + @echo "" + @echo "--- Aggregated Test Count for ${TRANSLATION_DIR}/cargo_${EVALUATION_TEST}.jsonl ---" + @find ${EXAMPLES} -maxdepth 2 -path '*/${TRANSLATION_DIR}/cargo_${EVALUATION_TEST}.jsonl' -exec cat {} + \ + | jq -r 'select(.type=="test" and (.event=="ok" or .event=="failed")) | .event' \ + | sed 's/failed/FAILED/' | sort -r | uniq -c examples/%/test:## Test specific translated example examples/%/test: FORCE - -@$(MAKE) -j1 -f $(IDEAS_MAKEFILE) -C $(@D) cmake - -@$(MAKE) -j1 -f $(IDEAS_MAKEFILE) -C $(@D) test + -@$(MAKE) --no-print-directory -f $(IDEAS_MAKEFILE) -C $(@D) bear + -@$(MAKE) --no-print-directory -f $(IDEAS_MAKEFILE) -C $(@D) test + +.PHONY: examples/cost +examples/cost:## Print cost of all translated examples +examples/cost: $(addsuffix /cost,${EXAMPLES}) + @echo "--- Aggregated Cost for ${TRANSLATION_DIR} ---" + @find ${EXAMPLES} -maxdepth 2 -path "*/${TRANSLATION_DIR}/cost.tsv" -exec cat {} + | sort -k1 | tr -d '$$,' | datamash -g1 sum 3 sum 4 sum 5 sum 6 | awk '{printf "%28s $$%10.4f %12\047d tok ( %12\047d in / %12\047d out)\n",$$1,$$2,$$3,$$4,$$5}' +examples/%/cost:## Print cost for specific translated example +examples/%/cost: FORCE + -@$(MAKE) --no-print-directory -f $(IDEAS_MAKEFILE) -C $(@D) bear + -@$(MAKE) --no-print-directory -f $(IDEAS_MAKEFILE) -C $(@D) cost + +.PHONY: examples/stats +examples/stats:## Print translation stats for all examples +examples/stats: + -@$(MAKE) --no-print-directory examples/build + @echo "" + -@$(MAKE) --no-print-directory examples/test + @echo "" + -@$(MAKE) --no-print-directory examples/cost VERBOSE=0 + +examples/%/stats:##Print translation stats for specific example + -@$(MAKE) --no-print-directory -f $(IDEAS_MAKEFILE) -C $(@D) build VERBOSE=1 + -@$(MAKE) --no-print-directory -f $(IDEAS_MAKEFILE) -C $(@D) test VERBOSE=1 + -@$(MAKE) --no-print-directory -f $(IDEAS_MAKEFILE) -C $(@D) cost VERBOSE=1 + .PHONY: examples/clean @@ -207,7 +243,7 @@ examples/clean:## Clean all examples examples/clean: $(addsuffix /clean,${EXAMPLES}) examples/%/clean:## Clean specific example examples/%/clean: FORCE - -@$(MAKE) -j1 -f $(IDEAS_MAKEFILE) -C $(@D) clean + -@$(MAKE) --no-print-directory -f $(IDEAS_MAKEFILE) -C $(@D) clean # Global clean clean: @@ -219,20 +255,21 @@ clean: RESET := \033[0;0m CYAN_COL := \033[0;36m YELLOW_COL:= \033[0;33m -GREY_COL := \033[1;30m +GREY_COL := \033[1;32m help: @echo "Usage:" @echo " make ${CYAN_COL}[target] ${YELLOW_COL}[variables]${RESET}" @echo "" @echo "Targets:" - @grep -E "^[a-zA-Z/_%%]+:.*?##.*$$" ${MAKEFILE_LIST} \ + @grep -hE "^[a-zA-Z/_%%]+:.*?##.*$$" ${MAKEFILE_LIST} \ | awk 'BEGIN { FS=":.*##" } ; \ { printf " ${CYAN_COL}%-30s${RESET}%s\n", $$1, $$2 }' @echo "" @echo "Variables:" - @grep -E "^[a-zA-Z_]+ [:?!+]?=.*?##.*$$" ${MAKEFILE_LIST} \ + @grep -hE "^[a-zA-Z_]+ [:?!+]?=.*?##.*$$" ${MAKEFILE_LIST} \ | awk 'BEGIN { FS=" [:?!+]?= |##" } ; \ - { printf " ${YELLOW_COL}%-30s${RESET}%s ${GREY_COL}(default: %s)${RESET}\n", $$1, $$3, $$2}' + { printf " ${YELLOW_COL}%-30s${RESET}%s ${GREY_COL}(default: %s)${RESET}\n", $$1, $$3, $$2}' \ + | sort @echo "" @echo "Example:" - @echo " make examples/test TRANSLATION_DIR=test_case ${GREY_COL}# Translate, build, and run tests on C examples ${RESET}" + @echo " make examples/test TRANSLATION_DIR=my_translation ${GREY_COL}# Translate, build, and run tests on C examples ${RESET}" diff --git a/README.md b/README.md index 727c889..b6e3c43 100644 --- a/README.md +++ b/README.md @@ -6,31 +6,45 @@ > IDEAS is a framework under active development which may go through major changes with each release. > If you encounter any issues or have questions about how to run the framework, please do not hesitate to [open a GitHub issue](https://github.com/IntelLabs/IDEAS/issues/new). +## Controlling translation costs on large-scale projects +> [!NOTE] +> The default C FFI wrapper generation context does not include already-generated C FFI wrappers. +> This can reduce the performance of the translator on projects with, e.g., function pointers, where a C FFI wrapper _must_ call others (without major logic duplication). + +> To enable full C FFI wrapper generation context, set the `REDUCED_CONTEXT=0` environment variable: +```bash +REDUCED_CONTEXT=0 make examples/C-project-name/translate +``` +> [!CAUTION] +> This is currently experimental and can lead to prohibitive costs and exceeding input context limits for large-scale, fragmented C projects (20k+ LoC). + + # Requirements Developed and tested on Ubuntu 24.04. -IDEAS requires a specific version of `clang` and Rust toolchains to translate C-to-Rust. -A docker image with the user-specific name `ideas-${UID}` can be built and launched in an interactive session using: +IDEAS requires specific versions of the `clang` and Rust toolchains to translate C to Rust. +A Docker image with the user-specific name `ideas-${UID}` can be built using: ```bash -make docker +make docker/build ``` We strongly recommend launching all runs in the Docker image. > [!NOTE] +> Setting the `TRANSLATION_DIR` environment is **mandatory** when mounting examples to the Docker image. > If the `OPENROUTER_API_KEY` or `OPENAI_API_KEY` environment variables are set on the host, they will be automatically passed to the interactive session. # Quickstart -To translate a single C project to a Rust workspace, ensure it uses Cmake as a build system, place it in the `examples` folder and generate an [OpenRouter API key](https://openrouter.ai/workspaces/default/keys). +To translate a single C project to a Rust workspace, ensure it uses CMake as a build system, place it in the `examples` folder, and generate an [OpenRouter API key](https://openrouter.ai/workspaces/default/keys). -Then, build and launch the official Docker image: +Then, build and mount your project and API key in an interactive Docker session: ```bash -make docker +TRANSLATION_DIR="translation.demo" OPENROUTER_API_KEY="your-key" make examples/C-project-name/docker ``` And trigger end-to-end translation: ```bash -make examples/C-project-name/translate OPENROUTER_API_KEY="your-key" +make examples/C-project-name/translate ``` # Expected C project structure @@ -53,36 +67,60 @@ IDEAS requires the [official DARPA TRACTOR folder structure](https://github.com/ See the [`examples/templates`](examples/templates/) folder for minimal examples. # Translated Rust structure -The translation tool identifies each Cmake target (library or binary), and translates it to a separate, self-contained Rust [crate](https://doc.rust-lang.org/book/ch07-01-packages-and-crates.html#packages-and-crates). -Crates are organized together in a Rust [workspace](https://doc.rust-lang.org/cargo/reference/workspaces.html) under the folder given by the `TRANSLATION_DIR` environment variable, alongside the original C `test_case` folder. +The translation tool identifies each CMake target (library or binary) and currently translates it to _three_ separate, self-contained Rust [crates](https://doc.rust-lang.org/book/ch07-01-packages-and-crates.html#packages-and-crates): + +- a `*` crate that holds C FFI compatibility wrappers. +- a `*-rs` crate holding the guaranteed-safe Rust translation. +- a `*-sys` crate that links the original C library. + +All crates are organized under a Rust [workspace](https://doc.rust-lang.org/cargo/reference/workspaces.html) in the folder given by the `TRANSLATION_DIR` environment variable, alongside the original C `test_case` folder. For example, running ```bash -make docker -make examples/templates/hello_world_lib/translate TRANSLATION_DIR="translation.demo" OPENROUTER_API_KEY="your-key" +TRANSLATION_DIR="translation.demo" OPENROUTER_API_KEY="sk-..." make examples/docker +make examples/templates/hello_world_lib/translate ``` Should produce the following translated folder structure: ``` 📂examples/templates/hello_world_lib ┣ 📂test_case - ┣ 📂test_vectors ┗ 📂translation.demo - ┣ 📂hello_world_lib + ┣ 📂libhello_world_lib # Safe Rust + C FFI-compatible wrappers ┃ ┣ 📂src - ┃ ┃ ┣ 📄lib.c # Stub C library - ┃ ┃ ┣ 📄lib.rs # Translated Rust library - ┃ ┃ ┣ 📄wrapper.rs # Wrapper module - ┃ ┃ ┗ 📂wrapper # Per-symbol C FFI compatibility wrappers for all symbols + ┃ ┃ ┣ 📄lib.rs + ┃ ┃ ┗ 📄wrap_hello_print.rs # `wrap_{name}`: C FFI compatibility wrapper ┃ ┣ 📂tests - ┃ ┃ ┗ 📄 smoke.rs # No tests by default - ┃ ┗ 📄Cargo.toml # Crate manifest - ┣ 📄build.rs # Hybrid build script + ┃ ┃ ┗ 📄smoke.rs # Always-passing tests by default + ┃ ┗ 📄Cargo.toml + ┣ 📂libhello_world_lib-rs # Guaranteed safe Rust translation + ┃ ┣ 📂src + ┃ ┃ ┗ 📄lib.rs + ┃ ┗ 📄Cargo.toml + ┣ 📂libhello_world_lib-sys # Links C/Rust along the translation trajectory + ┃ ┣ 📂src + ┃ ┃ ┣ 📄lib.c + ┃ ┃ ┗ 📄lib.rs + ┃ ┣ 📂tests + ┃ ┃ ┗ 📄smoke.rs # Always-passing tests by default + ┃ ┣ 📄build.rs # Hybrid build script + ┃ ┗ 📄Cargo.toml ┣ 🗄️cache.db # Resumable translation cache ┣ 📄Cargo.lock # Workspace lockfile ┗ 📄Cargo.toml # Workspace manifest ``` +For a library target named `hello_world_lib`, the three generated crates play the following roles: + +- **`libhello_world_lib`** — the C FFI-compatible translated crate. By default, it contains a (possibly unsafe) wrapper `wrap_{name}` for every symbol (including private ones) that wraps the safe Rust translation and restores C FFI compatibility. +- **`libhello_world_lib-rs`** — the guaranteed-safe Rust translation of the original C target. It enforces `#![forbid(unsafe_code)]` at the top of each module. +- **`libhello_world_lib-sys`** — the consolidated C code visible through the Rust C FFI. As symbols are translated, they are progressively depleted from the source file until it can be dropped entirely on complete translation. + +Binary targets have a similar expected crate structure, with a `main.rs` present in all crates. + +> [!NOTE] +> If translation fails and exits early, the crates are not guaranteed to be in a valid state, but are still useful for debugging and contain exact `git` logs. + IDEAS is capable of testing Rust translations with the DARPA TRACTOR evaluation schema. See [here](https://github.com/DARPA-TRACTOR-Program/PUBLIC-Test-Corpus?tab=readme-ov-file#test-vector-schema-json) for more details and the exact specification for writing test vectors and `cando2` runners. @@ -90,11 +128,10 @@ See [here](https://github.com/DARPA-TRACTOR-Program/PUBLIC-Test-Corpus?tab=readm Our translation framework treats [OpenRouter](https://openrouter.ai/) as the default provider, allowing easy switching between models. The `MODEL` environment variable controls which LLM will be used, and should be the model's name on OpenRouter. -To run LLM-based memory-safe translation of a single project and save the translated Rust workspace in a newly created `TRANSLATION_DIR` sub-folder run: +To run LLM-based memory-safe translation of a single project and save the translated Rust workspace in a newly created `TRANSLATION_DIR` sub-folder, run: ```bash -make examples/C-project-name/translate \ - TRANSLATION_DIR="translated_rust" \ - OPENROUTER_API_KEY="your-key" \ +TRANSLATION_DIR="translation.demo" OPENROUTER_API_KEY="sk-..." make examples/C-project-name/docker +make examples/C-project-name/translate ``` If a project (library or executable) was not already found under `TRANSLATION_DIR`, our dependency chain will first trigger its memory-safe translation, followed by C FFI wrappers (only for libraries). @@ -102,45 +139,87 @@ If a project (library or executable) was not already found under `TRANSLATION_DI # Usage with Anthropic API IDEAS can be used with any Anthropic model by setting the `PROVIDER`, `MODEL`, and `ANTHROPIC_API_KEY` variables: ```bash +TRANSLATION_DIR="translation.demo" ANTHROPIC_API_KEY="sk-..." make examples/C-project-name/docker make examples/C-project-name/translate \ - TRANSLATION_DIR="translated_rust" \ - ANTHROPIC_API_KEY="your-key" \ PROVIDER="anthropic" \ MODEL="claude-sonnet-4.6" ``` Note the `anthropic` prefix is missing from `MODEL` and is instead set as the `PROVIDER`. # Usage with OpenAI API -IDEAS can be used with any OpenAI model by setting the `PROVIDER`, `MODEL`, and `OPEN_API_KEY` variables: +IDEAS can be used with any OpenAI model by setting the `PROVIDER`, `MODEL`, and `OPENAI_API_KEY` variables: ```bash +TRANSLATION_DIR="translation.demo" OPENAI_API_KEY="sk-..." make examples/C-project-name/docker make examples/C-project-name/translate \ - TRANSLATION_DIR="translated_rust" \ - OPEN_API_KEY="your-key" \ PROVIDER="openai" \ MODEL="gpt-5.4" ``` Note the `openai` prefix is missing from `MODEL` and is instead set as the `PROVIDER`. # Usage with other APIs -IDEAS relies on [`litellm`](https://github.com/BerriAI/litellm), which supports many other model providers (e.g., Google Vertex, MS Azure, etc). +IDEAS relies on [`litellm`](https://github.com/BerriAI/litellm), which supports many other model providers (e.g., Google Vertex, MS Azure, etc.). -The instructions at https://docs.litellm.ai/docs/providers inform which parameters should be set in `litellm` and IDEAS flows through the [`dspy.LM`](https://dspy.ai/api/models/LM/) instance. +The instructions at https://docs.litellm.ai/docs/providers indicate which parameters should be set in `litellm`; IDEAS passes them through the [`dspy.LM`](https://dspy.ai/api/models/LM/) instance. -Developers can inspect [how a `dspy.LM` is instantiated by IDEAS](https://github.com/IntelLabs/IDEAS/blob/main/src/ideas/model.py) and infer any additional required parameters that need to be passed to `litellm`. +Developers can inspect [how a `dspy.LM` is instantiated by IDEAS](https://github.com/IntelLabs/IDEAS/blob/main/src/ideas/model.py) and infer any additional parameters that need to be passed to `litellm`. + +# Usage with locally-hosted models +We support a single-command launch of locally-hosted models using the [official `vllm` Docker image](https://docs.vllm.ai/en/stable/deployment/docker/#pre-built-images), pinned to an exact version. + +Running +```bash +make vllm/serve +``` + +will serve GLM-5.2 on your machine using the default recipe for single-instance, 8-way GPU inference. + +Consult the [official `vllm` recipes](https://recipes.vllm.ai/) to identify a model suitable for your platform. +For example, to use the [Qwen3.6-35B-A3B](https://recipes.vllm.ai/Qwen/Qwen3.6-35B-A3B) model, set the `VLLM_RECIPE` Makefile variable to its recipe and directly serve the model: +```bash +make VLLM_RECIPE=vllm serve Qwen/Qwen3.6-35B-A3B \ + --trust-remote-code \ + --tensor-parallel-size 1 \ + --enable-auto-tool-choice \ + --tool-call-parser qwen3_xml \ + --reasoning-parser qwen3 \ + --mm-encoder-tp-mode data vllm/serve +``` + +Then, run IDEAS on host using: +```bash +TRANSLATION_DIR="translation.demo" make examples/C-project-name/translate \ + PROVIDER="hosted_vllm" \ + MODEL="Qwen/Qwen3.6-35B-A3B" +``` # (Experimental) C/Rust equivalence test generation IDEAS has a submodule that automatically generates portable Rust C FFI tests for libraries and binary targets. -To directly launch the test generation agent in an isolated enviroment run (from host): +`TRANSLATION_DIR` is overloaded to also hold the test generation results, so the same directory is used whether or not a translation was already produced in it. + +First, mount the project into the Docker image: +```bash +TRANSLATION_DIR="translation.demo" OPENROUTER_API_KEY="sk-..." make examples/templates/hello_world_lib/docker +``` + +This mounts the repository read-only and bind-mounts only `test_case` and `TRANSLATION_DIR` as writable, so the agent cannot reach the rest of the host. +Use `make examples/docker` instead to mount every project in `EXAMPLES` at once. +Then, from the interactive session, launch the test generation agent: ```bash -make examples/templates/hello_world_lib/testgen_agent OPENROUTER_API_KEY="your-key" +make examples/templates/hello_world_lib/testgen \ + TESTGEN_BUDGET="4.0" \ + TESTGEN_COVERAGE="100" ``` +Note the `anthropic` prefix is missing from `MODEL` and is instead set as the `PROVIDER`. + +This launches a self-sufficient [KISS](https://github.com/ksenxx/kiss_ai) agent that is tasked with generating unit and integration tests with high branch coverage. +You can consult the agent prompt [here](src/ideas/agents/generate_io_tests.py). -This launches a self-sufficient KISS agent that is tasked with generating unit and integration tests with high branch coverage. -You can consult the agent prompt (for libraries; similar for binaries) [here](src/ideas/agents/testgen.py). +For every target, the agent writes two test files to the `*-sys` crate: `tests/collect.rs`, which records the behavior of the C code into `json/`, and `tests/io.rs`, which asserts that recorded behavior through the C FFI. +The full agent trajectory is logged next to them as `testgen-*.log`. > [!NOTE] -> This behavior is disabled by default and may not be reliable on large codebases and/or weaker LLMs. +> This behavior is disabled by default and may not be reliable on large codebases and/or with weaker LLMs. ## Acknowledgments -This material is based upon work supported by the Defense Advanced Research Projects Agency (DARPA) under Agreement No. HR00112590134. +This material is based upon work supported by the Defense Advanced Research Projects Agency (DARPA) Translating All C To Rust (TRACTOR) program under Agreement No. HR00112590134. diff --git a/VARIABLES.mk b/VARIABLES.mk new file mode 100644 index 0000000..8825fde --- /dev/null +++ b/VARIABLES.mk @@ -0,0 +1,54 @@ +MAKEFILE_DIR := $(realpath $(dir $(abspath $(lastword $(MAKEFILE_LIST))))) +IDEAS_MAKEFILE := $(MAKEFILE_DIR)/IDEAS.mk + +PROVIDER = openai## Provider to use with DSPy/LiteLLM +MODEL = gpt-5.6-sol## Model to use to translate +HOST = localhost +PORT = 8000## Port to use for vLLM +BASE_URL = http://${HOST}:${PORT}/v1## Base URL of vLLM server +GIT_HEAD := $(shell git --git-dir=${MAKEFILE_DIR}/.git rev-parse HEAD 2>/dev/null || echo "ideas") +TRANSLATION_DIR ?= translation.${GIT_HEAD}## Directory to put IDEAS translation +CARGO_NET_OFFLINE = true## Cargo offline mode +RUSTFLAGS = -Awarnings## Flags to build Rust translation +CC = clang +CFLAGS = -w## Ignore C compiler warnings +TRANSLATION_TEST = null## Translation test directory/name to run (empty/null: wrap globals only, no tests) +EVALUATION_TEST = test_cases## Evaluation test directory/name to run +VCS = git## Whether to use version control during translation. Options: ['git', 'none'] +GIT_AUTHOR_NAME = ideas.${MODEL} +GIT_AUTHOR_EMAIL = ${MODEL}@${PROVIDER} +TRANSLATE_ARGS = ## Args to pass to IDEAS translation +TESTGEN_BUDGET = 4.0## Budget [USD] for IDEAS test generation +TESTGEN_COVERAGE = 100## Requested branch coverage [%] for IDEAS test generation +VERBOSE = 0## Whether to output failed/partial projects in summaries + +DOCKER_RUN = mkdir -p ${MAKEFILE_DIR}/docker/venv && docker run --rm \ + --init \ + --mount type=bind,src=${MAKEFILE_DIR},dst=${MAKEFILE_DIR},readonly \ + --mount type=bind,src=${MAKEFILE_DIR}/docker/venv,dst=${MAKEFILE_DIR}/.venv \ + -w ${CURDIR} \ + -e OPENROUTER_API_KEY -e ANTHROPIC_API_KEY -e OPENAI_API_KEY \ + -e RUSTFLAGS \ + -e GIT_AUTHOR_NAME -e GIT_AUTHOR_EMAIL +USER_UID := $(shell id -u) +DOCKER_IMAGE = $(if ${DOCKER_RUN},ideas-${USER_UID},) + +ifeq (${PROVIDER},hosted_vllm) +override TRANSLATE_ARGS += model.base_url=${BASE_URL} +override TRANSLATE_ARGS += generate.timeout=5400 +endif + +# Directories reserved for IDEAS +TRANSLATION_DIR_RESERVED := test_case build-ninja test_vectors runner +ifeq ($(strip ${TRANSLATION_DIR}),) +$(error TRANSLATION_DIR must not be empty) +endif +ifneq ($(filter ${TRANSLATION_DIR},${TRANSLATION_DIR_RESERVED}),) +$(error TRANSLATION_DIR='${TRANSLATION_DIR}' is reserved (${TRANSLATION_DIR_RESERVED}); pick another name) +endif + +TARGETS_LIB := $(shell [ -d build-ninja ] && find build-ninja -maxdepth 1 -type d -name '*.so.d' -printf '%f\n' | sed 's/\.so\.d$$//') +TARGETS_BIN := $(shell [ -d build-ninja ] && find build-ninja -maxdepth 1 -type d -name '*.d' ! -name '*.so.d' -printf '%f\n' | sed 's/\.d$$//') +TARGETS = ${TARGETS_LIB} ${TARGETS_BIN} + +export diff --git a/docker/ideas.Dockerfile b/docker/ideas.Dockerfile index 39bc469..3cdd2cf 100644 --- a/docker/ideas.Dockerfile +++ b/docker/ideas.Dockerfile @@ -5,7 +5,7 @@ # -FROM docker.io/rust:bookworm +FROM docker.io/rust:bookworm@sha256:77fac8b98f9f46062bb680b6d25d5bcaabfc400143952ebc572e924bcbedc3fa RUN apt-get update && apt-get install -y \ build-essential \ @@ -16,7 +16,8 @@ RUN apt-get update && apt-get install -y \ lsb-release \ software-properties-common \ gnupg \ - vim + vim \ + datamash RUN wget https://apt.llvm.org/llvm.sh && \ chmod +x llvm.sh && \ @@ -42,6 +43,8 @@ RUN wget https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cm # Symlink /usr/bin/clang and set it as default compiler RUN ln -s /usr/bin/clang-21 /usr/bin/clang ENV CC=clang +# Symlink /usr/bin/ld.lld because Bear's intercept-preload needs lld +RUN ln -s /usr/bin/ld.lld-21 /usr/bin/ld.lld # Install uv ENV UV_INSTALL_DIR="/usr/local/bin" @@ -55,9 +58,16 @@ RUN cargo install bindgen-cli --version 0.72.1 RUN cargo install cargo-llvm-cov --version 0.8.6 RUN cargo install cargo-nextest --version 0.9.114 --locked +# Install Bear (Build EAR) for capturing build commands +ARG BEAR_VERSION=4.1.5 +RUN git clone --branch ${BEAR_VERSION} --depth 1 https://github.com/rizsotto/Bear /tmp/bear +RUN cd /tmp/bear && cargo build --release && ./scripts/install.sh +RUN rm -rf /tmp/bear + # Non-root user ARG USER_UID=1000 ARG USER_GID=1000 +RUN sed -i 's/UID_MAX.*/UID_MAX 20000000/' /etc/login.defs RUN groupadd -g ${USER_GID} ideas && \ useradd -m -u ${USER_UID} -g ${USER_GID} user && \ chown -R user:ideas /home/user && \ @@ -72,9 +82,32 @@ ENV GIT_AUTHOR_EMAIL="ideas@ideas.local" ENV GIT_COMMITTER_NAME="ideas" ENV GIT_COMMITTER_EMAIL="ideas@ideas.local" -# Configure Python and uv +# Shell quality-of-life for interactive use +COPY --chown=user:ideas ideas.bashrc /home/user/.bashrc + +# Cache Python dependencies ENV PYTHONDONTWRITEBYTECODE=1 ENV UV_LINK_MODE="copy" +COPY pyproject.toml uv.lock . +RUN uv sync --frozen --no-install-project && rm -rf pyproject.toml uv.lock .venv -# Shell quality-of-life for interactive use -COPY --chown=user:ideas ideas.bashrc /home/user/.bashrc +# Cache cargo dependencies +RUN cargo init --lib cargo_deps && \ + cd cargo_deps && \ + cargo add libc@0.2.185 \ + openssl@0.10.79 \ + flate2@1 \ + regex@1 && \ + cargo add --dev \ + assert_cmd@2.0.17 \ + predicates@3.1.3 \ + once_cell@1.21.3 \ + test-cdylib@1.1.0 \ + serde_json@1 \ + tempfile@3 && \ + cargo add --dev \ + --features derive serde@1 && \ + cargo add --build \ + cc@1.2.53 && \ + CARGO_NET_OFFLINE=false cargo metadata \ + --manifest-path /home/user/IDEAS/cargo_deps/Cargo.toml diff --git a/extract_info.cmake b/extract_info.cmake deleted file mode 100644 index 43758f4..0000000 --- a/extract_info.cmake +++ /dev/null @@ -1,78 +0,0 @@ -cmake_minimum_required(VERSION 3.20) - -set(EXTRACT_INFO_TARGET_TYPES EXECUTABLE SHARED_LIBRARY) - -# https://stackoverflow.com/questions/60211516/programmatically-get-all-targets-in-a-cmake-project -function(get_all_targets _result _dir) - get_property(_subdirs DIRECTORY "${_dir}" PROPERTY SUBDIRECTORIES) - foreach(_subdir IN LISTS _subdirs) - get_all_targets(${_result} "${_subdir}") - endforeach() - - get_directory_property(_sub_targets DIRECTORY "${_dir}" BUILDSYSTEM_TARGETS) - set(${_result} ${${_result}} ${_sub_targets} PARENT_SCOPE) -endfunction() - -function(get_target_sources SOURCES TARGET) - get_target_property(TARGET_DIR ${TARGET} SOURCE_DIR) - get_target_property(TARGET_SOURCES_RAW ${TARGET} SOURCES) - set(TARGET_SOURCES "") - if(NOT TARGET_SOURCES_RAW) - message(STATUS " No sources found for ${TARGET} in ${TARGET_DIR}") - else() - foreach(TARGET_SOURCE ${TARGET_SOURCES_RAW}) - # Complex projects may contain generator expressions so we handle that here since there is no way to evaluate that expression - if(TARGET_SOURCE MATCHES "\\$]+)>") - set(SOURCE_TARGET ${CMAKE_MATCH_1}) - if(TARGET "${SOURCE_TARGET}") - get_target_sources(SOURCE_SOURCES ${SOURCE_TARGET}) - if(SOURCE_SOURCES) - list(APPEND TARGET_SOURCES ${SOURCE_SOURCES}) - endif() - endif() - else() - if(NOT IS_ABSOLUTE "${TARGET_SOURCE}") - set(TARGET_SOURCE "${TARGET_DIR}/${TARGET_SOURCE}") - endif() - list(APPEND TARGET_SOURCES ${TARGET_SOURCE}) - endif() - endforeach() - endif() - set(${SOURCES} ${TARGET_SOURCES} PARENT_SCOPE) -endfunction() - -function(extract_info) - message(STATUS "Detecting targets ...") - get_all_targets(ALL_TARGETS ${PROJECT_SOURCE_DIR}) - foreach(TARGET ${ALL_TARGETS}) - get_target_property(TARGET_TYPE ${TARGET} TYPE) - - if(NOT TARGET_TYPE IN_LIST EXTRACT_INFO_TARGET_TYPES) - message(STATUS " Skipping ${TARGET_TYPE} ${TARGET}") - continue() - endif() - - get_target_property(TARGET_LINK_LIBRARIES ${TARGET} LINK_LIBRARIES) - - message(STATUS " Found ${TARGET_TYPE} ${TARGET}") - - # Recursively get target sources and shared library/object sources - get_target_sources(TARGET_SOURCES ${TARGET}) - foreach(LINK_TARGET IN LISTS TARGET_LINK_LIBRARIES) - if(TARGET ${LINK_TARGET}) - get_target_sources(LINK_SOURCES ${LINK_TARGET}) - list(APPEND TARGET_SOURCES ${LINK_SOURCES}) - endif() - endforeach() - list(JOIN TARGET_SOURCES "\n" TARGET_SOURCES) - - if(${TARGET_TYPE} STREQUAL "EXECUTABLE") - set(TARGET_NAME $) - else() - set(TARGET_NAME $) - endif() - file(GENERATE OUTPUT "${TARGET_NAME}.sources" CONTENT "${TARGET_SOURCES}") - endforeach() -endfunction() - -cmake_language(DEFER CALL extract_info) diff --git a/pyproject.toml b/pyproject.toml index 44b8f5c..c347a54 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -9,7 +9,7 @@ requires-python = "~=3.13.0" dependencies = [ "clang==21.1.7", "dspy==3.1.2", - "kiss-agent-framework==2026.5.22", + "kiss-agent-framework[core]", "hydra-core==1.3.2", "networkx==3.6.1", "tomlkit>=0.14.0", @@ -23,6 +23,7 @@ dev = [ "pre-commit==4.2.0", "pytest==9.0.3", "ruff==0.13.0", + "vulture==2.16", ] [build-system] @@ -47,3 +48,6 @@ line-length = 96 # Ensure that the setuptools v81.0.0 is used whenever a package has a build dependency # on setuptools. build-constraint-dependencies = ["setuptools==81.0.0"] + +[tool.uv.sources] +kiss-agent-framework = { git = "https://github.com/mariusarvinte/kiss_ai", rev = "f4f6cc1ef1fc5e2962aec6da71677be6e057f194" } diff --git a/scripts/test_log_stats.sh b/scripts/test_log_stats.sh index cd6701e..f0f411e 100755 --- a/scripts/test_log_stats.sh +++ b/scripts/test_log_stats.sh @@ -1,22 +1,25 @@ #!/bin/sh -# Count number of ok/FAILED inside specified file -PASS=`grep -aE "^test \S+ ... ok" $1 | wc -l` -FAIL=`grep -aE "^test \S+ ... FAILED" $1 | wc -l` +# Try one jq for the whole batch and fallback to per-record if any log is malformed +FILTER='select(.type=="suite" and .event!="started") | "\(input_filename) \(.passed // 0) \(.failed // 0)"' +STATS=$(jq -r "$FILTER" "$@" 2>/dev/null) +if [ $? -ne 0 ]; then + STATS=$(for LOG in "$@"; do jq -r "$FILTER" "$LOG" 2>/dev/null; done) +fi -# If no PASS nor FAIL, then tests are missing -if [ $PASS -eq 0 ] && [ $FAIL -eq 0 ]; then - echo MISSING $1 +# Print "STATUS path" per log, listing the logs first so ones jq skipped still get reported +{ printf '%s\n' "$@"; printf '%s\n' "$STATS"; } | awk ' + # Single-field lines are the log paths, kept in argument order + NF==1 { logs[++n]=$0; next } -# If some PASS and no FAILs, then consider translation complete -elif [ $PASS -gt 0 ] && [ $FAIL -eq 0 ]; then - echo COMPLETE $1 + # Sum every suite in a log, since one log can hold several test binaries + { pass[$1]+=$2; fail[$1]+=$3 } -# If some PASS and some FAIL, then consider translation in progress -elif [ $PASS -gt 0 ] && [ $FAIL -gt 0 ]; then - echo PARTIAL $1 + END { + for (i = 1; i <= n; i++) { + f = logs[i]; p = pass[f]+0; q = fail[f]+0 -# Otherwise, consider the translation a failure -else - echo FAILED $1 -fi + # No tests at all is MISSING, all passing is complete, a mix is PARTIAL, none passing is FAILED + print (p==0 && q==0 ? "MISSING" : q==0 ? "complete" : p>0 ? "PARTIAL" : "FAILED"), f + } + }' diff --git a/src/ideas/__init__.py b/src/ideas/__init__.py index a3693c7..2c2bc86 100644 --- a/src/ideas/__init__.py +++ b/src/ideas/__init__.py @@ -9,7 +9,6 @@ from .translate_recurrent import RecurrentTranslator from .translate_snippet import SnippetTranslator from .wrapper import WrapperGenerator -from .test_symbol import SymbolTester from clang.cindex import Config __all__ = [ @@ -21,7 +20,6 @@ "RecurrentTranslator", "SnippetTranslator", "WrapperGenerator", - "SymbolTester", ] # NOTE: .so is *nix specific diff --git a/src/ideas/init/__init__.py b/src/ideas/agents/__init__.py similarity index 100% rename from src/ideas/init/__init__.py rename to src/ideas/agents/__init__.py diff --git a/src/ideas/agents/build.py b/src/ideas/agents/build.py deleted file mode 100644 index 20e7a26..0000000 --- a/src/ideas/agents/build.py +++ /dev/null @@ -1,261 +0,0 @@ -# -# Copyright (C) 2026 Intel Corporation -# -# SPDX-License-Identifier: Apache-2.0 -# - -import re -import sys -import logging -import shutil -import textwrap -from pathlib import Path -from dataclasses import dataclass - -import hydra -from omegaconf import MISSING -from hydra.core.config_store import ConfigStore -from hydra.core.hydra_config import HydraConfig - -from ideas.tools import Crate, rustfmt -from ideas.tools import run_subprocess -from ideas.ast_rust import CodeRust, mangle -from ideas import create_translation_unit, extract_info_c -from ideas.init.consolidate import get_symbols_and_dependencies - -logger = logging.getLogger("ideas.init.build") - - -@dataclass -class BuildConfig: - instrumentation: str = MISSING - vcs: str = "none" - - def __post_init__(self): - if self.vcs not in ["git", "none"]: - raise ValueError(f"Invalid VCS: {self.vcs}!") - - if self.instrumentation not in ["coverage", "sanitizers"]: - raise ValueError(f"Invalid instrumentation: {self.instrumentation}!") - - -cs = ConfigStore.instance() -cs.store(name="init.build", node=BuildConfig) - - -def generate_build_script(instrumentation: str) -> tuple[str, str]: - build_options, build_commands = "", "" - if instrumentation == "coverage": - # With UBSan - build_options += '.flag("-fsanitize=undefined,nullability")' - build_options += '.flag("-fsanitize-trap=all")' - build_options += '.flag("-fprofile-instr-generate")' - build_options += '.flag("-fcoverage-mapping")' - - build_commands += 'println!("cargo:rustc-link-lib=dylib=crypto");' - build_commands += 'println!("cargo:rustc-link-lib=m");' - build_commands += ( - 'println!("cargo:rustc-link-search=/usr/lib/llvm-21/lib/clang/21/lib/linux/");' - ) - build_commands += ( - 'println!("cargo:rustc-link-lib=static=clang_rt.ubsan_standalone-x86_64");' - ) - elif instrumentation == "sanitizers": - # With UBSan and ASan - build_options += '.flag("-fsanitize=address,undefined,nullability")' - build_options += '.flag("-fsanitize-trap=all")' - build_commands += 'println!("cargo:rustc-link-lib=dylib=crypto");' - build_commands += 'println!("cargo:rustc-link-lib=m");' - build_commands += ( - 'println!("cargo:rustc-link-search=/usr/lib/llvm-21/lib/clang/21/lib/linux/");' - ) - build_commands += ( - 'println!("cargo:rustc-link-lib=static=clang_rt.ubsan_standalone-x86_64");' - ) - build_commands += 'println!("cargo:rustc-link-lib=static=clang_rt.asan-x86_64");' - elif instrumentation == "none": - build_commands += 'println!("cargo:rustc-link-lib=dylib=crypto");' - build_commands += 'println!("cargo:rustc-link-lib=m");' - - return build_options, build_commands - - -def write_build_script(crate: Crate, build_options: str = "", build_commands: str = "") -> Path: - c_src_path = crate.c_src_path.relative_to(crate.cargo_toml.parent) - build_rs_src = textwrap.dedent( - f""" - fn main() {{ - println!("cargo:rerun-if-changed={c_src_path}"); - - cc::Build::new() - .compiler("clang") - .warnings(false) - .file("{c_src_path}") - {build_options} - .compile("library"); - - {build_commands} - }} - """ - ) - - build_rs_path = crate.cargo_toml.parent / "build.rs" - build_rs_path.write_text(build_rs_src) - rustfmt(build_rs_path) - return build_rs_path - - -def write_symbol_binding(crate: Crate, symbol_name: str): - rust_spelling = mangle(symbol_name) - symbol_binding = get_linked_binding(rust_spelling, crate.c_src_path) - - symbol_binding_path = crate.rust_src_path.parent / "binding" / f"{rust_spelling}.rs" - symbol_binding_path.parent.mkdir(exist_ok=True) - symbol_binding_path.write_text( - "\n\n".join( - [ - "#![allow(unused_attributes)]", - str(symbol_binding), - ] - ) - ) - crate.vcs.add(symbol_binding_path) - - binding_path = crate.rust_src_path.parent / "binding.rs" - with binding_path.open("a+") as f: - f.write(f"pub mod {rust_spelling};\n") - crate.vcs.add(binding_path) - - -def get_linked_binding(function_name: str, c_src_path: Path, *bindgen_args: str) -> CodeRust: - # Use bindgen to generate binding to C symbol - bindgen = [ - "bindgen", - "--disable-header-comment", - "--no-doc-comments", - "--no-layout-tests", - "--allowlist-function", - function_name, - str(c_src_path), - "--", - *bindgen_args, - ] - ok, binding, error, _ = run_subprocess(bindgen) - if not ok: - raise ValueError(f"`{' '.join(bindgen)}` failed!\n{binding + error}") - - # Remove \u{1} prefix from link_name attribute - linked_binding = binding.replace('#[link_name = "\\u{1}', '#[link_name = "') - return CodeRust(linked_binding) - - -def strip_instrumentation(crate: Crate) -> Path: - # Remove coverage script and data files - if (crate.cargo_toml.parent / "measure_coverage.sh").is_file(): - (crate.cargo_toml.parent / "measure_coverage.sh").unlink() - for prof_file in crate.cargo_toml.parent.glob("**/*.profraw"): - prof_file.unlink() - for prof_file in crate.cargo_toml.parent.glob("**/*.profdata"): - prof_file.unlink() - if (crate.cargo_toml.parent / "profraw").is_dir(): - shutil.rmtree(crate.cargo_toml.parent / "profraw") - if (crate.cargo_toml.parent / "json").is_dir(): - shutil.rmtree(crate.cargo_toml.parent / "json") - - # Rewrite `build.rs` to remove all instrumentation - build_options, build_commands = generate_build_script("none") - build_rs_path = write_build_script( - crate, build_options=build_options, build_commands=build_commands - ) - - # Attempt to build the crate - builds, feedback = crate.cargo_build() - if not builds: - raise RuntimeError( - f"Crate at {crate.cargo_toml.parent} does not build without instrumentation!\n{feedback}" - ) - - return build_rs_path - - -def _main(cfg: BuildConfig) -> None: - output_dir = Path(HydraConfig.get().runtime.output_dir) - - # Fetch crate - crate = Crate(output_dir / "Cargo.toml", vcs=cfg.vcs) # type: ignore[reportArgumentType] - - # Get global symbol table - tu = create_translation_unit(crate.c_src_path) - asts = [extract_info_c(tu)] - symbols, _ = get_symbols_and_dependencies( - asts, external_symbol_names=["c:@F@main"] if crate.is_bin else None - ) - global_functions = [ - s for s in symbols.values() if s.is_global and (s.is_function and s.is_definition) - ] - - # Write build.rs file - build_options, build_commands = generate_build_script(cfg.instrumentation) - build_rs_path = write_build_script( - crate, build_options=build_options, build_commands=build_commands - ) - crate.vcs.add(build_rs_path) - msg = f"Wrote `build.rs` at {build_rs_path}" - logger.info(msg) - crate.vcs.commit(msg) - - # Verify build with build.rs - builds, feedback = crate.cargo_build() - if not builds: - raise RuntimeError(f"Crate at {output_dir} does not build with build.rs!\n{feedback}") - - # Write main function and binding to it - main_function = "#![no_main]" if crate.is_bin else "" - with crate.rust_src_path.open("a+") as f: - f.write(main_function) - crate.vcs.add(crate.rust_src_path) - crate.vcs.commit("Added main function (if any) to Rust source") - - # Generate Rust bindings for public library functions - if not crate.is_bin: - binding_path = crate.rust_src_path.parent / "binding.rs" - binding_path.write_text("") - for symbol in global_functions: - if not (symbol.is_function and symbol.is_definition and symbol.is_global): - continue - write_symbol_binding(crate, symbol.spelling) - logger.info("Generated bindings for all global functions") - - # Make the bindings module visible in the crate - rust_src = crate.rust_src_path.read_text() - BINDING_MOD = "pub mod binding;" - if not re.search(f"^{re.escape(BINDING_MOD)}$", rust_src, flags=re.MULTILINE): - crate.rust_src_path.write_text("\n\n".join([rust_src, BINDING_MOD])) - msg = f"Referenced `{BINDING_MOD}` in {crate.rust_src_path}" - logger.info(msg) - else: - msg = f"Binding module `{BINDING_MOD}` was already referenced in {crate.rust_src_path}!" - logger.warning(msg) - crate.vcs.add(crate.rust_src_path) - crate.vcs.commit(msg) - - # Attempt a final build - builds, feedback = crate.cargo_build() - if not builds: - raise RuntimeError(f"Crate at {output_dir} does not build with build.rs!\n{feedback}") - - # Clean on exit - crate.cargo_clean() - - -@hydra.main(version_base=None, config_name="init.build") -def main(cfg: BuildConfig) -> None: - try: - _main(cfg) - except Exception as e: - logger.exception(e) - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/src/ideas/agents/generate_io_tests.py b/src/ideas/agents/generate_io_tests.py new file mode 100644 index 0000000..11e0c0f --- /dev/null +++ b/src/ideas/agents/generate_io_tests.py @@ -0,0 +1,617 @@ +# +# Copyright (C) 2026 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + + +import sys +import os +import logging +import tempfile +import textwrap +import time +import shutil +from pathlib import Path +from dataclasses import dataclass + +import hydra +from omegaconf import MISSING +from hydra.core.config_store import ConfigStore +from hydra.core.hydra_config import HydraConfig + +from ideas import create_translation_unit, extract_info_c +from ideas.agents.printer import ConsoleTee, LoggingConsolePrinter +from ideas.agents.utils import ( + RESTRICT_COVERAGE, + strip_line_directives, + write_instrumentation_script, + write_assert_script, + write_collect_script, + write_profile_list, +) +from ideas.consolidate import get_symbols_and_dependencies, is_system_symbol +from ideas.tools import Crate + +from kiss.agents.sorcar.useful_tools import UsefulTools +from kiss.core.kiss_agent import KISSAgent +from kiss.core.kiss_error import KISSError + +logger = logging.getLogger("ideas.agents.generate_io_tests") + + +@dataclass +class TestgenConfig: + model: str = MISSING + output_path: Path = MISSING + manifest: Path = MISSING + + coverage: int = 100 # [%] branch coverage + + budget: float = 4.0 # USD + steps: int = 100 + + +@dataclass +class TestgenInstructions: + overview: str = textwrap.dedent( + """ + # Overview + The working directory is {work_dir}. All paths are relative to the working directory. + You must work **strictly** inside {work_dir} and never read, write, or list anything outside it. + This directory is self-contained and holds everything you need. + Never use absolute paths that leave it, never use `..` to climb above it, and never `cd` out of it. + + {work_dir} contains a Rust crate that links C code through the Rust C FFI. + Everything you need to know must be derived from the C source in `src/lib.c`. + The C source is amalgamated and has no `#include` directives. + Every declaration and definition you need is in the file. + Do not test or look for the definition of any functions with an `extern` declaration. + + Do not edit the `src/lib.c` file. + """ + ) + + analyze_library: str = textwrap.dedent( + """ + Public (exported) functions have pre-generated FFI bindings using `bindgen` in `src/lib.rs`. + The bindings and build files are correct and must not be changed. + + Understand the C source and, for each public function, determine which parameters are input-only, + which are output-only (written by the callee) and which are modified in place. + This tells you how to set up inputs and where to capture outputs. + + Note any logic that can loop forever and the conditions that trigger it. + Identify any input that would cause undefined behavior so you can avoid it. + The code can only be driven through calling public functions with their arguments set up, + environment variables (if any are used), and any input files it reads. + You cannot call private functions or edit the program to reach more code. + """ + ) + + analyze_binary: str = textwrap.dedent( + """ + The C `main` function (correctly placed in `src/lib.c` and linked in) + is the entry point of the final executable. + The build files are correct and must not be changed. + + Understand the C source and determine how the program uses `argc`/`argv`, whether it reads from stdin, + what it prints to stdout and stderr, and which exit codes it returns. + + Decide whether the program is batch (runs and exits) or interactive (loops on stdin). + If interactive, find its exit condition. + Note any input that can loop forever, and any input that would cause undefined behavior so you can avoid it. + The code can only be driven through the `main` function: command-line arguments, stdin, environment + variables, and any input files it reads. + You cannot call any other function directly or edit the program to reach more code. + """ + ) + + analyze_instrumentation: str = textwrap.dedent( + """ + # Instrumentation + The `{instrument_path}` script instruments the tests for coverage and sanitizers. + It should not be modified and should always be executed to get accurate coverage results. + It takes a single argument selecting which test file to run, either `collect` or `io`: + ```bash + {instrument_path} collect + {instrument_path} io + ``` + + Carefully read and understand this script, it is **critical** for correct measurements. + + Coverage is measured only if every test passes under every sanitizer build; if anything + fails, the script exits early and none of the coverage outputs below are generated. + {coverage_scope} + - `{coverage_dir}/coverage_summary.log`: the aggregate per-file and TOTAL coverage + summary table. Also printed to stdout under `Coverage summary:`. + - `{coverage_dir}/coverage_report.log`: the full per-line annotated coverage report, + including branch coverage details. + - `{coverage_dir}/uncovered_branches.log`: just the uncovered branches (a branch + whose True or False count is zero), or `none` if fully covered. Also printed to + stdout under `Uncovered branches:`. + + Running this script and inspecting its logs is the **only** allowed way to run the tests and get metrics. + You must never invoke `cargo test`, `cargo nextest`, `cargo llvm-cov`, or any other test runner directly, + and you must never derive coverage or output data by any other means. + Every test run must go through `{instrument_path} collect` or `{instrument_path} io`. + No ad-hoc or manual run can replace this script. + + A sanitizer `FAIL` only means the test failed under that sanitizer build. + It does not necessarily mean the test exercises undefined behavior. + The script does not tell the two apart, so read `{sanitizer_dir}/.log`: + - A sanitizer diagnostic (`ERROR: AddressSanitizer:`, `runtime error:`, + `SUMMARY: UndefinedBehaviorSanitizer:`, or a stack trace into the C source) means the + input drives the C code into UB. Change that input, or drop the case. + - A Rust panic (`assertion ... failed`, `panicked at tests/...`) with no sanitizer + diagnostic is a normal test failure. Your expected value is wrong, so fix the test. + - A test killed for running too long is a hang, usually a program waiting on stdin. + Give it the input it waits for, or close its stdin. + + Read the whole report, not just the last error: a sanitizer diagnostic often appears inside + an assertion message. The same test failing under several features is one problem, not many. + + The sanitizers are configured by this script. + Do not set or override `ASAN_OPTIONS`, `UBSAN_OPTIONS`, `LSAN_OPTIONS`, or any other sanitizer + variable, whether in a test, in the environment, or in a config file. + Never mark a test `#[ignore]`, comment it out, or otherwise skip it to get past a failure. + Silencing a check invalidates the whole result. + """ + ) + + goal_library: str = textwrap.dedent( + """ + # Goal + Your goal is to write input/output C FFI tests to `{test_path}` that call public C functions + directly and verify their outputs with `assert!` or `assert_eq!`. + """ + ) + + goal_binary: str = textwrap.dedent( + """ + # Goal + Your goal is to write input/output tests to `{test_path}` that run the binary + and verify its stdout, stderr, and exit code. + """ + ) + + goal_common: str = textwrap.dedent( + """ + You must achieve a branch coverage of at least {target_coverage}%. + You can **never** exercise UB (undefined behavior) in any test. + If you can no longer improve branch coverage (e.g., unreachable code without UB), you may early stop. + + Reach this goal in three steps: + + 1. Collect input/output pairs in `{collect_path}`. + Use the `{instrument_path} collect` command to instrument the tests for coverage and sanitizers. + + 2. Improve branch coverage by appending more collection tests to `{collect_path}`. + Any data collection attempt that exercises UB will be detected by the instrumentation and rejected. + + 3. Write pure assertion I/O tests to `{test_path}` from the collected data. + Use the `{instrument_path} io` command to run a final verification of the coverage and sanitizers. + The coverage should exactly match the outcome of `{instrument_path} collect` and all tests should pass without exercising UB. + """ + ) + + collect_common: str = textwrap.dedent( + """ + ## Step 1: Collect input/output pairs + + Collect input/output pairs of data by writing tests to `{collect_path}`. + + These tests should not assert outputs, but serialize them using `serde` for later conversion + to expected value tests. + + Build up complexity gradually, in two passes: + + 1a. Start with the simplest possible tests: one isolated invocation each, with straightforward inputs. + Cover the common path of every entry point this way before doing anything more elaborate. + 1b. Only once the simple tests are collected and passing, add chained tests that perform several + invocations in sequence, where earlier invocations set up the state for later ones. + + Prefer the simplest test that reaches a given behavior. Reach for a chained test only when the + behavior genuinely cannot be reached by a single invocation. + + Keep tests short and contained, if possible: one behavior per test, named after that behavior. + Never grow an existing test to cover something new. Many small tests are better than a few large ones. + """ + ) + + collect_library: str = textwrap.dedent( + """ + The `{collect_path}` file is pre-populated and imports every + FFI binding for the C library: all functions and all data structures. + Use them exactly as imported; do not redeclare or wrap them. + + It also provides a helper that must be used and must not be changed: + - `save_case(name, case)` serializes any `Serialize` value to `{json_dir}/.json`. + + Append each new test below the + `// ==== Add collection tests below this line ====` marker. + + Call each C function through its FFI binding, set up all of its inputs, and capture the resulting state. + For pointer parameters, allocate the pointed-to data as a local variable and pass a raw pointer to it; + never use hard-coded addresses. NUL-terminate any C strings, or the data will be silently corrupted. + + Start with one test per public function, calling that function exactly once. Only after those exist + should you write chained tests that call several functions in sequence on shared state (for example an + init/update/finalize sequence, or a function whose output feeds the next function's input). + + In a chained test, capture the intermediate state after every call, + not just the final one, so each step can be asserted later. + + If a function reads or writes files, run it inside a fresh temporary directory unique to the test. + Create any input files there first, then capture the files the function creates or modifies + (their paths and contents) as part of the output state. Never touch shared or absolute system paths, + so the collection and I/O tests stay isolated and reproducible. + + Each collection test should finish with a single `save_case("", &case)`, writing one JSON + file to `{json_dir}/.json` per collection test. + Write both inputs and outputs using the same data structure across all collection tests. + This data structure should contain the input and output state of all inputs + (in case functions modify data in-place) and any return value. + """ + ) + + collect_binary: str = textwrap.dedent( + """ + The `{collect_path}` file is pre-populated with two helpers that must be used and must not be changed: + - `run(args, stdin) -> Call` runs the binary once and returns its `stdout`, `stderr`, + and `exit_code`. + - `save_case(name, calls)` serializes an ordered slice of calls to `{json_dir}/.json`. + + Push every `Call` into a local `Vec` in the order it was made, and finish each test with a single + `save_case("", &calls)`, passing the test's own function name. + + Write one `#[test]` per collection test. + Append each new test to the end of the file, below the `// ==== Add collection tests below this line ====` + marker. Leave the helpers above that marker unchanged. + + Start with tests that call `run` exactly once, covering each subcommand or mode on its own with simple + arguments and stdin. Only after those exist should you write chained tests that call `run` several times + in sequence, where earlier invocations set up state (files, configuration) for later ones. In a chained + test, push every call, so each intermediate invocation can be asserted later. + + For programs with multiple subcommands or modes, cover each one, and cover sequential invocations + of subcommands in any relevant combination. + + If the program reads or writes files, set up a fresh temporary directory unique to the test. + Create any input files there first, then capture the files the program creates or modifies + (their paths and contents) as part of the collected output. Never touch shared or absolute system paths, + so tests stay isolated and reproducible. + + Each collection test produces one file, `{json_dir}/.json`, holding the test `name` + and its ordered `calls`. Each call has its `args` and `stdin` inputs and its `stdout`, `stderr`, + and `exit_code` outputs. The script empties `{json_dir}` before every collection run, so the + files left there always match the tests in `{collect_path}`. + """ + ) + + improvement: str = textwrap.dedent( + """ + # Step 2: Improve branch coverage + + After initial data collection, focus on adding more tests to increase branch coverage. + Ensure that additions do not introduce undefined behavior (UB). + + To review the current coverage status, execute: + ```bash + cat {coverage_dir}/coverage_summary.log + ``` + + To review a summary of the current uncovered code branches, execute: + ```bash + cat {coverage_dir}/uncovered_branches.log + ``` + + To review the current uncovered code branches in detail and the complete report, execute: + ```bash + cat {coverage_dir}/coverage_report.log + ``` + + Carefully reason about code paths and behavior to identify program states that may lead to uncovered branches. + + Keep preferring the simplest test that covers a branch: first try new inputs to a single invocation, + and only chain invocations when a branch depends on state left behind by an earlier one. + """ + ) + + assert_common: str = textwrap.dedent( + """ + # Step 3: Write pure assertion I/O tests + + Once branch coverage is satisfactory, write the pure assertion I/O tests to `{test_path}` + using the data collected in the `{json_dir}` JSON files. + + Write one I/O test per collection test, reading the expected values from the matching JSON file. + These tests must be pure: hard-code the expected values as plain Rust literals and do not depend on + `serde`, `serde_json`, the `{json_dir}` files, or the `{collect_path}` file in any way. Each test must + set up its own inputs and assert every recorded output, so it still passes if moved to another crate. + + An I/O test must mirror the structure of the collection test it came from. For a chained collection test, + replay the same sequence in the same order and assert the recorded intermediate state after every step, + not only the final result. An intermediate step that is not asserted is a missing assertion. + + Do not skip any assertion, and do not weaken one just to make a test pass. + """ + ) + + assert_library: str = textwrap.dedent( + """ + The `{test_path}` file is pre-populated with `use {lib_name}::*;`, the same FFI bindings + `{collect_path}` imports. Append each new test to the end of the file, below the + `// ==== Add assertion tests below this line ====` marker. + + For each JSON file, reconstruct the recorded input state as Rust literals, call the C function through + its FFI binding, and assert that every output field matches the recorded output state: + - use `assert_eq!` for integer and boolean fields; + - for floating-point fields, compare with a small relative epsilon rather than exact equality; + - for pointer outputs, dereference the pointer and compare the pointed-to value, not the address; + - for files the function created or modified, recreate the recorded inputs in a fresh temporary + directory and assert the resulting file paths and contents. + + For a chained case, call the same functions in the same order and assert the recorded state after each + call, including any state modified in place by an earlier call. + """ + ) + + assert_binary: str = textwrap.dedent( + """ + The `{test_path}` file is pre-populated with a helper that must be used and must not be changed: + - `run(args, stdin) -> Call` runs the binary once and returns its `stdout`, `stderr`, + and `exit_code`. + + This is the only way you may run the binary. It uses `stdbuf -e0 -o0`, exactly like the + collection helper, so it sees the same unbuffered output the recorded values came from. + Never use `Command::cargo_bin` or `std::process::Command` in this file. + It is the collection helper without the recording, so a collection test body carries over as is: + drop the `Vec` and the `save_case` call, and assert each `Call` instead. + + Append each new test to the end of the file, below the + `// ==== Add assertion tests below this line ====` marker, and leave the helper above it unchanged. + + For each JSON file, replay its `calls` in order and assert each one against literals: + + ```rust + let call = run(&["-E", "-"], Some("int x;\\n")); + assert_eq!(call.exit_code, 0); + assert_eq!(call.stdout, "int x;\\n"); + assert_eq!(call.stderr, ""); + /// TODO: File state assertions, if any + ``` + + When a value changes between runs (temporary paths, PIDs), assert the stable part with + `predicates::str::contains`. For files the program created or changed, recreate the recorded + inputs in a fresh temporary directory and assert the resulting paths and contents. + + Assert every call in the sequence, not just the last one. + """ + ) + + finish: str = textwrap.dedent( + """ + Once the target coverage is achieved and all I/O tests are written, exit immediately. + Do not generate extensive reports or perform redundant sanity checks. + """ + ) + + library: str = ( + overview + + analyze_library + + analyze_instrumentation + + goal_library + + goal_common + + collect_common + + collect_library + + improvement + + assert_common + + assert_library + + finish + ) + binary: str = ( + overview + + analyze_binary + + analyze_instrumentation + + goal_binary + + goal_common + + collect_common + + collect_binary + + improvement + + assert_common + + assert_binary + + finish + ) + + +cs = ConfigStore.instance() +cs.store(name="generate_io_tests", node=TestgenConfig) + + +def get_tools(): + useful_tools = UsefulTools() + return [useful_tools.Bash, useful_tools.Read, useful_tools.Edit, useful_tools.Write] + + +def _wrapup_notice(collect_path: Path, test_path: Path) -> str: + return ( + "Stop starting new work and consolidate what you already have: finish any " + f"test you left half-written, and make `{test_path}` mirror `{collect_path}` " + "exactly, one assertion test per collected case. Running out of steps is not " + "a reason to cut corners: keep every assertion, do not weaken or drop one to " + "make a test pass, do not mark tests `#[ignore]`, and do not touch the " + "sanitizer configuration. If a case cannot be finished properly, remove it " + "from both files rather than leaving a broken version of it behind." + ) + + +def _main(cfg: TestgenConfig) -> None: + output_dir = Path(HydraConfig.get().runtime.output_dir) + logger.info(f"Saving results to {output_dir}") + + # Separately log the complete trajectory + logger_trajectory = logging.getLogger("ideas.agents.generate_io_tests.trajectory") + logger_trajectory.propagate = False + trajectory_log = output_dir / f"testgen-{int(time.time())}.log" + fh = logging.FileHandler(trajectory_log) + fh.setFormatter(ConsoleTee.StripANSIFormatter("%(asctime)s %(message)s")) + logger_trajectory.addHandler(fh) + # Simultaneous print and log to file + printer = LoggingConsolePrinter(logger=logger_trajectory) + agent = KISSAgent(name="C code reviewer") + + # -sys crate setup + sys_crate = Crate(cfg.manifest) + sys_root = sys_crate.cargo_toml.parent + sys_test_dir = sys_root / "tests" + sys_json_dir = sys_root / "json" + sys_test_dir.mkdir(parents=True, exist_ok=True) + sys_json_dir.mkdir(parents=True, exist_ok=True) + + # Analyze consolidated code to find all reachable symbols + template = "bin" if len(sys_crate.bin_targets) > 0 else "lib" + assert sys_crate.lib_src_path is not None, "Expected lib.rs to exist in -sys crate!" + c_src_path = sys_crate.lib_src_path.with_suffix(".c") + tu = create_translation_unit(c_src_path) + asts = [extract_info_c(tu)] + symbols, _ = get_symbols_and_dependencies( + asts, external_symbol_names=["c:@F@main"] if template == "bin" else None + ) + profile_list = None + if RESTRICT_COVERAGE: + profile_functions = [ + s.spelling + for s in symbols.values() + if s.is_function and s.is_definition and not is_system_symbol(s) + ] + if profile_functions: + logger.info(f"Restricting coverage to {len(profile_functions)} functions") + profile_list = write_profile_list(sys_root / "profile.lst", profile_functions) + else: + logger.warning("No instrumentable functions found, coverage stays unrestricted!") + + # Add testing dependencies + if template == "bin": + sys_crate.cargo_add(dep="assert_cmd@2.0.17", section="dev") + sys_crate.cargo_add(dep="predicates@3.1.3", section="dev") + sys_crate.cargo_add(dep="libc@0.2", section="dev") + sys_crate.cargo_add(dep="serde@1", section="dev", features=["derive"]) + sys_crate.cargo_add(dep="serde_json@1", section="dev") + sys_crate.invalidate_metadata() + + # Write instrumentation and test scripts + write_collect_script( + sys_test_dir / "collect.rs", + template=template, + lib_name=sys_crate.lib_name if template == "lib" else None, + ) + write_assert_script( + sys_test_dir / "io.rs", + template=template, + lib_name=sys_crate.lib_name if template == "lib" else None, + ) + write_instrumentation_script( + sys_root / "instrument.sh", features=["cc_asan", "cc_ubsan"], profile_list=profile_list + ) + + # Isolate the crate + work_dir = Path(tempfile.mkdtemp()) / sys_root.name + shutil.copytree( + sys_root, work_dir, dirs_exist_ok=True, ignore=shutil.ignore_patterns("*.log", ".*") + ) + + # Strip line directives + for c_file in (work_dir / "src").glob("*.c"): + strip_line_directives(c_file) + + # If this is a binary, remove the `lib.rs` file and simplify `main.rs` + if template == "bin": + lib_rs = work_dir / "src" / "lib.rs" + assert lib_rs.exists(), "Expected to find lib.rs in -sys crate" + lib_rs.unlink() + main_rs = work_dir / "src" / "main.rs" + main_rs.write_text("#![no_main]\n") + + # Build the task prompt + task_description = ( + TestgenInstructions.library if template == "lib" else TestgenInstructions.binary + ).strip() + collect_path = (sys_test_dir / "collect.rs").relative_to(sys_root) + test_path = (sys_test_dir / "io.rs").relative_to(sys_root) + coverage_scope = ( + f"Coverage is restricted to the program's own functions through `{profile_list.name}`;" + " libc and system code is never counted, so do not try to cover it, and never modify" + " that file:" + if profile_list is not None + else "The coverage outputs are:" + ) + task_description = task_description.format( + work_dir=work_dir, + instrument_path="instrument.sh", + target_coverage=cfg.coverage, + collect_path=collect_path, + test_path=test_path, + json_dir=sys_json_dir.relative_to(sys_root), + sanitizer_dir="sanitizer_logs", + coverage_dir="coverage_logs", + coverage_scope=coverage_scope, + lib_name=sys_crate.lib_name or "", + ) + + # Run the agent + os.chdir(work_dir) + agent.wrapup_steps = 10 + agent.wrapup_notice = _wrapup_notice(collect_path, test_path) + try: + agent.run( + model_name=cfg.model, + prompt_template=task_description, + max_steps=cfg.steps, + max_budget=cfg.budget, + tools=get_tools(), + printer=printer, + verbose=True, + ) + except KISSError as e: + logger.warning(f"Agent claims it failed with error: {e}") + + # Copy the generated tests back + work_test_dir = work_dir / "tests" + sys_test_dir.mkdir(parents=True, exist_ok=True) + for name in ("collect.rs", "io.rs"): + src = work_test_dir / name + if src.exists(): + shutil.copy2(src, sys_test_dir / name) + + # Copy the collected JSON data back + work_json_dir = work_dir / "json" + if work_json_dir.is_dir(): + shutil.copytree(work_json_dir, sys_json_dir, dirs_exist_ok=True) + else: + logger.warning("The agent removed the JSON directory!") + + # Guarantee tests are generated + placeholder = "#[test]\nfn placeholder() {\n assert_eq!(1, 1);\n}\n" + for name in ("collect.rs", "io.rs"): + test_file = sys_test_dir / name + if not test_file.exists(): + test_file.write_text(placeholder) + logger.warning( + f"{name} not generated by the agent, writing always-pass placeholder!" + ) + sys_crate.vcs.add(sys_test_dir / "collect.rs", sys_test_dir / "io.rs", sys_json_dir) + sys_crate.vcs.commit("Generated I/O equivalence tests") + + +@hydra.main(version_base=None, config_name="generate_io_tests") +def main(cfg: TestgenConfig) -> None: + try: + _main(cfg) + except Exception as e: + logger.exception(e) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/src/ideas/agents/printer.py b/src/ideas/agents/printer.py index 17b347a..4eb2218 100644 --- a/src/ideas/agents/printer.py +++ b/src/ideas/agents/printer.py @@ -6,14 +6,12 @@ import sys import logging -from typing import Any, TextIO +from typing import TextIO from rich.text import Text from rich.console import Console -from rich.panel import Panel from kiss.core.print_to_console import ConsolePrinter -from kiss.core.printer import truncate_result class ConsoleTee: @@ -64,34 +62,3 @@ def __init__(self, logger: logging.Logger, level: int = logging.INFO): tee = ConsoleTee(sys.__stdout__ or sys.stdout, logger, level) super().__init__(file=tee) self._console = Console(highlight=False, file=tee) # type: ignore[reportArgumentType] - - def print(self, content: Any, type: str = "text", **kwargs: Any) -> str: - if type == "tool_result": - self._flush_newline() - self._print_tool_result(str(content), kwargs.get("is_error", False)) - return "" - - if type == "usage_info": - self._flush_newline() - self._console.print( - Panel( - Text(str(content).strip(), style="dim italic"), - border_style="dim", - padding=(0, 1), - expand=True, - ) - ) - return "" - - return super().print(content, type=type, **kwargs) - - def _print_tool_result(self, content: str, is_error: bool = True) -> None: - style = "red" if is_error else "green" - self._console.rule("FAILED" if is_error else "OK", style=style, align="center") - if not self._bash_streamed: - display = truncate_result(content) - for line in display.splitlines(): - self._file.write(line + "\n") - self._file.flush() - self._bash_streamed = False - self._console.rule(style=style) diff --git a/src/ideas/agents/testgen.py b/src/ideas/agents/testgen.py deleted file mode 100644 index d676d95..0000000 --- a/src/ideas/agents/testgen.py +++ /dev/null @@ -1,515 +0,0 @@ -# -# Copyright (C) 2026 Intel Corporation -# -# SPDX-License-Identifier: Apache-2.0 -# - - -import sys -import os -import logging -import tempfile -import textwrap -import time -import shutil -from pathlib import Path -from dataclasses import dataclass - -import hydra -from omegaconf import MISSING -from hydra.core.config_store import ConfigStore -from hydra.core.hydra_config import HydraConfig - -from ideas.tools import Crate -from ideas.agents.printer import ConsoleTee, LoggingConsolePrinter -from ideas.agents.build import strip_instrumentation -from ideas.agents.utils import ( - NEXTEST_DUMMY_TEST, - nextest_config, - write_coverage_script, - write_collect_script, - write_extract_json_script, -) - -from kiss.agents.sorcar.useful_tools import UsefulTools -from kiss.core.relentless_agent import RelentlessAgent -from kiss.core.kiss_error import KISSError - -logger = logging.getLogger("ideas.agents.testgen") - - -@dataclass -class TestgenConfig: - cargo_toml: Path = MISSING - model: str = MISSING - c_code: Path = MISSING - project_name: str = MISSING - test_crate_out: Path = MISSING - - guarantee_assert_tests: bool = False - collect_to_assert: bool = False - target_coverage: int = 90 - - def __post_init__(self): - if not self.c_code.is_file(): - raise ValueError(f"c_code must be a single C file, got: {self.c_code}") - - -@dataclass -class TestgenInstructions: - analyze_file: str = textwrap.dedent( - """ - ## Step 1 – Analyze the standalone C file ## - Carefully read and understand the single C source file at - `{c_proj_path}/lib.c`. - This is a **standalone** C library file that should **never** be edited. - - For each exported function analyze: which parameters are **input-only**, - which are **output-only** (written by the callee), and which are **modified in-place** to understand - how to set up its test data and collect its outputs. - - **Infinite looping:** If any function (including static ones) loops infinitely, - identify all relevant paths and their trigger conditions. - - **Undefined behavior:** Carefully analyze the C code for possible undefined behavior (UB). - """ - ) - - build_rs: str = textwrap.dedent( - """ - ## Step 2 – Analyze test crate ## - The directory at {rs_crate_path} contains a Rust crate that links the C code - and has **no** code of its own. - - Analyze the `Cargo.toml` file and the `build.rs` files, and understand how they link the C code. - The crate is a **library** crate, and all exported functions have pre-generated C FFI bindings. - - These bindings have been generated by `bindgen` and placed in the `src/binding` directory, one file per function. - They are **correct** and **definitive** and their interfaces should **never** be changed. - - The `build.rs` and `bindings` modules can **never** be modified, no matter the circumstances. - - ### Working directory ### - Execute `cd {rs_crate_path}` to enter the crate directory before any `cargo` command. - Build using `cargo build` to confirm the C code compiles and links. - - ### Test framework ### - The crate uses `cargo nextest` as the test framework exclusively. - This **guarantees** that all tests are run in parallel and that no test can rely on side effects from another test. - You **must** use `cargo nextest` to run tests, and you **must not** write any test that relies on shared state or side effects. - """ - ) - - coverage_script: str = textwrap.dedent( - """ - ### Measuring coverage ### - The crate is set up to measure source-based coverage of the C code with LLVM's sanitizers and coverage tools. - Understand how this is done by analyzing the `build.rs` file and the `Cargo.toml`. - - The script `{rs_crate_path}/measure_coverage.sh` is used to run tests and measure C code coverage of the tests that will be written. - This script must be run at any time to get an updated coverage report and identify untested code paths. - - This is the **only** way to measure coverage, so do not attempt to use other tools or methods. - Instead write any relevant experiments as collection tests, as indicated in Step 3 below. - """ - ) - - write_collect_tests: str = textwrap.dedent( - """ - ## Step 3 – Generate a data-collection test harness ## - Based on the C program analysis, design input tests that achieve high coverage of the program. - - Each test must be **independent** and **self-contained**: it must set up its own input data, - call the function under test, and capture all relevant output data without - relying on any shared state or side effects from other tests. - Because of `cargo nextest`'s parallel execution, clean-up on exit is **not** required. - - ### 3a – FFI linkage ### - Import the FFI functions through the binding modules using - **absolute crate paths**. The crate name is derived from the `name` field in - `Cargo.toml` (with hyphens replaced by underscores). Import like this: - ```rust - use ::binding::::; - ``` - This is **mandatory** because the C static library is attached to the - library crate by `build.rs`. - - ### 3b – `#[repr(C)]` struct mirrors ### - Import the `#[repr(C)]` struct types through the binding modules (they were - generated by `bindgen` and placed in `src/binding/.rs`): - ```rust - use ::binding::::; - ``` - - If multiple functions use **exactly** the same struct type, - it will be generated in each relevant binding module with the exact name and layout, - so you can import it **only once** from any of them. - - Then add `#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]` to - **local** wrapper types or re-definitions of those structs that you need for - JSON serialization. Because `serde` derives cannot be added to a type imported - from another crate, you may need to define local copies of the structs in the - test file with the exact same layout and field names, adding the serde derives. - Make sure the field names, types, and order match the `bindgen`-generated - definitions exactly. - - ### 3c – Helper: JSON state container ### - Define a `#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]` struct - called `LibState` whose field names match the parameters of the C function - (one field per parameter, using the parameter name from the C header). - **Important – pointer-typed fields**: any field in the C signature that is a - pointer (e.g. `*mut T`, `*const T`) must NOT be hard-coded as a numeric - address. Instead: - - Instantiate the pointed-to data structure as a local `let mut` variable - with values populating all its fields. - - Store the underlying data structure in the `LibState` struct. - - Obtain a raw pointer to it (e.g. `&mut local_var as *mut T`) and pass that pointer - to the function-under-test. - This ensures the pointer is valid for the duration of the call and that the - test does not rely on hard-coded addresses. - If the function returns a value, add a `returns` field. - Nested C structs must be represented by the mirrored Rust struct – never flatten - them. - - Define a wrapper: - ```rust - #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] - struct TestVector {{ - lib_state_in: LibState, - lib_state_out: LibState, - }} - ``` - - ### 3d – Data-collection test functions ### - For each set of input values, write a `#[test]` function named `collect_vector_` that: - 1. Constructs a `LibState` with the chosen inputs (and outputs / return field zeroed). - 2. Clones it into `lib_state_in`. - 3. Calls the C function through `unsafe`, using the symbol imported from the - binding module, passing (and receiving) mutable references where needed. - 4. Captures the post-call state into `lib_state_out`. - 5. Asserts the call did not obviously fail (e.g. no null-pointer dereference – a - simple `assert!` that pointers are non-null or that expected invariants hold). - 6. Serializes a `TestVector {{ lib_state_in, lib_state_out }}` to pretty JSON and - prints it to stdout with: - ```rust - println!("{{}}", serde_json::to_string_pretty(&vector).unwrap()); - ``` - - The chosen inputs **cannot exercise undefined behavior (UB)**! - If they do, instrumentation will make `cargo nextest run` output a - failed test and return an error, and test generation should be re-attempted. - If issues are identified, do **not** give up early. - - The chosen inputs should exercise a variety of code-paths in the C function, including: - - zeroed / default / neutral input - - "normal" inputs with representative non-trivial values - - The tests generated in this step are not meant to **assert** outputs, but only collect them - and they should **not** assume any state in the test file. - They are only meant to be a harness to collect input/output data and coverage information. - - ### NUL-terminated strings in C ### - If the C function takes string inputs, remember that they must be NUL-terminated. - Not respecting this will cause silent memory corruption and make it impossible to collect meaningful data! - To create a NUL-terminated string in Rust, you can create a `Vec` with the string bytes and a trailing `0`, - and then pass a pointer to its first element. - - ### Non-persistence ### - **All** collection tests must be designed to be run repeatedly without any clean-up, - and they must not rely on any side effects or shared state. - - ### Portable, self-contained tests ### - If the C code relies on pre-existing files on disk (e.g., through hardcoded paths), - you must ensure that all tests **locally** create any required files with the expected content before calling the function under test, - and that they do not rely on any pre-existing state on disk. - If multiple tests reference **exactly** the same file, place a safe Lock around all accesses to that file to - prevent race conditions; `cargo nextest` handles parallel execution by default otherwise. - You **cannot** rely on files on-disk: the goal is for the test file to be moved to some other crate and still work. - - If the C code relies on network access, you must ensure that all tests mock the network interactions locally - and do not rely on any external network state or connectivity. - """ - ) - - coverage_improvement: str = textwrap.dedent( - """ - ### The coverage metric ### - Use **branch coverage** to identify and exercise untested code paths. - - To improve branch coverage, generate interesting combinations of input arguments - with special attention to edge cases and boundary conditions. - - Ensure the new input values exercise **well-defined** code paths that improve branch coverage. - Exercising UB will be caught and rejected by the sanitizers! - - Aim to achieve branch coverage of at least {target_coverage}%%. - If this is not possible because of unreachable static functions, a lower coverage is acceptable. - After **three** consecutive attempts where branch coverage has not improved by at least 1 percentage point, - you may stop trying to improve coverage and proceed to the next step. - - Verify that all tests pass and JSON is correctly printed for **all** of them, - including tests on new symbols added for improving coverage. - """ - ) - - analyze_data_collection_tests: str = textwrap.dedent( - """ - ## Step 3 – Analyze the data-collection tests ## - The crate already contains some data-collection tests in `tests/test_collect.rs` - designed to print JSON outputs by running them and capturing their stdout. - - Carefully analyze the `tests/test_collect.rs` file and understand how it imports the FFI symbols, - how it defines the `LibState` struct and the `collect_vector_` tests, and how it prints the JSON output. - - Execute `cargo nextest run --test test_collect --nocapture 2>/dev/null` - to run the tests and see the JSON output they print on the `stdout` channel. - Validate **all** collection tests run successfully and print valid JSON with the expected structure. - - If tests pass **do NOT** modify them in any way at this stage. - If tests exercise UB or trip sanitizers, you must remove them. - If tests fail functionally, attempt to fix them until they pass and print the expected JSON. - """ - ) - - write_test_vectors: str = textwrap.dedent( - """ - ## Step 4 – Save outputs as JSON files ## - Create the directory `{test_vectors_path}`. - - Run each data-collection test **individually** and capture its stdout. - Write the JSON output of each `collect_vector_` test to - `{test_vectors_path}/.json`, where `` is the 1-based index. - - You **must** collect data from **all** tests, not just the initial - ones. - - Use the provided `extract_json.py` script to extract the JSON reliably – do NOT rely on grep/sed: - ```bash - cargo nextest run --nocapture -- collect_vector_ --exact 2>/dev/null | \ - uv run extract_json.py > {test_vectors_path}/.json - ``` - - Verify each file is valid JSON with the expected `lib_state_in` / `lib_state_out` - structure by running `uv run python -m json.tool {test_vectors_path}/.json`. - """ - ) - - write_assert_tests: str = textwrap.dedent( - """ - ## Step 5 – Write assert-style Rust tests ## - Create `{rs_crate_path}/tests/test_assert.rs`. - - Import the FFI symbols through the binding modules, the same way - `test_collect.rs` does: - ```rust - use ::binding::::; - ``` - If the function call requires `#[repr(C)]` structs, import them too: - ```rust - use ::binding::::; - ``` - - ### Plaintext literals and no serde ### - `test_assert.rs` must **not** depend on `serde` or `serde_json`. - Because the crate's types are exact `bindgen` output (no serde derives), the - assert tests reconstruct all values as **plain Rust literals** taken from the - JSON files saved in Step 4. Do **not** `#[derive(Serialize, Deserialize)]` on - any type in this file and do **not** add `use serde*` or `use serde_json*`. - - You **must** write an assertion test for each collection test, no matter - how many collection tests are there! - Write them one-by-one if there are too many. - - For **each** JSON output saved in Step 4, write a `#[test]` function named - `test_vector_` that: - 1. Reconstructs the `lib_state_in` values from the JSON file as Rust literals. - 2. Calls the C function through `unsafe` using the imported symbol. - 3. Asserts **every** field of the output state matches `lib_state_out` from the - JSON file. - - For floating-point fields use an epsilon comparison: - ```rust - assert!((actual - expected).abs() / expected.abs() < 1e-3, - "field ``: expected {{expected}}, got {{actual}}"); - ``` - - For integer / bool fields use `assert_eq!`. - - For pointer-typed output fields, dereference the pointer (inside `unsafe`) - and compare the pointed-to value rather than the pointer address itself. - - Focus on writing meaningful assertions that compare relevant output fields, - rather than writing minimal assertions that only check a few fields or non-null pointers. - - Once done, run: - ```bash - cargo nextest run --test test_assert --cargo-quiet - ``` - All tests **must** pass and not exercise any undefined behavior. - """ - ) - - simple_exit: str = textwrap.dedent( - """ - Once the task is complete, exit immediately. - Do not over-verify or generate extensive reports. - """ - ) - - @classmethod - def coverage_based(cls) -> str: - return ( - cls.analyze_file - + cls.build_rs - + cls.coverage_script - + cls.write_collect_tests - + cls.coverage_improvement - + cls.write_test_vectors - + cls.write_assert_tests - + cls.simple_exit - ) - - @classmethod - def collect_to_assert(cls) -> str: - return ( - cls.analyze_file - + cls.build_rs - + cls.analyze_data_collection_tests - + cls.write_test_vectors - + cls.write_assert_tests - + cls.simple_exit - ) - - -cs = ConfigStore.instance() -cs.store(name="testgen", node=TestgenConfig) - - -def get_tools(): - useful_tools = UsefulTools() - return [useful_tools.Bash, useful_tools.Read, useful_tools.Edit, useful_tools.Write] - - -@hydra.main(version_base=None, config_name="testgen") -def main(cfg: TestgenConfig) -> None: - try: - _main(cfg) - except Exception as e: - logger.exception(e) - sys.exit(1) - - -def _main(cfg: TestgenConfig) -> None: - output_dir = Path(HydraConfig.get().runtime.output_dir) - logger.info(f"Saving results to {output_dir}") - - # Separately log the complete trajectory - logger_trajectory = logging.getLogger("ideas.testgen.trajectory") - logger_trajectory.propagate = False - fh = logging.FileHandler(output_dir / f"testgen_trajectory-{int(time.time())}.log") - fh.setFormatter(ConsoleTee.StripANSIFormatter("%(asctime)s %(message)s")) - logger_trajectory.addHandler(fh) - # Simultaneous print and log to file - printer = LoggingConsolePrinter(logger=logger_trajectory) - agent = RelentlessAgent(name="C library test generator") - - # Generate helper scripts and files in the crate - crate = Crate(output_dir / "Cargo.toml") - nextest_config(crate) - if not cfg.collect_to_assert: - write_coverage_script(crate) - write_collect_script(crate) - write_extract_json_script(crate) - - # Workspace - work_dir = Path(tempfile.mkdtemp()) - workspace_dir = work_dir / "test_crates" - shutil.copytree("test_crates", workspace_dir) - - # Remove all log files - for log_file in workspace_dir.glob("**/*.log"): - log_file.unlink() - - # Paths the agent will populate - rs_crate_path = work_dir / cfg.test_crate_out - test_vectors_path = rs_crate_path / "json" - - # If assertion tests already exist, they must be correct - if (rs_crate_path / "tests" / "test_assert.rs").is_file(): - crate = Crate(rs_crate_path / "Cargo.toml") - ok, output, error, _ = crate.cargo_test("test_assert", quiet=True) - if not ok: - raise RuntimeError( - "Existing assertion tests failed to pass, previous agent did not clean them up!" - ) - logger.info( - f"Assertion tests already exist at {rs_crate_path / 'tests/test_assert.rs'}, skipping agent!" - ) - return - - # Hide instrumentation from conversion agent - if cfg.collect_to_assert: - strip_instrumentation(crate) - - # Build the task prompt - task_description = ( - TestgenInstructions.collect_to_assert() - if cfg.collect_to_assert - else TestgenInstructions.coverage_based() - ) - arguments = { - "c_proj_path": cfg.c_code.parent, - "rs_crate_path": rs_crate_path.relative_to(work_dir), - "test_vectors_path": test_vectors_path.relative_to(work_dir), - "target_coverage": cfg.target_coverage, - } - task_description = task_description.format(**arguments) - - # Run agent in the work directory - os.chdir(work_dir) - try: - agent.run( - model_name=cfg.model, - prompt_template=task_description, - max_steps=100, - max_budget=4, - max_sub_sessions=1, - work_dir=str(work_dir), - tools=get_tools(), - printer=printer, - verbose=True, - ) - except KISSError as e: - logger.warning(f"Agent claims it failed with error: {e}. Clean-up will continue.") - - # Verify that collection tests exist - if not (rs_crate_path / "tests" / "test_collect.rs").is_file(): - raise RuntimeError( - f"Data collection tests were not found at {rs_crate_path / 'tests/test_collect.rs'}!" - ) - - # Strip instrumentation to ensure tests are correct and do not rely on it - crate = Crate(rs_crate_path / "Cargo.toml") - strip_instrumentation(crate) - ok, output, error, _ = crate.cargo_test("test_collect", quiet=True) - if not ok: - raise RuntimeError( - f"Data collection tests failed to pass without instrumentation! Output:\n{output}\nError:\n{error}" - ) - - # Check if assertion tests pass - ok, output, error, _ = crate.cargo_test("test_assert", quiet=True) - if not ok: - logger.error(f"Assertion tests failed to pass! Output:\n{output}\nError:\n{error}") - # Remove incomplete assertion tests, if any - if (rs_crate_path / "tests" / "test_assert.rs").is_file(): - (rs_crate_path / "tests" / "test_assert.rs").unlink() - - # And replace with an always-passing test (nextest does not allow empty test files) - if cfg.guarantee_assert_tests: - logger.warning("Writing dummy test_assert.rs that always passes") - (rs_crate_path / "tests" / "test_assert.rs").write_text(NEXTEST_DUMMY_TEST) - - # Clean the crate and copy it back to the project directory - crate.cargo_clean() - shutil.copytree(rs_crate_path, output_dir, dirs_exist_ok=True) - - -if __name__ == "__main__": - main() diff --git a/src/ideas/agents/testgen_bin.py b/src/ideas/agents/testgen_bin.py deleted file mode 100644 index c6ac1f7..0000000 --- a/src/ideas/agents/testgen_bin.py +++ /dev/null @@ -1,464 +0,0 @@ -# -# Copyright (C) 2026 Intel Corporation -# -# SPDX-License-Identifier: Apache-2.0 -# - - -import sys -import os -import logging -import tempfile -import textwrap -import time -import shutil -from pathlib import Path -from dataclasses import dataclass - -import hydra -from omegaconf import MISSING -from hydra.core.config_store import ConfigStore -from hydra.core.hydra_config import HydraConfig - -from ideas.tools import Crate -from ideas.agents.printer import ConsoleTee, LoggingConsolePrinter -from ideas.agents.build import strip_instrumentation -from ideas.agents.utils import ( - NEXTEST_DUMMY_TEST, - nextest_config, - write_coverage_script, - write_collect_script, - write_extract_json_script, -) - -from kiss.agents.sorcar.useful_tools import UsefulTools -from kiss.core.relentless_agent import RelentlessAgent -from kiss.core.kiss_error import KISSError - -logger = logging.getLogger("ideas.agents.testgen_bin") - - -@dataclass -class TestgenConfig: - cargo_toml: Path = MISSING - model: str = MISSING - c_code: Path = MISSING - project_name: str = MISSING - test_crate_out: Path = MISSING - - guarantee_assert_tests: bool = False - collect_to_assert: bool = False - target_coverage: int = 90 - - def __post_init__(self): - if not self.c_code.exists(): - raise ValueError(f"c_code must be a single C file, got: {self.c_code}") - - -@dataclass -class TestgenInstructions: - analyze_file: str = textwrap.dedent( - """ - ## Step 1 – Analyze the standalone C file ## - Carefully read and understand the single C source file at - `{c_proj_path}/main.c`. - - This is a **standalone** C file that should **never** be edited. - If the file is too large to analyze in one go, focus on its `main` function. - - Analyze how the program uses `argc`/`argv`, whether it reads from `stdin`, - what it prints to `stdout`/`stderr`, which exit codes it returns, and which - system libraries it links against. - - **Interactive program handling:** Determine whether the program is **batch** - (runs and exits) or **interactive** (loops on `stdin`, e.g. `while(1)` + - `fgets`/`scanf`). If interactive, identify the exit condition (menu - choice, special command, or EOF only). - - **Infinite looping:** If the program loops infinitely, identify all relevant paths - and their trigger conditions. - - **Undefined behavior:** Carefully analyze the C code for possible undefined behavior (UB). - """ - ) - - build_rs: str = textwrap.dedent( - """ - ## Step 2 – Analyze test crate ## - The directory at {rs_crate_path} contains a Rust crate that links the C code - and has **no** code of its own. - - Analyze the `Cargo.toml` file and the `build.rs` files, and understand how they link the C code. - The crate is a **binary** crate, so the C `main` function is the real entry point of the final executable. - - ### Working directory ### - Execute `cd {rs_crate_path}` to enter the crate directory before any `cargo` command. - Build using `cargo build` to confirm the C code compiles and links. - - ### Test framework ### - The crate uses `cargo nextest` as the test framework exclusively. - This **guarantees** that all tests are run in parallel and that no test can rely on side effects from another test. - You **must** use `cargo nextest` to run tests, and you **must not** write any test that relies on shared state or side effects. - """ - ) - - coverage_script: str = textwrap.dedent( - """ - ### Measuring coverage ### - The crate is set up to measure source-based coverage of the C code with LLVM's sanitizers and coverage tools. - Understand how this is done by analyzing the `build.rs` file and the `Cargo.toml`. - - The script `{rs_crate_path}/measure_coverage.sh` is used to run tests and measure C code coverage of the tests that will be written. - This script must be run at any time to get an updated coverage report and identify untested code paths. - This script is **complete and correct** as-is, and the task is to write tests that can be measured with it. - - This is the **only** way to measure coverage, so do not attempt to use other tools or methods. - Instead write any relevant experiments as collection tests, as indicated in Step 3 below. - """ - ) - - write_collect_tests: str = textwrap.dedent( - """ - ## Step 3 – Generate a data-collection test harness ## - Based on the C program analysis, design input tests that achieve high coverage of the program. - - Each test must be **independent** and **self-contained**: it must set up its own input data, - call the function under test, and capture all relevant output data without - relying on any shared state or side effects from other tests. - Because of `cargo nextest`'s parallel execution, clean-up on exit is **not** required. - - The test cases **cannot exercise undefined behavior (UB) or infinite loops**! - If they do, instrumentation will make `cargo nextest run` output a - failed test and return an error, and they should be re-attempted. - - Include at least: - - a default / no-argument invocation (if the program supports it) - - a typical invocation with representative arguments - - an edge-case or boundary invocation (empty input, very long input, - special characters, etc.) - - an error path that triggers a non-zero exit code or stderr output - (if the program has any such path) - - The `{rs_crate_path}/tests/test_collect.rs` file begins with the `collect_and_print` - function that **must** be used to collect all test cases. - This file can be considered **complete and correct** as-is, and the task is to write test cases that call `collect_and_print`. - Note the `stdbuf` approach is **required** to ensure proper `libc` output buffering. - - For **each** test case, write a - `#[test]` function named `collect_` that calls `collect_and_print` - with the test case's name, args, and stdin. Example: - ```rust - #[test] - fn collect_() {{ - collect_and_print("", &["arg1", "arg2"], Some("stdin data")); - }} - ``` - Pass `None` for stdin when the test case has no input. - - The tests generated in this step are not meant to **assert** outputs, but only collect them - and they should **not** assume any state in the test file. - They are only meant to be a harness to collect input/output data and coverage information. - - ### NUL-terminated strings in C ### - If the C function takes string inputs, remember that they must be NUL-terminated. - Not respecting this will cause silent memory corruption and make it impossible to collect meaningful data! - To create a NUL-terminated string in Rust, you can create a `Vec` with the string bytes and a trailing `0`, - and then pass a pointer to its first element. - - ### Non-persistence ### - **All** collection tests must be designed to be run repeatedly without any clean-up, - and they must not rely on any side effects or shared state. - - ### Portable, self-contained tests ### - If the C code relies on pre-existing files on disk (e.g., through hardcoded paths), - you must ensure that all tests **locally** create any required files with the expected content before calling the function under test, - and that they do not rely on any pre-existing state on disk. - If multiple tests reference **exactly** the same file, place a safe Lock around all accesses to that file to - prevent race conditions; `cargo nextest` handles parallel execution by default otherwise. - You **cannot** rely on files on-disk: the goal is for the test file to be moved to some other crate and still work. - - If the C code relies on network access, you must ensure that all tests mock the network interactions locally - and do not rely on any external network state or connectivity. - """ - ) - - coverage_improvement: str = textwrap.dedent( - """ - ### The coverage metric ### - Use **branch coverage** to identify and exercise untested code paths. - - To improve branch coverage, generate interesting combinations of input arguments - and the `stdin` stream, with special attention to edge cases and boundary conditions. - Pay special attention to `libc` functions that may be used in the C code, - and generate inputs that trigger different code paths in them - (e.g. `strlen` with short vs long strings, `fgets` with input shorter vs longer than the buffer size, etc.). - - Ensure the new input values exercise **well-defined** code paths that improve branch coverage. - Exercising UB will be caught and rejected by the sanitizers! - - Aim to achieve branch coverage of at least {target_coverage}%%. - After **three** consecutive attempts where branch coverage has not improved by at least 1 percentage point, - you may stop trying to improve coverage and proceed to the next step. - - Verify that all tests pass and JSON is correctly printed for **all** of them, - including tests on new symbols added for improving coverage. - """ - ) - - analyze_data_collection_tests: str = textwrap.dedent( - """ - ## Step 3 – Analyze the data-collection tests ## - The crate already contains some data-collection tests in `tests/test_collect.rs` - designed to print JSON outputs by running them and capturing their stdout. - - Carefully analyze the `tests/test_collect.rs` file and understand how it imports the FFI symbols, - how it defines the `LibState` struct and the `collect_vector_` tests, and how it prints the JSON output. - - Execute `cargo nextest run --test test_collect --nocapture 2>/dev/null` - to run the tests and see the JSON output they print on the `stdout` channel. - Validate **all** collection tests run successfully and print valid JSON with the expected structure. - - If tests pass **do NOT** modify them in any way at this stage. - If tests exercise UB or trip sanitizers, you must remove them. - If tests fail functionally, attempt to fix them until they pass and print the expected JSON. - """ - ) - - write_test_vectors: str = textwrap.dedent( - """ - ## Step 4 – Save outputs as JSON files ## - Create the directory `{test_vectors_path}`. - - Run each data-collection test **individually** and capture its stdout. - Write the JSON output of each `collect_` test to - `{test_vectors_path}/.json`, where `` is the 1-based index. - - Use the provided `extract_json.py` script to extract the JSON reliably – do NOT rely on grep/sed: - ```bash - cargo nextest run --nocapture -- collect_vector_ --exact 2>/dev/null | \ - uv run extract_json.py > {test_vectors_path}/.json - ``` - - Verify each file is valid JSON by running - `uv run python -m json.tool {test_vectors_path}/.json`. - """ - ) - - write_assert_tests: str = textwrap.dedent( - """ - ## Step 5 – Write assert-style Rust tests ## - Create `{rs_crate_path}/tests/test_assert.rs`. - - Hardcode all expected values as Rust string literals taken from the - JSON files saved in Step 4. - - The file **must** use these imports and the structure below: - ```rust - use assert_cmd::Command; - use predicates::prelude::*; - ``` - - IMPORTANT: You **must** write an assertion test for each collection test, no matter - how many collection tests are there! - Write them one-by-one if there are too many. - - For **each** test case from the JSON, write a `#[test]` function named - `test_case_` following this exact pattern: - ```rust - #[test] - fn test_case_() {{ - let pkg_name_path = assert_cmd::cargo::cargo_bin(assert_cmd::pkg_name!()); - let pkg_name_path_str = pkg_name_path.to_str().unwrap(); - - Command::new("stdbuf") - .args(&["-e0", "-o0", pkg_name_path_str]) - // .args(&["a1", "a2"]) // only if args non-empty - // .write_stdin("data") // only if stdin non-empty - .assert() - .stdout("") - .stderr("") - .code(); - }} - ``` - - Rules: - - Use the **exact** stdout/stderr strings from the JSON, properly escaped - in Rust string literals. - - If expected stderr is empty use `.stderr("")`. - - If stderr contains variable content (PIDs, paths) use - `predicates::str::contains(...)` to capture **path-invariant** contents. - - Once done, run: - ```bash - cargo nextest run --test test_assert --cargo-quiet - ``` - All tests **must** pass and not exercise any undefined behavior. - """ - ) - - simple_exit: str = textwrap.dedent( - """ - Once the task is complete, exit immediately. - Do not over-verify or generate extensive reports. - """ - ) - - @classmethod - def coverage_based(cls) -> str: - return ( - cls.analyze_file - + cls.build_rs - + cls.coverage_script - + cls.write_collect_tests - + cls.coverage_improvement - + cls.write_test_vectors - + cls.write_assert_tests - + cls.simple_exit - ) - - @classmethod - def collect_to_assert(cls) -> str: - return ( - cls.analyze_file - + cls.build_rs - + cls.analyze_data_collection_tests - + cls.write_test_vectors - + cls.write_assert_tests - + cls.simple_exit - ) - - -cs = ConfigStore.instance() -cs.store(name="testgen", node=TestgenConfig) - - -def get_tools(): - useful_tools = UsefulTools() - return [useful_tools.Bash, useful_tools.Read, useful_tools.Edit, useful_tools.Write] - - -@hydra.main(version_base=None, config_name="testgen") -def main(cfg: TestgenConfig) -> None: - try: - _main(cfg) - except Exception as e: - logger.exception(e) - sys.exit(1) - - -def _main(cfg: TestgenConfig) -> None: - output_dir = Path(HydraConfig.get().runtime.output_dir) - - # Separately log the complete trajectory - logger_trajectory = logging.getLogger("ideas.testgen.trajectory") - logger_trajectory.propagate = False - fh = logging.FileHandler(output_dir / f"testgen_trajectory-{int(time.time())}.log") - fh.setFormatter(ConsoleTee.StripANSIFormatter("%(asctime)s %(message)s")) - logger_trajectory.addHandler(fh) - # Simultaneous print and log to file - printer = LoggingConsolePrinter(logger=logger_trajectory) - agent = RelentlessAgent(name="C executable test generator") - - # Generate helper scripts and files in the crate - crate = Crate(output_dir / "Cargo.toml") - nextest_config(crate) - if not cfg.collect_to_assert: - write_coverage_script(crate) - write_collect_script(crate) - write_extract_json_script(crate) - - # Workspace - work_dir = Path(tempfile.mkdtemp()) - workspace_dir = work_dir / "test_crates" - shutil.copytree("test_crates", workspace_dir) - - # Remove all log files - for log_file in workspace_dir.glob("**/*.log"): - log_file.unlink() - - # Paths the agent will populate - rs_crate_path = work_dir / cfg.test_crate_out - test_vectors_path = rs_crate_path / "json" - - # If assertion tests already exist, they must be correct - if (rs_crate_path / "tests" / "test_assert.rs").is_file(): - crate = Crate(rs_crate_path / "Cargo.toml") - ok, output, error, _ = crate.cargo_test("test_assert", quiet=True) - if not ok: - raise RuntimeError( - "Existing assertion tests failed to pass, previous agent did not clean them up!" - ) - logger.info( - f"Assertion tests already exist at {rs_crate_path / 'tests/test_assert.rs'}, skipping agent!" - ) - return - - # Hide instrumentation from conversion agent - if cfg.collect_to_assert: - strip_instrumentation(crate) - - # Build the task prompt - task_description = ( - TestgenInstructions.collect_to_assert() - if cfg.collect_to_assert - else TestgenInstructions.coverage_based() - ) - arguments = { - "c_proj_path": cfg.c_code.parent, - "rs_crate_path": rs_crate_path.relative_to(work_dir), - "test_vectors_path": test_vectors_path.relative_to(work_dir), - "target_coverage": cfg.target_coverage, - } - task_description = task_description.format(**arguments) - - # Run agent in the work directory - os.chdir(work_dir) - try: - agent.run( - model_name=cfg.model, - prompt_template=task_description, - max_steps=100, - max_budget=4, - max_sub_sessions=1, - work_dir=str(work_dir), - tools=get_tools(), - printer=printer, - verbose=True, - ) - except KISSError as e: - logger.warning(f"Agent claims it failed with error: {e}. Clean-up will continue.") - - # Verify that collection tests exist - if not (rs_crate_path / "tests" / "test_collect.rs").is_file(): - raise RuntimeError( - f"Data collection tests were not found at {rs_crate_path / 'tests/test_collect.rs'}!" - ) - - # Strip instrumentation to ensure tests are correct and do not rely on it - crate = Crate(rs_crate_path / "Cargo.toml") - strip_instrumentation(crate) - ok, output, error, _ = crate.cargo_test("test_collect", quiet=True) - if not ok: - raise RuntimeError( - f"Data collection tests failed to pass without instrumentation! Output:\n{output}\nError:\n{error}" - ) - - ok, output, error, _ = crate.cargo_test("test_assert", quiet=True) - if not ok: - logger.error(f"Assertion tests failed to pass! Output:\n{output}\nError:\n{error}") - # Remove incomplete assertion tests, if any - if (rs_crate_path / "tests" / "test_assert.rs").is_file(): - (rs_crate_path / "tests" / "test_assert.rs").unlink() - - # And replace with an always-passing test (nextest does not allow empty test files) - if cfg.guarantee_assert_tests: - logger.warning("Writing dummy test_assert.rs that always passes") - (rs_crate_path / "tests" / "test_assert.rs").write_text(NEXTEST_DUMMY_TEST) - - # Clean the crate and copy it back to the project directory - crate.cargo_clean() - shutil.copytree(rs_crate_path, output_dir, dirs_exist_ok=True) - - -if __name__ == "__main__": - main() diff --git a/src/ideas/agents/utils.py b/src/ideas/agents/utils.py index 4b7f956..de88340 100644 --- a/src/ideas/agents/utils.py +++ b/src/ideas/agents/utils.py @@ -4,68 +4,157 @@ # SPDX-License-Identifier: Apache-2.0 # +import os import textwrap -import tomlkit +import stat from pathlib import Path -from ideas.tools import Crate -from ideas.convert_tests import rustfmt +from ideas.tools import run_subprocess -NEXTEST_DUMMY_TEST = textwrap.dedent( - """ - #[test] - fn dummy_ideas_placeholder() { - assert_eq!(1 + 1, 2); - } - """ -).strip() +def strip_line_directives(path: Path) -> None: + success, output, error, _ = run_subprocess( + ["clang", "--preprocess", "--no-line-commands", str(path)] + ) + if not success: + raise RuntimeError(f"Failed to strip line directives from {path}!{output + error}") -def nextest_config(crate: Crate): - nextest_config_path = crate.cargo_toml.parent / ".config" / "nextest.toml" - nextest_config_path.parent.mkdir(parents=True, exist_ok=True) - nextest_config_contents = { - "profile": { - "default": { - "fail-fast": False, - "slow-timeout": {"period": "30s", "terminate-after": 2}, - } - } - } - nextest_config_path.write_text(tomlkit.dumps(nextest_config_contents)) +def write_profile_list(path: Path, functions: list[str]) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text("".join(f"fun:{name}\n" for name in sorted(set(functions)))) + return path -def write_coverage_script(crate: Crate) -> Path: - coverage_script_path = crate.cargo_toml.parent / "measure_coverage.sh" - coverage_script_contents = textwrap.dedent( - """ - cargo llvm-cov nextest --include-ffi --no-report --test test_collect --no-fail-fast 2>/dev/null - cargo llvm-cov report --include-ffi - cargo llvm-cov report --include-ffi --text +def write_instrumentation_script( + path: Path, features: list[str], profile_list: Path | None = None +) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + # Resolve the list relative to the script so the crate stays relocatable. + profile_list_export = ( + "# Restrict C instrumentation to the program's own functions.\n " + 'script_dir=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)\n ' + f'export CFLAGS="${{CFLAGS:-}} -fprofile-list=$script_dir/' + f'{profile_list.resolve().relative_to(path.parent.resolve())}"\n ' + if profile_list is not None + else "" + ) + contents = textwrap.dedent( + f""" + #!/usr/bin/env bash + ## This script is generated automatically and should not be modified! ## + set -euo pipefail + + export ASAN_OPTIONS="detect_leaks=0" + export UBSAN_OPTIONS="halt_on_error=1:print_stacktrace=1" + export LSAN_OPTIONS= + + # Discard raw profiles from previous runs so coverage reflects only this run's tests. + cargo llvm-cov clean --profraw-only + + test_name="${{1:-}}" + if [ "$test_name" != "collect" ] && [ "$test_name" != "io" ]; then + echo "Usage: $0 " >&2 + exit 2 + fi + + log_dir="coverage_logs" + mkdir -p "$log_dir" + sanitizer_dir="sanitizer_logs" + features="{" ".join(features)}" + sanitizer_status=0 + if [ "$test_name" = "collect" ]; then + rm -f json/*.json + fi + + # Run each sanitizer: print basic diagnostics to stdout and, only on + # failure, save a detailed per-sanitizer log (including stderr diagnostics). + for feature in $features; do + err_file=$(mktemp) + if ! output=$(NEXTEST_EXPERIMENTAL_LIBTEST_JSON=1 cargo nextest run --features "$feature" --test "$test_name" --no-fail-fast --test-threads 1 --message-format libtest-json 2>"$err_file"); then + sanitizer_status=1 + mkdir -p "$sanitizer_dir" + log_file="$sanitizer_dir/$feature.log" + + # Basic diagnostics -> stdout + echo "=== $feature ===" + printf '%s\\n' "$output" | jq -r 'select(.type == "suite" and .event != "started") + | "\\(.passed + .failed) tests run: \\(.passed) passed, \\(.failed) failed"' + printf '%s\\n' "$output" | jq -r 'select(.type == "test" and .event == "failed") + | "FAIL \\(.name)"' + echo " detailed log: $log_file" + + # Detailed diagnostics -> per-sanitizer log file + {{ + echo "=== $feature ===" + printf '%s\\n' "$output" | jq -r 'select(.type == "test" and .event == "failed") + | "FAIL \\(.name)\\n\\(.stdout // "")"' + echo "--- stderr (sanitizer diagnostics) ---" + cat "$err_file" + }} > "$log_file" + fi + rm -f "$err_file" + done + + if [ "$sanitizer_status" -ne 0 ]; then + exit 1 + fi + + echo "All sanitizer checks passed" + {profile_list_export} + cargo llvm-cov nextest --features cc_coverage --include-ffi --no-report --test "$test_name" --no-fail-fast --test-threads 1 > /dev/null 2>&1 + cargo llvm-cov report --include-ffi --text > "$log_dir/coverage_report.log" + + # Coverage summary table -> stdout and log file. + echo "Coverage summary:" + cargo llvm-cov report --include-ffi --summary-only | tee "$log_dir/coverage_summary.log" + + # Emit only uncovered branches (a branch whose True or False count is zero). + echo "Uncovered branches:" + {{ grep -E 'Branch \\(.*(True: 0,|False: 0\\])' "$log_dir/coverage_report.log" || echo " none"; }} | tee "$log_dir/uncovered_branches.log" """ ).strip() - coverage_script_path.write_text(coverage_script_contents) - return coverage_script_path - + path.unlink(missing_ok=True) + path.write_text(contents + "\n") + # Read + execute for the current user + path.chmod(stat.S_IRUSR | stat.S_IXUSR) + return path -def write_collect_script(crate: Crate) -> Path: - collect_path = crate.cargo_toml.parent / "tests" / "test_collect.rs" - collect_path.parent.mkdir(parents=True, exist_ok=True) - if crate.is_bin: +def write_collect_script(path: Path, template: str, lib_name: str | None = None): + path.parent.mkdir(parents=True, exist_ok=True) + if template == "bin": collect_stub = textwrap.dedent( """ + #![allow(unused_imports, dead_code)] use std::os::unix::process::ExitStatusExt; use assert_cmd::Command; + use serde::Serialize; use serde_json; - fn collect_and_print(name: &str, args: &[&str], stdin: Option<&str>) { - let pkg_name_path = assert_cmd::cargo::cargo_bin(assert_cmd::pkg_name!()); - let pkg_name_path_str = pkg_name_path.to_str().unwrap(); + /// A single invocation of the binary and everything it produced. + #[derive(Serialize)] + struct Call { + args: Vec, + stdin: Option, + stdout: String, + stderr: String, + exit_code: i32, + } + + /// One collection test and the calls it made, in order. + #[derive(Serialize)] + struct Case<'a> { + name: &'a str, + calls: &'a [Call], + } + + fn run(args: &[&str], stdin: Option<&str>) -> Call { + let bin_path = assert_cmd::cargo::cargo_bin(assert_cmd::pkg_name!()); + let bin_path_str = bin_path.to_str().unwrap(); let mut cmd = Command::new("stdbuf"); - cmd.args(&["-e0", "-o0", pkg_name_path_str]); + cmd.args(&["-e0", "-o0", bin_path_str]); if !args.is_empty() { cmd.args(args); } @@ -73,40 +162,113 @@ def write_collect_script(crate: Crate) -> Path: cmd.write_stdin(input); } let output = cmd.output().expect("failed to execute process"); - let stdout = String::from_utf8_lossy(&output.stdout); - let stderr = String::from_utf8_lossy(&output.stderr); - let code = output.status.code().unwrap_or(-1); - if output.status.signal() == Some(libc::SIGILL) { - panic!("UBSAN detected during collection!"); + if matches!(output.status.signal(), Some(libc::SIGILL) | Some(libc::SIGABRT)) { + panic!("Sanitizer detected an error during collection!"); + } + Call { + args: args.iter().map(|s| s.to_string()).collect(), + stdin: stdin.map(|s| s.to_string()), + stdout: String::from_utf8_lossy(&output.stdout).into_owned(), + stderr: String::from_utf8_lossy(&output.stderr).into_owned(), + exit_code: output.status.code().unwrap_or(-1), } - println!("{{"); - println!(" \\"name\\": \\"{}\\",", name); - println!(" \\"stdout\\": {},", serde_json::to_string(&*stdout).unwrap()); - println!(" \\"stderr\\": {},", serde_json::to_string(&*stderr).unwrap()); - println!(" \\"exit_code\\": {}", code); - println!("}}"); } + + /// Serialize one collection test's ordered calls to `json/.json`. + fn save_case(name: &str, calls: &[Call]) { + std::fs::create_dir_all("json").unwrap(); + let case = Case { name, calls }; + std::fs::write( + format!("json/{name}.json"), + serde_json::to_string_pretty(&case).unwrap(), + ) + .unwrap(); + } + + // ==== Add collection tests below this line ==== """ ).strip() else: - collect_stub = "" + if lib_name is None: + raise ValueError("lib_name is required for the library collect script") - with collect_path.open("a+", encoding="utf-8") as f: - f.write(collect_stub) - rustfmt(collect_path) - return collect_path + collect_stub = textwrap.dedent( + f""" + #![allow(unused_imports, dead_code)] + use {lib_name}::*; + use serde::Serialize; + use serde_json; + /// Serialize one collection test's input/output state to `json/.json`. + fn save_case(name: &str, case: &T) {{ + std::fs::create_dir_all("json").unwrap(); + let path = format!("json/{{name}}.json"); + std::fs::write(path, serde_json::to_string_pretty(case).unwrap()).unwrap(); + }} -def write_extract_json_script(crate: Crate) -> Path: - extract_json_path = crate.cargo_toml.parent / "extract_json.py" - extract_json_contents = textwrap.dedent( - """ - import sys, json - buf = sys.stdin.read() - decoder = json.JSONDecoder() - obj, _ = decoder.raw_decode(buf, buf.index('{')) - print(json.dumps(obj, indent=2)) - """ - ).strip() - extract_json_path.write_text(extract_json_contents) - return extract_json_path + // ==== Add collection tests below this line ==== + """ + ).strip() + path.write_text(collect_stub + "\n") + + +def write_assert_script(path: Path, template: str, lib_name: str | None = None) -> Path: + path.parent.mkdir(parents=True, exist_ok=True) + if template == "bin": + assert_stub = textwrap.dedent( + """ + #![allow(unused_imports, dead_code)] + use std::os::unix::process::ExitStatusExt; + use assert_cmd::Command; + use predicates::prelude::*; + + /// What a single invocation of the binary produced. + struct Call { + stdout: String, + stderr: String, + exit_code: i32, + } + + fn run(args: &[&str], stdin: Option<&str>) -> Call { + let bin_path = assert_cmd::cargo::cargo_bin(assert_cmd::pkg_name!()); + let bin_path_str = bin_path.to_str().unwrap(); + + let mut cmd = Command::new("stdbuf"); + cmd.args(&["-e0", "-o0", bin_path_str]); + if !args.is_empty() { + cmd.args(args); + } + if let Some(input) = stdin { + cmd.write_stdin(input); + } + let output = cmd.output().expect("failed to execute process"); + if matches!(output.status.signal(), Some(libc::SIGILL) | Some(libc::SIGABRT)) { + panic!("Sanitizer detected an error while running the binary!"); + } + Call { + stdout: String::from_utf8_lossy(&output.stdout).into_owned(), + stderr: String::from_utf8_lossy(&output.stderr).into_owned(), + exit_code: output.status.code().unwrap_or(-1), + } + } + + // ==== Add assertion tests below this line ==== + """ + ).strip() + else: + if lib_name is None: + raise ValueError("lib_name is required for the library assert script") + + assert_stub = textwrap.dedent( + f""" + #![allow(unused_imports, dead_code)] + use {lib_name}::*; + + // ==== Add assertion tests below this line ==== + """ + ).strip() + path.write_text(assert_stub + "\n") + return path + + +RESTRICT_COVERAGE = os.environ.get("RESTRICT_COVERAGE", "0") not in ("0", "", "false", "False") diff --git a/src/ideas/ast.py b/src/ideas/ast.py index 75e7655..3488b8c 100644 --- a/src/ideas/ast.py +++ b/src/ideas/ast.py @@ -6,13 +6,14 @@ import logging from pathlib import Path +from functools import cmp_to_key from collections import defaultdict from collections.abc import Iterable -from dataclasses import dataclass, field, fields, replace +from dataclasses import astuple, dataclass, field, fields, replace from typing import get_args from clang.cindex import TranslationUnit, TranslationUnitLoadError, Diagnostic -from clang.cindex import Cursor, CursorKind, SourceRange, TokenKind +from clang.cindex import Cursor, CursorKind, SourceRange, TokenKind, Type, TypeKind from clang.cindex import PrintingPolicy, PrintingPolicyProperty, LinkageKind, StorageClass from clang.cindex import conf, SourceLocation, _CXString from ctypes import byref, pointer, c_size_t, c_char_p, c_uint @@ -23,6 +24,109 @@ FILENAME = "file.c" CodeC = Code["c"] +# Cursor kinds that become symbols, ranked by order in which they should be translated +_KIND_RANK = { + CursorKind.ENUM_DECL: 0, + CursorKind.ENUM_CONSTANT_DECL: 0, + CursorKind.STRUCT_DECL: 1, + CursorKind.UNION_DECL: 1, + CursorKind.TYPEDEF_DECL: 1, + CursorKind.VAR_DECL: 2, + CursorKind.FUNCTION_DECL: 3, +} + + +@dataclass(frozen=True) +class TypeShape: + # Declared hardest construct first, because `rank` reads the fields off in order. A + # `void *` leads: it cannot be given a meaningful Rust type until whatever it is cast + # to has been translated, and static analysis cannot recover that edge, so the more + # erased of two types sorts last. + void_pointers: int = 0 + function_pointers: int = 0 + unions: int = 0 # untagged unions need manual discrimination + pointers: int = 0 + arrays: int = 0 # arrays and flexible array members + fields: int = 0 + + @property + def rank(self) -> tuple[int, ...]: + # Compared lexicographically, so the presence of a harder construct outweighs any + # amount of an easier one and no weights have to be invented to say which is worse + return astuple(self) + + # Deliberately left unannotated so that `dataclass` does not treat these as fields + CURSOR_KINDS = ( + CursorKind.STRUCT_DECL, + CursorKind.UNION_DECL, + CursorKind.TYPEDEF_DECL, + CursorKind.ENUM_DECL, + ) + _ARRAY_KINDS = ( + TypeKind.CONSTANTARRAY, + TypeKind.INCOMPLETEARRAY, + TypeKind.VARIABLEARRAY, + TypeKind.DEPENDENTSIZEDARRAY, + ) + _FUNCTION_KINDS = (TypeKind.FUNCTIONPROTO, TypeKind.FUNCTIONNOPROTO) + + @classmethod + def _ultimate_pointee(cls, c_type: Type) -> Type | None: + canonical = c_type.get_canonical() + while canonical.kind in cls._ARRAY_KINDS: + canonical = canonical.get_array_element_type().get_canonical() + if canonical.kind != TypeKind.POINTER: + return None + while canonical.kind == TypeKind.POINTER: + canonical = canonical.get_pointee().get_canonical() + return canonical + + @classmethod + def from_cursor(cls, cursor: Cursor) -> "TypeShape": + if cursor.kind not in cls.CURSOR_KINDS: + return cls() + + num_fields = num_void_ptrs = num_fn_ptrs = num_ptrs = num_arrays = num_unions = 0 + + # An inline anonymous record is presented both as a sibling declaration and as a + # child of the field that uses it, so nodes must only be counted once. + seen: set[tuple[str, str, str, str, int, int, int, int]] = set() + + for node in cursor.walk_preorder(): + if node.kind not in (CursorKind.UNION_DECL, CursorKind.FIELD_DECL): + continue + key = _cursor_key(node) + if key in seen: + continue + seen.add(key) + + if node.kind == CursorKind.UNION_DECL: + num_unions += 1 + continue + + num_fields += 1 + if node.type.get_canonical().kind in cls._ARRAY_KINDS: + num_arrays += 1 + + pointee = cls._ultimate_pointee(node.type) + if pointee is None: + continue + if pointee.kind == TypeKind.VOID: + num_void_ptrs += 1 + elif pointee.kind in cls._FUNCTION_KINDS: + num_fn_ptrs += 1 + else: + num_ptrs += 1 + + return cls( + void_pointers=num_void_ptrs, + function_pointers=num_fn_ptrs, + unions=num_unions, + pointers=num_ptrs, + arrays=num_arrays, + fields=num_fields, + ) + @dataclass(frozen=True) class Symbol: @@ -38,8 +142,6 @@ class Symbol: # Symbol semantics is_definition: bool - is_variable: bool - is_function: bool is_global: bool is_system: bool is_top_level: bool @@ -52,6 +154,15 @@ class Symbol: line_directive: CodeC | None declaration_line_directive: CodeC | None + # Structural summary of the type, empty for symbols that are not types + type_shape: TypeShape = field(default_factory=TypeShape) + + @property + def difficulty(self) -> tuple[int, ...]: + # A sort key that puts the easiest symbol first: the symbol kind decides the + # ordering, and how complicated its type is only breaks ties within a kind. + return (_KIND_RANK[self.kind], *self.type_shape.rank) + @classmethod def from_cursor( cls, @@ -74,8 +185,6 @@ def from_cursor( declaration=get_cursor_code(decl, pretty_print=True) if decl else None, code=code, is_definition=cursor.is_definition(), - is_variable=cursor.kind == CursorKind.VAR_DECL, - is_function=cursor.kind == CursorKind.FUNCTION_DECL, is_global=cursor.linkage == LinkageKind.EXTERNAL, is_system=cursor.location.is_in_system_header, tu_path=Path(cursor.translation_unit.spelling).resolve(), @@ -85,8 +194,29 @@ def from_cursor( declaration_line_directive=_line_directive_for(decl), storage_class=cursor.storage_class, is_top_level=parent is None, + type_shape=TypeShape.from_cursor(parent_or_cursor), + ) + + @property + def is_variable(self) -> bool: + return self.kind == CursorKind.VAR_DECL + + @property + def is_function(self) -> bool: + return self.kind == CursorKind.FUNCTION_DECL + + @property + def is_type(self) -> bool: + return self.kind in ( + CursorKind.STRUCT_DECL, + CursorKind.UNION_DECL, + CursorKind.TYPEDEF_DECL, ) + @property + def is_struct(self) -> bool: + return self.kind == CursorKind.STRUCT_DECL + def with_declaration(self, decl_symbol: "Symbol") -> "Symbol": return replace( self, @@ -117,6 +247,7 @@ class TreeResult: complete_graph: dict[str, list[str]] = field(default_factory=lambda: defaultdict(list)) filename: str | None = None arguments: list[str] | None = None + local_names: frozenset[str] = field(default_factory=frozenset) def _synthesize_llm_context_declaration(cursor: Cursor, fallback_code: CodeC) -> str: @@ -226,17 +357,14 @@ def extract_info_c(tu: TranslationUnit) -> TreeResult: ) graph = { name: extract_referenced_symbols(reference_nodes[name], symbols.keys()) - for name, symbol in symbols.items() + for name in symbols } - return TreeResult(symbols=symbols, complete_graph=graph) - - -def extract_symbol_info_c(node: Cursor, parent: Cursor | None = None) -> dict[str, Symbol]: - tu_preorder_index_map = _cursor_order_map(node) - symbols, _ = _extract_symbol_info_c( - node, parent=parent, tu_preorder_index_map=tu_preorder_index_map + local_names = frozenset( + cursor.spelling + for cursor in tu.cursor.walk_preorder() + if cursor.spelling and not cursor.location.is_in_system_header ) - return symbols + return TreeResult(symbols=symbols, complete_graph=graph, local_names=local_names) def _extract_symbol_info_c( @@ -255,16 +383,7 @@ def _extract_symbol_info_c( # Add declarative nodes to symbols usr = node.get_usr() - # FIXME: Use node.kind.is_declaration()? - if node.kind in ( - CursorKind.STRUCT_DECL, - CursorKind.UNION_DECL, - CursorKind.ENUM_DECL, - CursorKind.ENUM_CONSTANT_DECL, - CursorKind.FUNCTION_DECL, - CursorKind.VAR_DECL, - CursorKind.TYPEDEF_DECL, - ): + if node.kind in _KIND_RANK: symbols[usr] = Symbol.from_cursor( usr, node, @@ -379,12 +498,12 @@ def get_cursor_code(cursor: Cursor, pretty_print: bool = False) -> CodeC: return code -def clang_rename_( - tu: TranslationUnit, renames: dict[str, str], sources: dict[Path, bytes] | None = None -): - logger.info( - f"Renaming {len(renames)} symbols in {tu.spelling}: {', '.join(renames.keys())}" - ) +def clang_rename( + tu: TranslationUnit, renames: dict[str, str] +) -> dict[Path, dict[tuple[int, int], bytes]]: + renames_str = "\n ".join([f"{k} => {v}" for k, v in renames.items()]) + logger.info(f"Renaming {len(renames)} symbols in {tu.spelling}:\n {renames_str}") + # Group edits by file path and source offsets because cursor traversal may revisit tokens. edits_by_file: dict[Path, dict[tuple[int, int], bytes]] = {} assert tu.cursor is not None @@ -412,11 +531,7 @@ def clang_rename_( extent = (token.extent.start.offset, token.extent.end.offset) edits_by_file.setdefault(file_path, {})[extent] = renames[target_usr].encode() - # Apply edits for each file and optionally save the pre-edit source snapshot. - for file_path, edits in edits_by_file.items(): - if sources is not None and file_path not in sources: - sources[file_path] = file_path.read_bytes() - _apply_edits(file_path, edits) + return edits_by_file DEFINITION_START_TOKEN = {CursorKind.FUNCTION_DECL: "{", CursorKind.VAR_DECL: "="} @@ -728,3 +843,110 @@ def mangle(name: str) -> str: name = "_" + name return name + + +SymbolName = str +SymbolGroup = tuple[SymbolName, ...] + + +def create_symbol_lexical_key_fn( + symbols: dict[SymbolName, Symbol], + ast_order: dict[Path, TreeResult] | None = None, +): + def compare_symbol_lexical(a: SymbolName | SymbolGroup, b: SymbolName | SymbolGroup) -> int: + # Support symbol groups by using the first symbol in the group. + a_name = a[0] if isinstance(a, tuple) else a + b_name = b[0] if isinstance(b, tuple) else b + + a_symbol = symbols[a_name] + b_symbol = symbols[b_name] + + a_tu = a_symbol.tu_path + b_tu = b_symbol.tu_path + + # If symbols are from the same translation unit, compare their + # preorder traversal indices for lexical ordering. + if a_tu == b_tu: + return _cmp_symbol_tu_order(a_symbol, b_symbol) + + if ast_order is None: + raise RuntimeError( + f"Cannot compare symbols from different translation units without ast_order: {a} ({a_tu}) vs {b} ({b_tu})." + ) + + # If a's USR appears in b's TU with matching code, both symbols are + # present in b_tu and can be compared by TU preorder index there. + b_ast = ast_order.get(b_tu) + if ( + b_ast is not None + and a_name in b_ast.symbols + and b_ast.symbols[a_name].code == a_symbol.code + ): + return _cmp_symbol_tu_order(b_ast.symbols[a_name], b_symbol) + + # If b's USR appears in a's TU with matching code, both symbols are + # present in a_tu and can be compared by TU preorder index there. + a_ast = ast_order.get(a_tu) + if ( + a_ast is not None + and b_name in a_ast.symbols + and a_ast.symbols[b_name].code == b_symbol.code + ): + return _cmp_symbol_tu_order(a_symbol, a_ast.symbols[b_name]) + + # The symbol's USR is not shared across TUs, so fall back to ordering + # by the position of each symbol's TU in ast_order (source priority) + ast_rank = {path: i for i, path in enumerate(ast_order)} + try: + a_rank = ast_rank[a_tu] + b_rank = ast_rank[b_tu] + except KeyError as ex: + raise RuntimeError( + f"Cannot compare symbols because one or both translation units are missing from ast_order: {a_tu}, {b_tu}." + ) from ex + + if a_rank < b_rank: + return -1 + if a_rank > b_rank: + return 1 + raise RuntimeError("Distinct translation units cannot have identical ranks!") + + return cmp_to_key(compare_symbol_lexical) + + +def create_symbol_ordering_key_fn( + symbols: dict[SymbolName, Symbol], + ast_order: dict[Path, TreeResult] | None = None, +): + # Order symbols by translation difficulty, falling back to lexical source order. This + # only breaks ties between symbols that are incomparable in the dependency graph, so + # the difficulty preference is one the topological constraint silently overrides. + lexical_key = create_symbol_lexical_key_fn(symbols, ast_order) + + def symbol_ordering_key(node: SymbolName | SymbolGroup): + # Rank a group by its hardest member, then break any remaining tie lexically + names = node if isinstance(node, tuple) else (node,) + return max(symbols[name].difficulty for name in names), lexical_key(node) + + return symbol_ordering_key + + +def _cmp_symbol_tu_order(symbol_a: Symbol, symbol_b: Symbol) -> int: + if symbol_a.tu_path != symbol_b.tu_path: + raise ValueError( + "Cannot compare TU preorder indices for symbols from different translation units:" + f" {symbol_a.name} @ {symbol_a.tu_path} vs {symbol_b.name} @ {symbol_b.tu_path}" + ) + + order_a = symbol_a.tu_preorder_index + order_b = symbol_b.tu_preorder_index + if order_a < order_b: + return -1 + if order_b < order_a: + return 1 + if symbol_a.name != symbol_b.name: + raise ValueError( + f"Unable to order distinct symbols with identical lexical priority and location:" + f" {symbol_a.name} @ {order_a} vs {symbol_b.name} @ {order_b}" + ) + return 0 diff --git a/src/ideas/ast_rust.py b/src/ideas/ast_rust.py index 7467ffb..dad7691 100644 --- a/src/ideas/ast_rust.py +++ b/src/ideas/ast_rust.py @@ -107,6 +107,10 @@ def validate_changes(code: CodeRust, template: CodeRust) -> OrderedDict[str, str template_nodes = get_nodes(template_root) allowed_change_nodes = get_macro_nodes(template_root, "unimplemented") + # If the template has no unimplemented!() markers there are no scope constraints + if not allowed_change_nodes: + return OrderedDict() + scope_feedback = OrderedDict() # Check for top-level changes @@ -172,9 +176,12 @@ def mangle(name: str) -> str: return name -def _rust_node_signature(node: Node, source: bytes) -> str | None: +def _rust_node_signature(node: Node, source: bytes, delete: bool = False) -> str | None: ntype = node.type if ntype in ("function_item", "function_signature_item"): + if delete: + return None + # Find the block body and remove it body = node.child_by_field_name("body") if body: @@ -186,7 +193,7 @@ def _rust_node_signature(node: Node, source: bytes) -> str | None: return source[node.start_byte : node.end_byte].decode() -def get_signatures(code: CodeRust) -> CodeRust: +def strip_fns(code: CodeRust, delete: bool = False) -> CodeRust: if not str(code).strip(): return code @@ -195,8 +202,8 @@ def get_signatures(code: CodeRust) -> CodeRust: parts: list[str] = [] for node in root.children: - sig = _rust_node_signature(node, source) - if sig: + sig = _rust_node_signature(node, source, delete=delete) + if sig is not None: parts.append(sig) return CodeRust("\n".join(parts)) if parts else CodeRust("") diff --git a/src/ideas/bear.py b/src/ideas/bear.py new file mode 100644 index 0000000..48665a0 --- /dev/null +++ b/src/ideas/bear.py @@ -0,0 +1,362 @@ +# +# Copyright (C) 2026 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +import sys +import re +import json +import shlex +import logging +import argparse +from pathlib import Path +from dataclasses import dataclass +from functools import cached_property + +from .tools import run_subprocess + +logger = logging.getLogger("ideas.bear") + +TargetName = str # normalized target filename, e.g. "libfoo.so" or "mybin" + + +@dataclass +class CompileCommand: + source: Path # resolved absolute source path + output: Path # resolved absolute .o path + arguments: list[str] + working_dir: Path + + def to_dict(self) -> dict: + return { + "file": str(self.source), + "arguments": self.arguments, + "directory": str(self.working_dir), + "output": str(self.output), + } + + @staticmethod + def _is_compile_command(arguments: list[str]) -> bool: + if not arguments or "-c" not in arguments: + return False + executable = Path(arguments[0]).name + return any(executable == c or executable.startswith(c + "-") for c in _COMPILERS) + + @classmethod + def from_arguments(cls, arguments: list[str], working_dir: Path) -> "CompileCommand | None": + if not cls._is_compile_command(arguments): + return None + obj_path = source_path = None + filtered: list[str] = [] + i = 0 + while i < len(arguments): + arg = arguments[i] + if arg in _DEP_FLAGS_WITH_ARG: + i += 2 # skip flag and its argument + elif arg in _DEP_FLAGS: + i += 1 # skip standalone flag + else: + filtered.append(arg) + if arg == "-o" and i + 1 < len(arguments): + obj_path = (working_dir / Path(arguments[i + 1])).resolve() + elif arg == "-c" and i + 1 < len(arguments): + p = Path(arguments[i + 1]) + if p.suffix in _SOURCE_EXTS: + source_path = (working_dir / p).resolve() + i += 1 + if obj_path and source_path: + return cls( + source=source_path, output=obj_path, arguments=filtered, working_dir=working_dir + ) + return None + + +@dataclass +class LinkCommand: + arguments: list[str] # original argv from the linker invocation + working_dir: Path + + @staticmethod + def _is_link_command(arguments: list[str]) -> bool: + if not arguments: + return False + executable_basename = Path(arguments[0]).name + # Match exact names and versioned variants e.g. clang-21, gcc-13 + if not any( + executable_basename == d or executable_basename.startswith(d + "-") + for d in _LINKERS + ): + return False + # Must not be a compile-only step + if "-c" in arguments: + return False + # Must have at least one object file input to distinguish from non-linker calls + return any(arg.endswith(".o") for arg in arguments[1:]) + + @classmethod + def from_arguments(cls, arguments: list[str], working_dir: Path) -> "LinkCommand | None": + if not cls._is_link_command(arguments): + return None + cmd = cls(arguments=arguments, working_dir=working_dir) + if cmd.target is None: + logger.warning("Linker invocation has no -o flag: %s", arguments[:6]) + return None + return cmd + + @cached_property + def target(self) -> TargetName | None: + for i, arg in enumerate(self.arguments): + if arg == "-o" and i + 1 < len(self.arguments): + name = Path(self.arguments[i + 1]).name + # Strip version suffix from shared libraries: libfoo.so.1.2.3 -> libfoo.so + return re.sub(r"\.so(\.\d+)+$", ".so", name) + return None + + @cached_property + def output_path(self) -> Path | None: + for i, arg in enumerate(self.arguments): + if arg == "-o" and i + 1 < len(self.arguments): + p = Path(self.arguments[i + 1]) + return (p if p.is_absolute() else self.working_dir / p).resolve() + return None + + def _input_args(self) -> list[str]: + result = [] + it = iter(self.arguments[1:]) # skip argv[0] + for arg in it: + if arg == "-o": + next(it, None) # consume and discard the output path + elif not arg.startswith("-"): + result.append(arg) + return result + + @cached_property + def object_files(self) -> list[Path]: + objects: list[Path] = [] + for arg in self._input_args(): + p = Path(arg) + if p.suffix == ".o": + resolved = p if p.is_absolute() else self.working_dir / p + objects.append(resolved.resolve()) + return objects + + @cached_property + def linked_binary_inputs(self) -> list[Path]: + binaries: list[Path] = [] + for arg in self._input_args(): + p = Path(arg) + if (".so" in p.name and p.suffix != ".o") or p.suffix == ".a": + resolved = p if p.is_absolute() else self.working_dir / p + binaries.append(resolved.resolve()) + return binaries + + @cached_property + def link_libs(self) -> list[str]: + return [arg[2:] for arg in self.arguments if arg.startswith("-l") and arg[2:]] + + +_SOURCE_EXTS = {".c", ".cpp", ".cxx", ".cc", ".C", ".s", ".S", ".m"} +_COMPILERS = {"gcc", "clang", "cc", "g++", "c++", "clang++"} +_LINKERS = { + "gcc", + "clang", + "cc", + "g++", + "c++", + "clang++", + "ld", + "ld.bfd", + "ld.lld", + "ld.gold", + "lld", +} +# CMake injects these dependency-tracking flags into every compile command. +# They cause libclang to try writing .d files at relative paths that don't +# exist during analysis, which fails TranslationUnit parsing. +_DEP_FLAGS = frozenset({"-MD", "-MMD", "-MP", "-MG"}) +_DEP_FLAGS_WITH_ARG = frozenset({"-MF", "-MT", "-MQ"}) + + +@dataclass +class TargetOutputs: + entries: list[CompileCommand] + link_libs: list[str] + + +@dataclass +class BuildDatabase: + compile_commands: list[CompileCommand] + link_commands: list[LinkCommand] + + @classmethod + def from_events(cls, events_path: Path) -> "BuildDatabase": + compile_commands: list[CompileCommand] = [] + link_commands: list[LinkCommand] = [] + + with events_path.open() as f: + for line in f: + line = line.strip() + if not line: + continue + try: + event = json.loads(line) + except json.JSONDecodeError: + logger.warning("Skipping malformed events.jsonl line: %s", line[:80]) + continue + + arguments = event.get("arguments", []) + working_dir = Path(event.get("working_dir", ".")) + + if cmd := LinkCommand.from_arguments(arguments, working_dir=working_dir): + link_commands.append(cmd) + elif cmd := CompileCommand.from_arguments(arguments, working_dir=working_dir): + compile_commands.append(cmd) + else: + logger.debug("Skipping event: %s", arguments[:3]) + + return cls(compile_commands=compile_commands, link_commands=link_commands) + + def _collect_transitive_entries( + self, + link_cmd: "LinkCommand", + obj_map: dict[Path, CompileCommand], + binary_source_map: dict[Path, list[Path]], + ) -> list[CompileCommand]: + entries: list[CompileCommand] = [] + seen_sources: set[Path] = set() + seen_binaries: set[Path] = set() + + def _collect(binary: Path) -> None: + if binary in seen_binaries: + return + seen_binaries.add(binary) + for inp in binary_source_map.get(binary, []): + if (entry := obj_map.get(inp)) is not None: + # inp is a .o — add its compilation entry (dedup by source path) + if entry.source not in seen_sources: + seen_sources.add(entry.source) + entries.append(entry) + elif inp in binary_source_map: + # inp is a .so/.a built in this project — recurse + _collect(inp) + elif inp.suffix == ".o": + # No compile command for this object file. System CRT objects + # (crti.o, crtbeginS.o, etc.) under /usr are expected — log at + # debug. Any other gap is unexpected and logged as a warning. + if inp.is_relative_to(Path("/usr")): + logger.debug( + "Skipping system object file: %s (linked into %s)", + inp, + link_cmd.target, + ) + else: + logger.warning( + "No compile command found for object file: %s (linked into %s)", + inp, + link_cmd.target, + ) + else: + # External .so/.a not built in this project — skip but log so + # the user can verify it is intentionally external. + logger.debug("Skipping external binary input: %s", inp) + + if link_cmd.output_path: + _collect(link_cmd.output_path) + return entries + + def resolve_targets(self) -> dict[TargetName, TargetOutputs]: + obj_map: dict[Path, CompileCommand] = {cmd.output: cmd for cmd in self.compile_commands} + binary_source_map: dict[Path, list[Path]] = {} + for lc in self.link_commands: + if lc.output_path is None: + continue + inputs = lc.object_files + lc.linked_binary_inputs + binary_source_map[lc.output_path] = inputs + # Also index by the unversioned name (libfoo.so.1.2.3 → libfoo.so) so that + # consumers referencing the symlink name are resolved transitively. + normalized = lc.output_path.parent / re.sub( + r"\.so(\.\d+)+$", ".so", lc.output_path.name + ) + if normalized != lc.output_path: + binary_source_map[normalized] = inputs + result: dict[TargetName, TargetOutputs] = {} + for link_cmd in self.link_commands: + assert link_cmd.target is not None + entries = self._collect_transitive_entries(link_cmd, obj_map, binary_source_map) + result[link_cmd.target] = TargetOutputs( + entries=entries, link_libs=link_cmd.link_libs + ) + return result + + def write_outputs(self, output_dir: Path) -> None: + for target, outputs in self.resolve_targets().items(): + # CompilationDatabase.fromDirectory() requires the file to be named compile_commands.json + # inside the directory passed to it. Use a .d/ suffix to avoid colliding with + # the build artifact (e.g. libcjson.so already exists as a file in the build dir). + target_dir = output_dir / f"{target}.d" + target_dir.mkdir(exist_ok=True) + (target_dir / "compile_commands.json").write_text( + json.dumps([e.to_dict() for e in outputs.entries], indent=2) + ) + link_entries: list[dict] = [{"source": str(e.source)} for e in outputs.entries] + link_entries += [{"lib": lib} for lib in outputs.link_libs] + (target_dir / "links.json").write_text( + json.dumps({"entries": link_entries}, indent=2) + ) + + +def _main(build_command: list[str], output_dir: Path): + output_dir.mkdir(parents=True, exist_ok=True) + + # Run the build under bear intercept to capture all compiler and linker invocations. + # events.jsonl contains every execve call: compilations, links, and system tools. + events_path = output_dir / "events.jsonl" + intercept_cmd = ["bear", "intercept", "--output", str(events_path), "--"] + build_command + success, output, error, _ = run_subprocess(intercept_cmd) + if not success: + raise RuntimeError(f"{shlex.join(intercept_cmd)} failed:\n{output + error}") + if not events_path.exists(): + raise RuntimeError(f"bear intercept succeeded but {events_path} was not written") + + # Parse events.jsonl once and write per-target outputs. + db = BuildDatabase.from_events(events_path) + db.write_outputs(output_dir) + + +def main(): + logging.basicConfig(level=logging.INFO) + + parser = argparse.ArgumentParser( + prog="ideas.bear", + description="Runs bear intercept and extracts per-target compile_commands + links files.", + epilog="example: python -m ideas.bear --output-dir build-ninja -- cmake --build build-ninja --target all", + ) + parser.add_argument( + "--output-dir", + type=Path, + default=Path("."), + metavar="DIR", + help="Directory to write output files (default: current directory)", + ) + parser.add_argument( + "build_command", + nargs=argparse.REMAINDER, + help="Build command to pass to bear", + ) + args = parser.parse_args() + # build_command will include "--" so strip it if present + if args.build_command and args.build_command[0] == "--": + args.build_command.pop(0) + if not args.build_command: + parser.error("A build command must be provided") + + try: + _main(args.build_command, args.output_dir) + except Exception as e: + logger.exception(e) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/src/ideas/cmake.py b/src/ideas/cmake.py deleted file mode 100644 index 6cdbe16..0000000 --- a/src/ideas/cmake.py +++ /dev/null @@ -1,150 +0,0 @@ -# -# Copyright (C) 2026 Intel Corporation -# -# SPDX-License-Identifier: Apache-2.0 -# - -import sys -import os -import json -import logging -import shutil - -from dataclasses import dataclass -from pathlib import Path - -import hydra -from omegaconf import MISSING -from hydra.core.config_store import ConfigStore - -from .tools import run_subprocess, LARGE_PROJECT - -logger = logging.getLogger("ideas.cmake") - - -@dataclass -class CmakeConfig: - source_dir: Path = MISSING - build_dir: Path = MISSING - - -cs = ConfigStore.instance() -cs.store(name="cmake", node=CmakeConfig) - - -def _normalize_isystem(compile_commands_path: Path) -> None: - """Replace -isystem with -I in compile_commands.json""" - if not compile_commands_path.exists(): - return - db = json.loads(compile_commands_path.read_text()) - for entry in db: - if "command" in entry: - entry["command"] = entry["command"].replace("-isystem", "-I") - - if "arguments" in entry: - entry["arguments"] = [ - "-I" + arg[len("-isystem") :] if arg.startswith("-isystem") else arg - for arg in entry["arguments"] - ] - compile_commands_path.write_text(json.dumps(db, indent=2)) - - -def configure( - source_dir: Path, - build_dir: Path, - preset: str | None = None, -) -> None: - # Clean existing build directory - shutil.rmtree(build_dir, ignore_errors=True) - - flags = [ - "-DCMAKE_EXPORT_COMPILE_COMMANDS=ON", - "-DCMAKE_C_COMPILER=clang", - ] - if extract_info_cmake := os.environ.get("EXTRACT_INFO_CMAKE"): - flags.append(f"-DCMAKE_PROJECT_TOP_LEVEL_INCLUDES={extract_info_cmake}") - if cflags := os.environ.get("CFLAGS"): - flags.append(f"-DCMAKE_C_FLAGS={cflags}") - - if not preset: - cmd = ["cmake", "-S", str(source_dir), "-B", str(build_dir), "-G", "Ninja"] + flags - else: - cmd = [ - "cmake", - "-S", - ".", - "--preset", - preset, - "-B", - str(build_dir), - "-G", - "Ninja", - ] + flags - - success, output, error, _ = run_subprocess(cmd) - if not success: - raise RuntimeError(f"CMake configuration failed:{' '.join(cmd)}\n{output + error}") - - # Replace -isystem with -I in compile_commands.json so that all project - # headers get consistent USRs regardless of CMake SYSTEM keyword usage. - if LARGE_PROJECT: - _normalize_isystem(build_dir / "compile_commands.json") - - -def build(build_dir: Path, preset: str | None = None) -> None: - if not preset: - cmd = ["cmake", "--build", str(build_dir), "--target", "all"] - else: - cmd = ["cmake", "--build", str(build_dir), "--target", "all", "--preset", preset] - - build_log_path = build_dir / "build.log" - success, output, error, _ = run_subprocess(cmd) - if not success: - with open(build_log_path, "w") as log_file: - log_file.write(output + error) - raise RuntimeError(f"CMake build failed: {' '.join(cmd)}\n{output + error}") - - -def patch_preset_binary_dir(preset_path: Path, build_dir: Path) -> None: - """Ensure binaryDir and generator in all configure presets are set to ninja.""" - data = json.loads(preset_path.read_text()) - for preset in data.get("configurePresets", []): - if "binaryDir" in preset and preset["binaryDir"] != str(build_dir): - preset["binaryDir"] = str(build_dir) - - if "generator" in preset and preset["generator"] != "Ninja": - preset["generator"] = "Ninja" - - preset_path.write_text(json.dumps(data, indent=2)) - - -def _main(cfg: CmakeConfig) -> None: - # Determine Cmake preset - preset = "test" if os.path.exists("CMakePresets.json") else None - - # Patch binaryDir in presets to match our expected build directory - if preset: - patch_preset_binary_dir(Path("CMakePresets.json"), cfg.build_dir) - - # Configure Cmake - configure( - source_dir=cfg.source_dir, - build_dir=cfg.build_dir, - preset=preset, - ) - - # Build with Cmake - build(cfg.build_dir, preset) - - -@hydra.main(version_base=None, config_name="cmake") -def main(cfg: CmakeConfig) -> None: - try: - _main(cfg) - except Exception as e: - logger.exception(e) - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/src/ideas/consolidate.py b/src/ideas/consolidate.py new file mode 100644 index 0000000..152c41a --- /dev/null +++ b/src/ideas/consolidate.py @@ -0,0 +1,643 @@ +# +# Copyright (C) 2026 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +import os +import sys +import json +import logging +import textwrap +from pathlib import Path +from dataclasses import dataclass +from graphlib import TopologicalSorter, CycleError +from concurrent.futures import ProcessPoolExecutor + +import hydra +import networkx as nx +from omegaconf import MISSING +from hydra.core.config_store import ConfigStore +from clang.cindex import CompilationDatabase, CompileCommand, TranslationUnit, CursorKind +from clang.cindex import TranslationUnitLoadError, Diagnostic, StorageClass + +from ideas.tools import run_subprocess, Crate +from ideas.ast_rust import mangle as mangle_rs +from ideas.ast import extract_info_c, TreeResult, Symbol, clang_rename, mangle, CodeC +from ideas.ast import SymbolName, SymbolGroup, create_symbol_lexical_key_fn +from ideas.ast import create_symbol_ordering_key_fn + +logger = logging.getLogger("ideas.consolidate") + +SymbolSpelling = str + + +@dataclass +class ConsolidateConfig: + cargo_toml: Path = MISSING + vcs: str = "none" + template: str = "bin" + + compile_commands: Path = MISSING + links: Path = MISSING + include_line_directives: bool = True + + +cs = ConfigStore.instance() +cs.store(name="consolidate", node=ConsolidateConfig) + + +def analyze( + compile_commands: Path, source_priority: list[Path] +) -> tuple[dict[SymbolName, Symbol], list[SymbolGroup]]: + # Get symbol table and dependencies taking into account source priority + asts = get_asts(compile_commands, source_priority) + ast_order = create_ast_order(source_priority, asts) + symbols, dependencies = get_symbols_and_dependencies( + asts, ast_order=ast_order, filter_system_symbols=False + ) + + # Sort symbols in topological order, ordering types before functions and simple types + # before complex ones so the consolidated source matches the translation order. + symbol_ordering_key = create_symbol_ordering_key_fn(symbols, ast_order) + sorted_symbol_groups: list[SymbolGroup] = list( + nx.lexicographical_topological_sort( + nx.from_dict_of_lists(dependencies, create_using=nx.DiGraph).reverse(copy=False), # type: ignore + key=symbol_ordering_key, + ) + ) + return symbols, sorted_symbol_groups + + +def is_system_symbol(symbol: Symbol) -> bool: + if symbol.is_system: + logger.debug(f"Ignoring system symbol `{symbol.name}`") + return True + if symbol.presumed_path is None: + return False + if os.path.commonpath([symbol.presumed_path, symbol.tu_path]) == "/": + logger.debug(f"Ignoring system symbol {symbol.name}") + return True + return False + + +def get_symbols_and_dependencies( + asts: list[TreeResult], + external_symbol_names: list[SymbolName] | None = None, + ast_order: dict[Path, TreeResult] | None = None, + filter_system_symbols: bool = True, +) -> tuple[dict[SymbolName, Symbol], dict[SymbolGroup, list[SymbolGroup]]]: + list_of_symbols: list[dict[SymbolName, Symbol]] = [ast.symbols for ast in asts] + if filter_system_symbols: + list_of_symbols = [ + {name: symbol for name, symbol in symbols.items() if not is_system_symbol(symbol)} + for symbols in list_of_symbols + ] + + # Merge ASTs into project dependencies + project_symbols = merge_symbols(list_of_symbols, ast_order) + project_dependencies = nx.compose_all( + [ + nx.from_dict_of_lists(ast.complete_graph, create_using=nx.DiGraph) # type: ignore + for ast in asts + ] + ).subgraph(project_symbols.keys()) + + # Find all reachable symbols and subgraph of dependencies from symbols with global functions/variables + symbols = project_symbols.copy() + dependencies = project_dependencies.copy() + if external_symbol_names is None: + # Use global function/variables as desired external symbol names + external_symbol_names = [ + name + for name, symbol in symbols.items() + if symbol.is_global + and (symbol.is_variable or (symbol.is_function and symbol.is_definition)) + and not is_system_symbol(symbol) + ] + if external_symbol_names: + paths = nx.multi_source_dijkstra_path(project_dependencies, external_symbol_names) + symbols = {k: v for k, v in symbols.items() if k in paths} + dependencies = dependencies.subgraph(symbols.keys()).copy() + else: + logger.warning("No external symbols were found/specified!") + + # Remove cycles from graph by combining strongly-connected components. Note that we sort + # members in a SCC so they are ordered lexically. + C = nx.condensation(dependencies) + symbol_lexical_key = create_symbol_lexical_key_fn(symbols, ast_order) + scc_map = {n: tuple(sorted(C.nodes[n]["members"], key=symbol_lexical_key)) for n in C.nodes} + dependencies = {scc_map[n]: [scc_map[s] for s in C.successors(n)] for n in C.nodes} + + # Make sure dependencies are topologically sortable + try: + list(TopologicalSorter(dependencies).static_order()) + except CycleError as ex: + logger.error(ex) + raise ex + return symbols, dependencies + + +def create_ast_order( + source_priority: list[Path], asts: list[TreeResult] +) -> dict[Path, TreeResult]: + # Preserve explicit source priority ordering first, then deterministically append + # any remaining TUs. + ast_by_path: dict[Path, TreeResult] = {} + for tree in asts: + first_symbol = next(iter(tree.symbols.values()), None) + if first_symbol is None: + continue + path = first_symbol.tu_path + ast_by_path[path] = tree + + ast_order: dict[Path, TreeResult] = {} + seen: set[Path] = set() + + for path in source_priority: + resolved = path.resolve() + if resolved not in seen: + seen.add(resolved) + if resolved in ast_by_path: + ast_order[resolved] = ast_by_path[resolved] + + for tu_path in sorted(ast_by_path.keys()): + if tu_path not in seen: + logger.info("Adding translation unit not seen in source_priority: %s", tu_path) + seen.add(tu_path) + ast_order[tu_path] = ast_by_path[tu_path] + return ast_order + + +def get_asts(compile_commands: Path, valid_paths: list[Path]) -> list[TreeResult]: + assert compile_commands.name == "compile_commands.json" + db = CompilationDatabase.fromDirectory(compile_commands.parent) + cmds = db.getAllCompileCommands() + if cmds is None or len(cmds) == 0: + return [] + + valid_paths_set = {path.resolve() for path in valid_paths} + source_root = Path( + os.path.commonpath([Path(cmd.filename).resolve().parent for cmd in cmds]) + ) + maybe_asts = _parse_asts_parallel( + [(cmd.filename, _get_args(cmd, source_root), valid_paths_set) for cmd in cmds] + ) + asts = [ast for ast in maybe_asts if ast is not None] + if tu_renames := _get_conflicting_symbols(asts): + asts = _apply_renames(asts, tu_renames) + return asts + + +def _get_args(cmd: CompileCommand, source_root: Path) -> list[str]: + args = list(cmd.arguments) + + # Make `__FILE__` expand relative to the project root rather than an absolute path. + # A degenerate root (e.g. "/") would only strip the leading slash, so skip it. + if source_root.parent != source_root: + args.append(f"-fmacro-prefix-map={source_root.as_posix()}/=") + + return args + + +def _apply_renames( + asts: list[TreeResult], tu_renames: dict[Path, dict[SymbolName, SymbolSpelling]] +) -> list[TreeResult]: + ast_paths: dict[Path, TreeResult] = {} + for ast in asts: + assert ast.filename is not None + ast_paths[Path(ast.filename).resolve()] = ast + + # Collect raw edits from each renamed TU parsed against original on-disk sources. + # Parsing without any modified_sources avoids poisoning the header view seen by + # subsequent TUs before all renames have been computed. Since _get_conflicting_symbols + # assigns a single canonical new spelling per USR, the same token in a shared header + # will produce identical (offset, replacement) pairs from every TU that includes it, + # so merging edit dicts with update() is safe. + merged_edits: dict[Path, dict[tuple[int, int], bytes]] = {} + for tu_path, renames in tu_renames.items(): + ast = ast_paths[tu_path] + assert ast.filename is not None + assert ast.arguments is not None + tu = _get_tu(ast.filename, ast.arguments) + for file_path, edits in clang_rename(tu, renames).items(): + file_edits = merged_edits.setdefault(file_path, {}) + for extent, replacement in edits.items(): + existing = file_edits.get(extent) + if existing is not None and existing != replacement: + raise ValueError( + f"Conflicting rename edits for {file_path} at {extent}: {existing!r} vs {replacement!r}" + ) + file_edits[extent] = replacement + + # Apply all merged edits once to produce the final in-memory sources. + modified_sources: dict[Path, bytes] = {} + for file_path, edits in merged_edits.items(): + source = file_path.read_bytes() + for (start, end), replacement in sorted( + edits.items(), key=lambda e: e[0][0], reverse=True + ): + source = source[:start] + replacement + source[end:] + modified_sources[file_path] = source + + # Re-parse ALL TUs (not just renamed ones) with modified_sources so that TUs + # that only include a renamed header also get updated ASTs. + reparsed = _parse_asts_parallel( + [ + (ast.filename, ast.arguments, set(), modified_sources) + for ast in ast_paths.values() + if ast.filename is not None and ast.arguments is not None + ] + ) + updated: dict[Path, TreeResult] = {} + for ast, ast_reparsed in zip(ast_paths.values(), reparsed): + assert ast.filename is not None + tu_path = Path(ast.filename).resolve() + updated[tu_path] = ast_reparsed if ast_reparsed is not None else ast + return list(updated.values()) + + +def _parse_asts_parallel(args: list[tuple]) -> list[TreeResult | None]: + if len(args) <= 1: + return [_get_ast(*a) for a in args] + with ProcessPoolExecutor() as pool: + futures = [pool.submit(_get_ast, *a) for a in args] + return [f.result() for f in futures] + + +def _get_tu( + filename: str, arguments: list[str], unsaved_files: dict[Path, bytes] | None = None +) -> TranslationUnit: + try: + uf = [(str(p), content) for p, content in (unsaved_files or {}).items()] or None + tu = TranslationUnit.from_source(None, args=arguments, unsaved_files=uf) + if any(d.severity >= Diagnostic.Error for d in tu.diagnostics): + raise TranslationUnitLoadError("\n".join(d.format() for d in tu.diagnostics)) + except TranslationUnitLoadError as e: + raise TranslationUnitLoadError( + f"Failed to parse '{filename}' with compile arguments:\n" + f" {' '.join(arguments)}\n\n" + f"{e}" + ) + return tu + + +def _get_ast( + filename: str, + arguments: list[str], + valid_paths: set[Path], + unsaved_files: dict[Path, bytes] | None = None, +) -> TreeResult | None: + logger.info(f"Parsing {filename} ...") + tu = _get_tu(filename, arguments, unsaved_files) + assert tu.cursor is not None + source_path = Path(tu.cursor.spelling).resolve() + if valid_paths and source_path not in valid_paths: + return None + tree = extract_info_c(tu) + tree.filename = filename + tree.arguments = arguments + return tree + + +def _get_conflicting_symbols( + asts: list[TreeResult], +) -> dict[Path, dict[SymbolName, SymbolSpelling]]: + # Gather best representative symbol per spelling per AST into a single dict + symbols_with_spelling: dict[SymbolSpelling, list[Symbol]] = {} + for ast in asts: + seen: dict[SymbolSpelling, Symbol] = {} + for symbol in ast.symbols.values(): + spelling = symbol.spelling + if not spelling: + continue + + if symbol.kind == CursorKind.STRUCT_DECL: + spelling = "struct " + spelling + if symbol.kind == CursorKind.UNION_DECL: + spelling = "union " + spelling + if symbol.kind == CursorKind.ENUM_DECL: + spelling = "enum " + spelling + + # Save this symbol if we haven't seen it before + if spelling not in seen: + seen[spelling] = symbol + # Or replace it if it's a definition and existing symbol is a declaration + elif symbol.is_definition and not seen[spelling].is_definition: + seen[spelling] = symbol + for spelling, sym in seen.items(): + symbols_with_spelling.setdefault(spelling, []).append(sym) + + # Find symbols with common spelling but different definitions across ASTs. + # Group definitions by code to avoid O(n^2) pairwise comparison. + tu_renames: dict[Path, dict[SymbolName, SymbolSpelling]] = {} + used_spellings = set(symbols_with_spelling.keys()) + for ast in asts: + used_spellings.update(ast.local_names) + new_spellings: dict[tuple[Path, SymbolSpelling], SymbolSpelling] = {} + for spelling, symbols in symbols_with_spelling.items(): + # Only definitions and variables can conflict + definitions = [s for s in symbols if s.is_definition or s.is_variable] + if len(definitions) <= 1: + continue + + # If there is only one unique presumed path, then nothing to rename + if len({sym.presumed_path or sym.tu_path for sym in definitions}) <= 1: + continue + + # Byte-identical definitions are one entity redeclared, not a conflict + if len({str(sym.code) for sym in definitions}) <= 1: + continue + + # Multiple distinct definitions exist - rename any symbol that can safely be + # renamed. Only true linker symbols (global functions and global variables) must + # preserve their spelling across TUs. Struct/union/enum tags and typedefs have + # no linker visibility in C, so they can differ freely between TUs. However, + # clang reports EXTERNAL linkage for all of these — including anonymous tags that + # inherit the name of their enclosing typedef — so we cannot rely on is_global + # to filter them out and must check the cursor kind explicitly. + NON_LINKED_KINDS = ( + CursorKind.STRUCT_DECL, + CursorKind.UNION_DECL, + CursorKind.ENUM_DECL, + CursorKind.TYPEDEF_DECL, + ) + for sym in definitions: + if sym.is_system: + continue + if sym.is_global and sym.is_top_level and sym.kind not in NON_LINKED_KINDS: + continue + + path = sym.presumed_path or sym.tu_path + if (path, sym.spelling) in new_spellings: + new_spelling = new_spellings[(path, sym.spelling)] + else: + new_spelling = mangle(path.stem) + "_" + sym.spelling + while new_spelling in used_spellings: + path = path.parent + new_spelling = mangle(path.stem) + "_" + new_spelling + used_spellings.add(new_spelling) + new_spellings[(path, sym.spelling)] = new_spelling + tu_renames.setdefault(sym.tu_path, {})[sym.name] = new_spelling + return tu_renames + + +def merge_symbols( + list_of_symbols: list[dict[SymbolName, Symbol]], + ast_order: dict[Path, TreeResult] | None = None, +) -> dict[SymbolName, Symbol]: + ast_order = ast_order or {} + ast_rank: dict[Path, int] = {path: i for i, path in enumerate(ast_order)} + global_symbols: dict[SymbolName, Symbol] = {} + for symbols in list_of_symbols: + # Gather symbols + for name, symbol in symbols.items(): + # If not in global symbol table add it + if name not in global_symbols: + global_symbols[name] = symbol + continue + + # If code matches, then don't bother replacing + if global_symbols[name].code == symbol.code: + continue + + global_source = global_symbols[name].tu_path + symbol_source = symbol.tu_path + + # If overwriting a symbol, then prefer one with a definition + if global_symbols[name].is_definition and not symbol.is_definition: + continue + elif not global_symbols[name].is_definition and symbol.is_definition: + global_symbols[name] = symbol + # Prefer non-extern variable declaration over extern one (e.g. tentative definition) + elif ( + symbol.kind == CursorKind.VAR_DECL + and global_symbols[name].storage_class == StorageClass.EXTERN + and symbol.storage_class != StorageClass.EXTERN + ): + global_symbols[name] = symbol + # Never replace a non-extern variable with an extern one + elif ( + global_symbols[name].kind == CursorKind.VAR_DECL + and global_symbols[name].storage_class != StorageClass.EXTERN + and symbol.storage_class == StorageClass.EXTERN + ): + continue + # Or prefer the symbol with source priority + elif global_source in ast_order and symbol_source not in ast_order: + continue + elif global_source not in ast_order and symbol_source in ast_order: + global_symbols[name] = symbol + elif ( + global_source in ast_order + and symbol_source in ast_order + and ast_rank[global_source] > ast_rank[symbol_source] + ): + global_symbols[name] = symbol + elif ( + global_source in ast_order + and symbol_source in ast_order + and ast_rank[global_source] < ast_rank[symbol_source] + ): + continue + else: + # Two symbols have similar names but different declarations or definitions and no source priority! + raise RuntimeError( + f"Unable to handle symbol {name} with multiple different definitions and unknown source priority!\nSymbol found in {global_source} and {symbol_source}." + ) + return global_symbols + + +def consolidate( + symbols: dict[SymbolName, Symbol], + symbol_order: list[SymbolGroup], + include_line_directives: bool = True, +) -> CodeC: + # Consolidate C sources keyed by raw snippet with optional line directive. + sources: dict[CodeC, str | None] = {} + for group in symbol_order: + # Add forward declarations if more than one symbol in group + if len(group) > 1: + for name in group: + symbol = symbols[name] + declaration = symbol.declaration + if declaration is None: + continue + if declaration in sources: + continue + sources[declaration] = None + if include_line_directives and symbol.declaration_line_directive is not None: + sources[declaration] = str(symbol.declaration_line_directive) + + # Add symbol code + for name in group: + symbol = symbols[name] + code = symbol.code + if code in sources: + continue + sources[code] = None + if include_line_directives and symbol.line_directive is not None: + sources[code] = str(symbol.line_directive) + + return CodeC.join( + snippet if directive is None else CodeC(directive + str(snippet)) + for snippet, directive in sources.items() + ) + + +def _generate_build_rs(extra_link_libs: list[str]) -> str: + body_lines = [ + 'println!("cargo:rerun-if-changed=src/lib.c");', + 'let ubsan = std::env::var("CARGO_FEATURE_CC_UBSAN").is_ok();', + 'let asan = std::env::var("CARGO_FEATURE_CC_ASAN").is_ok();', + 'let coverage = std::env::var("CARGO_FEATURE_CC_COVERAGE").is_ok();', + "assert!(", + " [ubsan, asan, coverage].iter().filter(|&&x| x).count() <= 1,", + ' "features `ubsan`, `asan`, and `coverage` are mutually exclusive"', + ");", + "let mut build = cc::Build::new();", + "build", + ' .compiler("clang")', + " .warnings(false)", + ' .file("src/lib.c");', + "if ubsan {", + ' build.flag("-fsanitize=undefined,nullability")', + ' .flag("-fno-sanitize-recover=all");', + "}", + "", + "if asan {", + ' build.flag("-fsanitize=address")', + ' .flag("-fno-sanitize-recover=all");', + "}", + "", + "if coverage {", + ' build.flag("-fprofile-instr-generate")', + ' .flag("-fcoverage-mapping");', + "}", + 'build.compile("library");', + "", + "if ubsan {", + ' println!("cargo:rustc-link-search=/usr/lib/llvm-21/lib/clang/21/lib/linux/");', + ' println!("cargo:rustc-link-lib=static=clang_rt.ubsan_standalone-x86_64");', + "}", + "if asan {", + ' println!("cargo:rustc-link-search=/usr/lib/llvm-21/lib/clang/21/lib/linux/");', + ' println!("cargo:rustc-link-lib=static=clang_rt.asan-x86_64");', + "}", + ] + for lib in extra_link_libs: + body_lines.append(f'println!("cargo:rustc-link-lib=dylib={lib}");') + body = textwrap.indent("\n".join(body_lines), " ") + return f"fn main() {{\n{body}\n}}\n" + + +def _generate_bindings(symbols: dict[SymbolName, Symbol], c_src_path: Path) -> str: + allowed_functions = [ + mangle_rs(s.spelling) + for s in symbols.values() + if s.is_global and s.is_function and s.is_definition and not is_system_symbol(s) + ] + allowed_variables = [ + mangle_rs(s.spelling) + for s in symbols.values() + if s.is_global and s.is_variable and not is_system_symbol(s) + ] + return _bindgen( + c_src_path, + allowlist_functions=allowed_functions, + allowlist_vars=allowed_variables, + ) + + +def _bindgen( + c_src_path: Path, + allowlist_functions: list[str], + allowlist_vars: list[str], +) -> str: + cmd = [ + "bindgen", + "--disable-header-comment", + "--no-doc-comments", + "--no-layout-tests", + "--merge-extern-blocks", + ] + for fn in allowlist_functions: + cmd += ["--allowlist-function", fn] + for var in allowlist_vars: + cmd += ["--allowlist-var", var] + cmd.append(str(c_src_path)) + logger.info(f"Running `{' '.join(cmd)}` ...") + + ok, bindings, error, _ = run_subprocess(cmd) + if not ok: + raise ValueError(f"`{' '.join(cmd)}` failed!\n{bindings + error}") + return bindings + + +def _main(cfg: ConsolidateConfig): + # Create fresh Cargo.toml so cargo init always runs and registers the crate in workspace.members + crate = Crate(cfg.cargo_toml, vcs=cfg.vcs, template=cfg.template, reinit=True) # type: ignore[reportArgumentType] + # Binary -sys crates always have a lib.rs file with bindings + if cfg.template == "bin": + assert crate.main_src_path is not None, "Expected main.rs to exist in -sys crate!" + (crate.main_src_path.parent / "lib.rs").touch() + crate.invalidate_metadata() + + # Read source priority and link libs from the links file. + links_data = json.loads(cfg.links.read_text()) + source_priority: list[Path] = [] + extra_link_libs: list[str] = [] + for entry in links_data.get("entries", []): + if "source" in entry: + source_priority.append(Path(entry["source"]).resolve()) + elif "lib" in entry: + extra_link_libs.append(entry["lib"]) + + symbols, symbol_order = analyze(cfg.compile_commands, source_priority) + logger.info( + f"Found {len(symbols)} symbols and {len(symbol_order)} groups in {cfg.compile_commands}!" + ) + + assert crate.lib_src_path is not None, "Expected lib.rs to exist in -sys crate!" + c_src_path = crate.lib_src_path.with_suffix(".c") + + # Consolidate and write C code to disk + c_src = consolidate(symbols, symbol_order, cfg.include_line_directives) + c_src_path.parent.mkdir(exist_ok=True, parents=True) + c_src_path.write_text(str(c_src)) + + # Write build.rs to disk with dependencies and the sanitizer/coverage config + (crate.cargo_toml.parent / "build.rs").write_text(_generate_build_rs(extra_link_libs)) + crate.cargo_add("cc@1.2.53", section="build") + crate.cargo_feature(cc_ubsan=[], cc_asan=[], cc_coverage=[]) + + # Write bindings to disk + crate.lib_src_path.write_text(_generate_bindings(symbols, c_src_path)) + if crate.main_src_path is not None: + assert crate.lib_name is not None, "Expected a library target in the -sys crate!" + crate.main_src_path.write_text(f"#![no_main]\nuse {crate.lib_name}::*;\n") + + # Write cargo configurations to disk + # NOTE: We configure nextest for both the crate and the workspace such that the -sys crate is standalone + crate.cargo_nextest_config(crate.cargo_toml.parent / ".config" / "nextest.toml") + crate.cargo_nextest_config(crate.workspace_root / ".config" / "nextest.toml") + + # Commit the crate + crate.vcs.add(crate.cargo_toml.parent) + workspace_cargo_toml = crate.workspace_root / "Cargo.toml" + if workspace_cargo_toml != crate.cargo_toml and workspace_cargo_toml.exists(): + crate.vcs.add(workspace_cargo_toml) + crate.vcs.commit(f"Created C bindings crate '{crate.name}'") + + +@hydra.main(version_base=None, config_name="consolidate") +def main(cfg: ConsolidateConfig): + try: + _main(cfg) + except Exception as e: + logger.exception(e) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/src/ideas/evaluate.py b/src/ideas/evaluate.py deleted file mode 100644 index 234da2f..0000000 --- a/src/ideas/evaluate.py +++ /dev/null @@ -1,101 +0,0 @@ -# -# Copyright (C) 2026 Intel Corporation -# -# SPDX-License-Identifier: Apache-2.0 -# - - -import re -import sys -import logging -from dataclasses import dataclass -from pathlib import Path - -import hydra -from omegaconf import MISSING -from hydra.core.config_store import ConfigStore - -from ideas.tools import Crate, nextest_json_to_libtest - - -logger = logging.getLogger("ideas.evaluate") - - -@dataclass -class EvaluateConfig: - manifest: Path = MISSING - test_cases: str = MISSING - - output_file: Path = MISSING - - -cs = ConfigStore.instance() -cs.store(name="evaluate", node=EvaluateConfig) - - -_TEST_FN_RE = re.compile(r"^\s*fn\s+(\w+)\s*\(", re.MULTILINE) - - -def list_tests(test_file: Path) -> list[str]: - """Parse #[test] function names directly from a .rs integration test file.""" - source = test_file.read_text() - tests = [] - lines = source.splitlines() - for i, line in enumerate(lines): - if line.strip() == "#[test]": - for subsequent in lines[i + 1 :]: - m = _TEST_FN_RE.match(subsequent) - if m: - tests.append(m.group(1)) - break - # Skip attributes/comments between #[test] and fn - if subsequent.strip() and not subsequent.strip().startswith(("#", "/")): - break - return tests - - -def _main(cfg: EvaluateConfig) -> None: - # Resolve integration test file (error loudly if missing) - crate = Crate(cfg.manifest, vcs="none") - test_file = crate.cargo_toml.parent / "tests" / f"{cfg.test_cases}.rs" - if not test_file.exists(): - raise FileNotFoundError(f"Integration test file not found: {test_file}") - - # Attempt to build the evaluation test - builds, _, _, _ = crate.cargo_test( - name=cfg.test_cases, quiet=False, fail_fast=True, build_only=True - ) - if builds: - # Use libtest-json output, parse it, and reformat for readability - # stderr contains the native nextest output - _, stdout, stderr, _ = crate.cargo_test( - name=cfg.test_cases, message_format="libtest-json" - ) - output = nextest_json_to_libtest(stdout) + stderr - else: - output = f"Failed to build test target {cfg.test_cases} for evaluation!\n" - names = list_tests(test_file) - lines = [f"test {name} ... FAILED" for name in names] - output += "\n".join(lines) - if lines: - output += "\n" - - # Write to output file - cfg.output_file.parent.mkdir(parents=True, exist_ok=True) - cfg.output_file.write_text(output) - crate.vcs.add(cfg.output_file) - crate.vcs.commit(f"Evaluation results for {cfg.test_cases}") - print(output) - - -@hydra.main(version_base=None, config_name="evaluate") -def main(cfg: EvaluateConfig) -> None: - try: - _main(cfg) - except Exception as e: - logger.exception(e) - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/src/ideas/init/build.py b/src/ideas/init/build.py deleted file mode 100644 index 52283bd..0000000 --- a/src/ideas/init/build.py +++ /dev/null @@ -1,254 +0,0 @@ -# -# Copyright (C) 2026 Intel Corporation -# -# SPDX-License-Identifier: Apache-2.0 -# - -import re -import sys -import logging -import textwrap -from pathlib import Path -from dataclasses import dataclass -from concurrent.futures import ThreadPoolExecutor - -import hydra -from omegaconf import MISSING -from hydra.core.config_store import ConfigStore -from hydra.core.hydra_config import HydraConfig - -from ideas.ast import Symbol -from ideas.tools import Crate, run_subprocess -from ideas.ast_rust import CodeRust, mangle -from ideas import create_translation_unit, extract_info_c -from ideas.init.consolidate import get_symbols_and_dependencies - -logger = logging.getLogger("ideas.init.build") - - -@dataclass -class BuildConfig: - cargo_toml: Path = MISSING - vcs: str = "none" - - def __post_init__(self): - if self.vcs not in ["git", "none"]: - raise ValueError(f"Invalid VCS: {self.vcs}!") - - -cs = ConfigStore.instance() -cs.store(name="init.build", node=BuildConfig) - - -def write_build_script(crate: Crate) -> Path: - c_src_path = crate.c_src_path.relative_to(crate.cargo_toml.parent) - build_options = '.define("main", "_main")' if crate.is_bin else "" - build_rs_path = crate.cargo_toml.parent / "build.rs" - build_rs_path.write_text( - f""" - fn main() {{ - println!("cargo:rerun-if-changed={c_src_path}"); - cc::Build::new() - .compiler("clang") - .warnings(false) - .file("{c_src_path}") - {build_options} - .compile("library"); - println!("cargo:rustc-link-lib=static=library"); - // FIXME: How do we statically add libraries to link to? - println!("cargo:rustc-link-lib=dylib=crypto"); - }} - """ - ) - return build_rs_path - - -def write_main_binding(crate: Crate) -> CodeRust: - main_binding_path, main_function = _write_main_binding( - crate.c_src_path, crate.rust_src_path.parent / "binding" - ) - crate.vcs.add(main_binding_path) - return CodeRust(main_function) - - -def _write_main_binding(c_src_path: Path, binding_dir: Path) -> tuple[Path, str]: - main_binding = _get_linked_binding("_main", c_src_path, "-Dmain=_main") - main_binding_path = binding_dir / "main.rs" - main_binding_path.parent.mkdir(exist_ok=True, parents=True) - main_binding_path.write_text( - "\n\n".join( - [ - "#![allow(unused_attributes)]", - str(main_binding), - ] - ) - ) - # Return appropriate main function instead of writing to binding.rs - if "fn _main()" in str(main_binding): - main_function = textwrap.dedent( - """ - pub fn main() { - let ret = unsafe { binding::main::_main() }; - std::process::exit(ret); - } - """ - ) - else: - main_function = textwrap.dedent( - """ - pub fn main() { - let mut args: Vec<_> = std::env::args().into_iter().map(|s| std::ffi::CString::new(s).unwrap().into_raw()).collect(); - let ret = unsafe { binding::main::_main(args.len() as i32, args.as_mut_ptr()) }; - std::process::exit(ret); - } - """ - ) - return main_binding_path, main_function - - -def _generate_binding(crate: Crate, symbol: Symbol) -> tuple[Path, str, str | None]: - c_src_path = crate.c_src_path - binding_dir = crate.rust_src_path.parent / "binding" - symbol_spelling = symbol.spelling - is_main = crate.is_bin and symbol_spelling == "main" - - logger.info(f"Generating binding for symbol '{symbol_spelling}' ...") - if is_main: - main_binding_path, main_function = _write_main_binding(c_src_path, binding_dir) - return main_binding_path, "main", main_function - - rust_spelling = mangle(symbol_spelling) - symbol_binding = _get_linked_binding(rust_spelling, c_src_path) - symbol_binding_path = binding_dir / f"{rust_spelling}.rs" - symbol_binding_path.parent.mkdir(exist_ok=True, parents=True) - symbol_binding_path.write_text( - "\n\n".join( - [ - "#![allow(unused_attributes)]", - str(symbol_binding), - ] - ) - ) - return symbol_binding_path, rust_spelling, None - - -def _get_linked_binding(function_name: str, c_src_path: Path, *bindgen_args: str) -> CodeRust: - # Use bindgen to generate binding to C symbol - bindgen = [ - "bindgen", - "--disable-header-comment", - "--no-doc-comments", - "--no-layout-tests", - "--allowlist-function", - function_name, - str(c_src_path), - "--", - *bindgen_args, - ] - ok, binding, error, _ = run_subprocess(bindgen) - if not ok: - raise ValueError(f"`{' '.join(bindgen)}` failed!\n{binding + error}") - - # Remove \u{1} prefix from link_name attribute - linked_binding = binding.replace('#[link_name = "\\u{1}', '#[link_name = "') - - # Enable the symbol to be re-exportable by rustc - linked_binding = re.sub( - r'unsafe extern "C" {\n(.*)\n}', - r'#[link(name="library", kind="static")]\nunsafe extern "C" {\n #[unsafe(no_mangle)]\n\1\n}', - linked_binding, - flags=re.DOTALL, - ) - if linked_binding == binding: - raise ValueError( - f"Failed to convert binding to linked binding for {function_name}!\n{binding}" - ) - return CodeRust(linked_binding) - - -def _main(cfg: BuildConfig) -> None: - output_dir = Path(HydraConfig.get().runtime.output_dir) - - # Fetch crate - crate = Crate(cfg.cargo_toml, vcs=cfg.vcs) # type: ignore[reportArgumentType] - - # Get global symbol table - tu = create_translation_unit(crate.c_src_path) - asts = [extract_info_c(tu)] - symbols, _ = get_symbols_and_dependencies( - asts, external_symbol_names=["c:@F@main"] if crate.is_bin else None - ) - global_functions = [ - s for s in symbols.values() if s.is_global and (s.is_function and s.is_definition) - ] - if not global_functions: - logger.info("No global functions to generate bindings for!") - return - - # Write build.rs file - build_rs_path = write_build_script(crate) - crate.vcs.add(build_rs_path) - - # Verify build with build.rs - builds, feedback = crate.cargo_build() - if not builds: - raise RuntimeError(f"Crate does not build!\n{feedback}") - - # Generate a Rust binding for any global function since we need to force the Rust - # linker to include that C function in the Rust artifact. - # FIXME: If we ever test variables we should generate bindings for those here too! - bindings: list[tuple[Path, str, str | None]] = [] - if len(global_functions) <= 1: - bindings.append(_generate_binding(crate, global_functions[0])) - else: - with ThreadPoolExecutor() as pool: - futures = [ - pool.submit(_generate_binding, crate, symbol) for symbol in global_functions - ] - bindings = [future.result() for future in futures] - - # Write bindings to file and stage with VCS - binding_modules_by_file: dict[Path, str] = {} - for binding_file_path, rust_spelling, main_function in bindings: - if main_function is not None: - with crate.rust_src_path.open("a+") as rust_file: - rust_file.write(main_function) - binding_modules_by_file[binding_file_path] = f"pub mod {rust_spelling};\n" - binding_path = crate.rust_src_path.parent / "binding.rs" - binding_path.write_text("".join(binding_modules_by_file.values())) - crate.vcs.add(*binding_modules_by_file.keys(), crate.rust_src_path, binding_path) - - # Make the bindings module visible in the crate - rust_src = crate.rust_src_path.read_text() - BINDING_MOD = "pub mod binding;" - if not re.search(f"^{re.escape(BINDING_MOD)}$", rust_src, flags=re.MULTILINE): - crate.rust_src_path.write_text("\n\n".join([rust_src, BINDING_MOD])) - crate.vcs.add(crate.rust_src_path) - - # Add hydra directory - if (output_subdir := HydraConfig.get().output_subdir) is not None: - crate.vcs.add(output_dir / output_subdir) - - # Attempt a final build - builds, feedback = crate.cargo_build() - if not builds: - raise RuntimeError(f"Crate does not build!\n{feedback}") - msg = f"Generated build artifacts for `{crate.root_package['name']}`" - logger.info(msg) - crate.vcs.commit(msg) - - # Clean on exit - crate.cargo_clean() - - -@hydra.main(version_base=None, config_name="init.build") -def main(cfg: BuildConfig) -> None: - try: - _main(cfg) - except Exception as e: - logger.exception(e) - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/src/ideas/init/consolidate.py b/src/ideas/init/consolidate.py deleted file mode 100644 index 0d55998..0000000 --- a/src/ideas/init/consolidate.py +++ /dev/null @@ -1,558 +0,0 @@ -# -# Copyright (C) 2026 Intel Corporation -# -# SPDX-License-Identifier: Apache-2.0 -# - -import os -import sys -import logging -from pathlib import Path -from functools import cmp_to_key -from dataclasses import dataclass -from graphlib import TopologicalSorter, CycleError -from concurrent.futures import ProcessPoolExecutor - -import hydra -import networkx as nx -from omegaconf import MISSING -from hydra.core.config_store import ConfigStore -from hydra.core.hydra_config import HydraConfig -from clang.cindex import CompilationDatabase, TranslationUnit, CursorKind -from clang.cindex import TranslationUnitLoadError, Diagnostic, StorageClass - -from ideas.ast import extract_info_c, TreeResult, Symbol, clang_rename_, mangle, CodeC -from ideas.tools import Crate, check_c - -logger = logging.getLogger("ideas.init.consolidate") - - -@dataclass -class ConsolidateConfig: - cargo_toml: Path = MISSING - vcs: str = "none" - - compile_commands: Path = MISSING - source_priority: Path | None = None - - -cs = ConfigStore.instance() -cs.store(name="init.consolidate", node=ConsolidateConfig) - - -def init(compile_commands: Path, source_priority: list[Path]) -> CodeC: - # Get symbol table and dependencies taking into account source priority - asts = get_asts(compile_commands, source_priority) - ast_order = create_ast_order(source_priority, asts) - symbols, dependencies = get_symbols_and_dependencies( - asts, ast_order=ast_order, filter_system_symbols=False - ) - logger.info(f"Found {len(symbols)} symbols in {compile_commands}!") - - # Sort symbols in lexicographical topological order - symbol_lexical_key = create_symbol_lexical_key_fn(symbols, ast_order) - sorted_symbol_groups = list( - nx.lexicographical_topological_sort( - nx.from_dict_of_lists(dependencies, create_using=nx.DiGraph).reverse(copy=False), # type: ignore - key=symbol_lexical_key, - ) - ) - - # Consolidate C sources keyed by raw snippet with optional line directive. - sources: dict[CodeC, str | None] = {} - for group in sorted_symbol_groups: - # Add forward declarations if more than one symbol in group - if len(group) > 1: - for name in group: - symbol = symbols[name] - declaration = symbol.declaration - if declaration is None: - continue - if declaration in sources: - continue - sources[declaration] = ( - str(symbol.declaration_line_directive) - if symbol.declaration_line_directive is not None - else None - ) - - # Add symbol code - for name in group: - symbol = symbols[name] - code = symbol.code - if code in sources: - continue - sources[code] = ( - str(symbol.line_directive) if symbol.line_directive is not None else None - ) - - return CodeC.join( - snippet if directive is None else CodeC(directive + str(snippet)) - for snippet, directive in sources.items() - ) - - -def get_symbols_and_dependencies( - asts: list[TreeResult], - external_symbol_names: list[str] | None = None, - ast_order: dict[Path, TreeResult] | None = None, - filter_system_symbols: bool = True, -) -> tuple[dict[str, Symbol], dict[tuple[str, ...], list[tuple[str, ...]]]]: - list_of_symbols: list[dict[str, Symbol]] = [ast.symbols for ast in asts] - if filter_system_symbols: - - def is_system_symbol(symbol: Symbol) -> bool: - if symbol.is_system: - logger.debug(f"Ignoring system symbol `{symbol.name}`") - return True - if symbol.presumed_path is None: - return False - if os.path.commonpath([symbol.presumed_path, symbol.tu_path]) == "/": - logger.debug(f"Ignoring system symbol {symbol.name}") - return True - return False - - list_of_symbols = [ - {name: symbol for name, symbol in symbols.items() if not is_system_symbol(symbol)} - for symbols in list_of_symbols - ] - - # Merge ASTs into project dependencies - project_symbols = merge_symbols(list_of_symbols, ast_order) - project_dependencies = nx.compose_all( - [ - nx.from_dict_of_lists(ast.complete_graph, create_using=nx.DiGraph) # type: ignore - for ast in asts - ] - ).subgraph(project_symbols.keys()) - - # Find all reachable symbols and subgraph of dependencies from symbols with global functions/variables - symbols = project_symbols.copy() - dependencies = project_dependencies.copy() - if external_symbol_names is None: - # Use global function/variables as desired external symbol names - external_symbol_names = [ - name - for name, symbol in symbols.items() - if symbol.is_global - and (symbol.is_variable or (symbol.is_function and symbol.is_definition)) - ] - if external_symbol_names: - paths = nx.multi_source_dijkstra_path(project_dependencies, external_symbol_names) - symbols = {k: v for k, v in symbols.items() if k in paths} - dependencies = dependencies.subgraph(symbols.keys()).copy() - else: - logger.warning("No external symbols were found/specified!") - - # Remove cycles from graph by combining strongly-connected components. Note that we sort - # members in a SCC so they are ordered lexically. - C = nx.condensation(dependencies) - symbol_lexical_key = create_symbol_lexical_key_fn(symbols, ast_order) - scc_map = {n: tuple(sorted(C.nodes[n]["members"], key=symbol_lexical_key)) for n in C.nodes} - dependencies = {scc_map[n]: [scc_map[s] for s in C.successors(n)] for n in C.nodes} - - # Make sure dependencies are topologically sortable - try: - list(TopologicalSorter(dependencies).static_order()) - except CycleError as ex: - logger.error(ex) - raise ex - return symbols, dependencies - - -def create_ast_order( - source_priority: list[Path], asts: list[TreeResult] -) -> dict[Path, TreeResult]: - # Preserve explicit source priority ordering first, then deterministically append - # any remaining TUs. - ast_by_path: dict[Path, TreeResult] = {} - for tree in asts: - first_symbol = next(iter(tree.symbols.values()), None) - if first_symbol is None: - continue - path = first_symbol.tu_path - ast_by_path[path] = tree - - ast_order: dict[Path, TreeResult] = {} - seen: set[Path] = set() - - for path in source_priority: - resolved = path.resolve() - if resolved not in seen: - seen.add(resolved) - if resolved in ast_by_path: - ast_order[resolved] = ast_by_path[resolved] - - for tu_path in sorted(ast_by_path.keys()): - if tu_path not in seen: - logger.info("Adding translation unit not seen in source_priority: %s", tu_path) - seen.add(tu_path) - ast_order[tu_path] = ast_by_path[tu_path] - return ast_order - - -def create_symbol_lexical_key_fn( - symbols: dict[str, Symbol], - ast_order: dict[Path, TreeResult] | None = None, -): - def compare_symbol_lexical(a: str | tuple[str, ...], b: str | tuple[str, ...]) -> int: - # Support symbol groups by using the first symbol in the group. - a_name = a[0] if isinstance(a, tuple) else a - b_name = b[0] if isinstance(b, tuple) else b - - a_symbol = symbols[a_name] - b_symbol = symbols[b_name] - - a_tu = a_symbol.tu_path - b_tu = b_symbol.tu_path - - # If symbols are from the same translation unit, compare their - # preorder traversal indices for lexical ordering. - if a_tu == b_tu: - return _cmp_symbol_tu_order(a_symbol, b_symbol) - - if ast_order is None: - raise RuntimeError( - f"Cannot compare symbols from different translation units without ast_order: {a} ({a_tu}) vs {b} ({b_tu})." - ) - - # If a's USR appears in b's TU with matching code, both symbols are - # present in b_tu and can be compared by TU preorder index there. - b_ast = ast_order.get(b_tu) - if ( - b_ast is not None - and a_name in b_ast.symbols - and b_ast.symbols[a_name].code == a_symbol.code - ): - return _cmp_symbol_tu_order(b_ast.symbols[a_name], b_symbol) - - # If b's USR appears in a's TU with matching code, both symbols are - # present in a_tu and can be compared by TU preorder index there. - a_ast = ast_order.get(a_tu) - if ( - a_ast is not None - and b_name in a_ast.symbols - and a_ast.symbols[b_name].code == b_symbol.code - ): - return _cmp_symbol_tu_order(a_symbol, a_ast.symbols[b_name]) - - # The symbol's USR is not shared across TUs, so fall back to ordering - # by the position of each symbol's TU in ast_order (source priority) - ast_rank = {path: i for i, path in enumerate(ast_order)} - try: - a_rank = ast_rank[a_tu] - b_rank = ast_rank[b_tu] - except KeyError as ex: - raise RuntimeError( - f"Cannot compare symbols because one or both translation units are missing from ast_order: {a_tu}, {b_tu}." - ) from ex - - if a_rank < b_rank: - return -1 - if a_rank > b_rank: - return 1 - raise RuntimeError("Distinct translation units cannot have identical ranks!") - - return cmp_to_key(compare_symbol_lexical) - - -def _cmp_symbol_tu_order(symbol_a: Symbol, symbol_b: Symbol) -> int: - if symbol_a.tu_path != symbol_b.tu_path: - raise ValueError( - "Cannot compare TU preorder indices for symbols from different translation units:" - f" {symbol_a.name} @ {symbol_a.tu_path} vs {symbol_b.name} @ {symbol_b.tu_path}" - ) - - order_a = symbol_a.tu_preorder_index - order_b = symbol_b.tu_preorder_index - if order_a < order_b: - return -1 - if order_b < order_a: - return 1 - if symbol_a.name != symbol_b.name: - raise ValueError( - f"Unable to order distinct symbols with identical lexical priority and location:" - f" {symbol_a.name} @ {order_a} vs {symbol_b.name} @ {order_b}" - ) - return 0 - - -def get_asts( - compile_commands: Path, valid_paths: list[Path], rename_conflicting_symbols: bool = True -) -> list[TreeResult]: - assert compile_commands.name == "compile_commands.json" - db = CompilationDatabase.fromDirectory(compile_commands.parent) - cmds = db.getAllCompileCommands() - if cmds is None or len(cmds) == 0: - return [] - - valid_paths_set = {path.resolve() for path in valid_paths} - maybe_asts: list[TreeResult | None] - if len(cmds) <= 1: - maybe_asts = [_get_ast(cmds[0].filename, list(cmds[0].arguments), valid_paths_set)] - else: - with ProcessPoolExecutor() as pool: - futures = [ - pool.submit(_get_ast, cmd.filename, list(cmd.arguments), valid_paths_set) - for cmd in cmds - ] - maybe_asts = [future.result() for future in futures] - asts = [ast for ast in maybe_asts if ast is not None] - - if rename_conflicting_symbols: - original_sources = rename_conflicting_symbols_(asts) - if original_sources: - try: - asts = get_asts(compile_commands, valid_paths, rename_conflicting_symbols=False) - finally: - # Always restore original source code after reparsing renamed symbols. - for path, source in original_sources.items(): - path.write_bytes(source) - return asts - - -def _get_ast(filename: str, arguments: list[str], valid_paths: set[Path]) -> TreeResult | None: - logger.info(f"Parsing {filename} ...") - try: - tu = TranslationUnit.from_source(None, args=arguments) - if any(d.severity >= Diagnostic.Error for d in tu.diagnostics): - raise TranslationUnitLoadError("\n".join(d.format() for d in tu.diagnostics)) - except TranslationUnitLoadError as e: - raise TranslationUnitLoadError( - f"Failed to parse '{filename}' with compile arguments:\n" - f" {' '.join(arguments)}\n\n" - f"{e}" - ) - - assert tu.cursor is not None - source_path = Path(tu.cursor.spelling).resolve() - if valid_paths and source_path not in valid_paths: - return None - tree = extract_info_c(tu) - tree.filename = filename - tree.arguments = arguments - return tree - - -def rename_conflicting_symbols_(asts: list[TreeResult]) -> dict[Path, bytes]: - # Gather best representative symbol per spelling per AST into a single dict - symbols_with_spelling: dict[str, list[Symbol]] = {} - for ast in asts: - seen: dict[str, Symbol] = {} - for symbol in ast.symbols.values(): - spelling = symbol.spelling - if not spelling: - continue - - if symbol.kind == CursorKind.STRUCT_DECL: - spelling = "struct " + spelling - if symbol.kind == CursorKind.UNION_DECL: - spelling = "union " + spelling - if symbol.kind == CursorKind.ENUM_DECL: - spelling = "enum " + spelling - - # Save this symbol if we haven't seen it before - if spelling not in seen: - seen[spelling] = symbol - # Or replace it if it's a definition and existing symbol is a declaration - elif symbol.is_definition and not seen[spelling].is_definition: - seen[spelling] = symbol - for spelling, sym in seen.items(): - symbols_with_spelling.setdefault(spelling, []).append(sym) - - # Find symbols with common spelling but different definitions across ASTs. - # Group definitions by code to avoid O(n^2) pairwise comparison. - tu_renames: dict[Path, dict[str, str]] = {} - used_spellings = set(symbols_with_spelling.keys()) - for spelling, symbols in symbols_with_spelling.items(): - # Only definitions and variables can conflict - definitions = [s for s in symbols if s.is_definition or s.is_variable] - if len(definitions) <= 1: - continue - - # Group definitions by their code text - identical code means no conflict - code_groups: dict[CodeC, list[Symbol]] = {} - for sym in definitions: - code_groups.setdefault(sym.code, []).append(sym) - if len(code_groups) <= 1: - continue - - # Multiple distinct definitions exist - rename any symbol that can safely be - # renamed. Only true linker symbols (global functions and global variables) must - # preserve their spelling across TUs. Struct/union/enum tags and typedefs have - # no linker visibility in C, so they can differ freely between TUs. However, - # clang reports EXTERNAL linkage for all of these — including anonymous tags that - # inherit the name of their enclosing typedef — so we cannot rely on is_global - # to filter them out and must check the cursor kind explicitly. - NON_LINKED_KINDS = ( - CursorKind.STRUCT_DECL, - CursorKind.UNION_DECL, - CursorKind.ENUM_DECL, - CursorKind.TYPEDEF_DECL, - ) - for sym in definitions: - if sym.is_system: - continue - if sym.is_global and sym.is_top_level and sym.kind not in NON_LINKED_KINDS: - continue - - path = sym.tu_path - new_spelling = mangle(path.stem) + "_" + sym.spelling - while new_spelling in used_spellings: - path = path.parent - new_spelling = mangle(path.stem) + "_" + new_spelling - used_spellings.add(new_spelling) - tu_renames.setdefault(sym.tu_path, {})[sym.name] = new_spelling - if not tu_renames: - return {} - - # Check that renaming won't cause clashes with existing symbols with the same spelling - existing_spellings = set(symbols_with_spelling.keys()) - for renames in tu_renames.values(): - new_spellings = set(renames.values()) - if existing_spellings.intersection(new_spellings): - raise NotImplementedError( - "Renaming symbols would cause clashes with existing symbols with the same spelling!\n" - f"Clashing: {existing_spellings.intersection(new_spellings)}" - ) - existing_spellings.update(new_spellings) - - # Write renames to disk while keeping track of original source bytes - tu_args: dict[Path, list[str]] = {} - for ast in asts: - if ast.filename is None or ast.arguments is None: - continue - tu_args[Path(ast.filename).resolve()] = list(ast.arguments) - - original_sources: dict[Path, bytes] = {} - try: - for tu_path, renames in tu_renames.items(): - args = tu_args.get(tu_path) - if args is not None: - tu = TranslationUnit.from_source(None, args=list(args)) - else: - tu = TranslationUnit.from_source(str(tu_path)) - clang_rename_(tu, renames, original_sources) - except Exception: - # Restore original source code if renaming fails before caller can reparse. - for path, source in original_sources.items(): - path.write_bytes(source) - raise - return original_sources - - -def merge_symbols( - list_of_symbols: list[dict[str, Symbol]], ast_order: dict[Path, TreeResult] | None = None -) -> dict[str, Symbol]: - ast_order = ast_order or {} - ast_rank: dict[Path, int] = {path: i for i, path in enumerate(ast_order)} - global_symbols: dict[str, Symbol] = {} - for symbols in list_of_symbols: - # Gather symbols - for name, symbol in symbols.items(): - # If not in global symbol table add it - if name not in global_symbols: - global_symbols[name] = symbol - continue - - # If code matches, then don't bother replacing - if global_symbols[name].code == symbol.code: - continue - - global_source = global_symbols[name].tu_path - symbol_source = symbol.tu_path - - # If overwriting a symbol, then prefer one with a definition - if global_symbols[name].is_definition and not symbol.is_definition: - continue - elif not global_symbols[name].is_definition and symbol.is_definition: - global_symbols[name] = symbol - # Prefer non-extern variable declaration over extern one (e.g. tentative definition) - elif ( - symbol.kind == CursorKind.VAR_DECL - and global_symbols[name].storage_class == StorageClass.EXTERN - and symbol.storage_class != StorageClass.EXTERN - ): - global_symbols[name] = symbol - # Never replace a non-extern variable with an extern one - elif ( - global_symbols[name].kind == CursorKind.VAR_DECL - and global_symbols[name].storage_class != StorageClass.EXTERN - and symbol.storage_class == StorageClass.EXTERN - ): - continue - # Or prefer the symbol with source priority - elif global_source in ast_order and symbol_source not in ast_order: - continue - elif global_source not in ast_order and symbol_source in ast_order: - global_symbols[name] = symbol - elif ( - global_source in ast_order - and symbol_source in ast_order - and ast_rank[global_source] > ast_rank[symbol_source] - ): - global_symbols[name] = symbol - elif ( - global_source in ast_order - and symbol_source in ast_order - and ast_rank[global_source] < ast_rank[symbol_source] - ): - continue - else: - # Two symbols have similar names but different declarations or definitions and no source priority! - raise RuntimeError( - f"Unable to handle symbol {name} with multiple different definitions and unknown source priority!\nSymbol found in {global_source} and {symbol_source}." - ) - return global_symbols - - -def _main(cfg: ConsolidateConfig): - output_dir = Path(HydraConfig.get().runtime.output_dir) - - # Get crate information - crate = Crate(cfg.cargo_toml, vcs=cfg.vcs) # type: ignore[reportArgumentType] - - source_priority: list[Path] = [] - if cfg.source_priority: - lines = cfg.source_priority.read_text().splitlines() - source_priority = [Path(line.strip()).resolve() for line in lines if line.strip()] - - output = init(cfg.compile_commands, source_priority) - - # Only run preprocess, compile, and assemble steps on C code - compiles, compile_errors = check_c(output, flags=["-c"]) - - # Write C code to disk - crate.c_src_path.parent.mkdir(exist_ok=True, parents=True) - crate.c_src_path.write_text(str(output)) - crate.vcs.add(crate.c_src_path) - - # Add hydra directory - if (output_subdir := HydraConfig.get().output_subdir) is not None: - crate.vcs.add(output_dir / output_subdir) - - # If the C code didn't compile, then error loudly - name = crate.root_package["name"] - msg = f"Consolidated `{name}` in {output_dir}" - if not compiles: - msg = f"Failed to consolidate `{name}` C code!" - msg += f"\n\n{compile_errors}" - logger.error(msg) - else: - logger.info(msg) - crate.vcs.commit(msg) - if not compiles: - raise ValueError(f"Failed to compile consolidated `{name}` C code!") - - -@hydra.main(version_base=None, config_name="init.consolidate") -def main(cfg: ConsolidateConfig): - try: - _main(cfg) - except Exception as e: - logger.exception(e) - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/src/ideas/init/crate.py b/src/ideas/init/crate.py deleted file mode 100644 index 4b870a4..0000000 --- a/src/ideas/init/crate.py +++ /dev/null @@ -1,129 +0,0 @@ -# -# Copyright (C) 2026 Intel Corporation -# -# SPDX-License-Identifier: Apache-2.0 -# - -import sys -import logging -import tomlkit -from pathlib import Path -from dataclasses import dataclass, field - -import hydra -from omegaconf import MISSING -from hydra.core.config_store import ConfigStore -from hydra.core.hydra_config import HydraConfig - -from ideas.tools import Crate, LARGE_PROJECT - -logger = logging.getLogger("ideas.init.crate") - - -@dataclass -class CrateConfig: - cargo_toml: Path = MISSING - template: str = MISSING - vcs: str = "none" - - reexport_lib: bool = True - workspace_dependencies: list[str] = field(default_factory=list) - - def __post_init__(self): - if self.template not in ["bin", "lib"]: - raise ValueError(f"Invalid crate template: {self.template}!") - if self.vcs not in ["git", "none"]: - raise ValueError(f"Invalid VCS: {self.vcs}!") - for dep in self.workspace_dependencies: - if not dep.strip(): - raise ValueError("workspace_dependencies entries must be non-empty crate names") - - -cs = ConfigStore.instance() -cs.store(name="init.crate", node=CrateConfig) - - -def _main(cfg: CrateConfig) -> None: - output_dir = Path(HydraConfig.get().runtime.output_dir) - - # Initialize crate - crate = Crate(cfg.cargo_toml, template=cfg.template, vcs=cfg.vcs) # type: ignore[reportArgumentType] - - # Delete default cargo init code - crate.rust_src_path.write_text("") - - # Add static dependencies - crate.cargo_add(dep="libc@0.2.185") - crate.cargo_add(dep="openssl@0.10.79") - if LARGE_PROJECT: - crate.cargo_add(dep="flate2@1") - crate.cargo_add(dep="regex@1") - crate.cargo_add(dep="serde@1", section="dev", features=["derive"]) - crate.cargo_add(dep="serde_json@1", section="dev") - crate.cargo_add(dep="tempfile@3", section="dev") - crate.cargo_add(dep="cc@1.2.53", section="build") - crate.add_workspace_dependencies(cfg.workspace_dependencies) - - if crate.is_bin: - # Add static test dependencies - crate.cargo_add(dep="assert_cmd@2.0.17", section="dev") - crate.cargo_add(dep="predicates@3.1.3", section="dev") - - # Disable default tests - cargo_toml = tomlkit.loads(crate.cargo_toml.read_text()) - if crate.is_bin: - bin, found = cargo_toml.get("bin", list()), False - for target in bin: - if target.get("name", None) == crate.root_package["name"]: - target["test"], found = False, True - break - if not found: - bin.append({"name": crate.root_package["name"], "test": False}) - cargo_toml["bin"] = bin - else: - lib = cargo_toml.get("lib", dict()) - lib.update({"test": False, "doctest": False}) - cargo_toml["lib"] = lib - crate.cargo_toml.write_text(tomlkit.dumps(cargo_toml)) - - # Export cdylib - if not crate.is_bin and cfg.reexport_lib: - cargo_toml = tomlkit.loads(crate.cargo_toml.read_text()) - lib = cargo_toml.get("lib", dict()) - lib.update({"crate-type": ["lib", "cdylib"]}) - cargo_toml["lib"] = lib - crate.cargo_toml.write_text(tomlkit.dumps(cargo_toml)) - - # Disable lints - cargo_toml = tomlkit.loads(crate.cargo_toml.read_text()) - lints = tomlkit.table(is_super_table=True) - lints.add("rust", {"nonstandard_style": "allow"}) - cargo_toml["lints"] = lints - crate.cargo_toml.write_text(tomlkit.dumps(cargo_toml)) - - # Configure testing - crate.cargo_nextest_config() - crate.invalidate_metadata() - - # Add cargo, workspace cargo, hydra log directory to VCS - crate.vcs.add(crate.cargo_toml, crate.rust_src_path) - if crate.metadata.get("workspace_root", None): - crate.vcs.add(Path(crate.metadata["workspace_root"]) / "Cargo.toml") - if (output_subdir := HydraConfig.get().output_subdir) is not None: - crate.vcs.add(output_dir / output_subdir) - msg = f"Initialized crate `{crate.root_package['name']}`" - logger.info(msg) - crate.vcs.commit(msg) - - -@hydra.main(version_base=None, config_name="init.crate") -def main(cfg: CrateConfig) -> None: - try: - _main(cfg) - except Exception as e: - logger.exception(e) - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/src/ideas/init/workspace.py b/src/ideas/init/workspace.py deleted file mode 100644 index a8f27dc..0000000 --- a/src/ideas/init/workspace.py +++ /dev/null @@ -1,71 +0,0 @@ -# -# Copyright (C) 2026 Intel Corporation -# -# SPDX-License-Identifier: Apache-2.0 -# - - -import sys -import logging - -from textwrap import dedent as d -from dataclasses import dataclass -from pathlib import Path - -import hydra -from omegaconf import MISSING -from hydra.core.config_store import ConfigStore - -from ideas.tools import Workspace - -logger = logging.getLogger("ideas.init.workspace") - - -@dataclass -class WorkspaceConfig: - cargo_toml: Path = MISSING - vcs: str = "none" - - def __post_init__(self): - if self.vcs not in ["git", "none"]: - raise ValueError(f"Invalid VCS: {self.vcs}!") - - -cs = ConfigStore.instance() -cs.store(name="init.workspace", node=WorkspaceConfig) - - -def _main(cfg: WorkspaceConfig) -> None: - # Initialize workspace - workspace = Workspace(cfg.cargo_toml, vcs=cfg.vcs) # type: ignore[reportArgumentType] - - if cfg.vcs == "git": - # Write .gitignore - (cfg.cargo_toml.parent / ".gitignore").write_text( - d(""" - Cargo.lock - target/ - *.log - *.jsonl - """).strip() - ) - - # Commit initial repo - workspace.vcs.add(Path("Cargo.toml")) - workspace.vcs.add(Path(".gitignore")) - msg = "Created cargo workspace" - logger.info(msg) - workspace.vcs.commit(msg) - - -@hydra.main(version_base=None, config_name="init.workspace") -def main(cfg: WorkspaceConfig) -> None: - try: - _main(cfg) - except Exception as e: - logger.exception(e) - sys.exit(1) - - -if __name__ == "__main__": - main() diff --git a/src/ideas/model.py b/src/ideas/model.py index cbeac35..34f30d0 100644 --- a/src/ideas/model.py +++ b/src/ideas/model.py @@ -4,16 +4,25 @@ # SPDX-License-Identifier: Apache-2.0 # +import logging from typing import Any from dataclasses import dataclass import dspy +from omegaconf import MISSING +from litellm import cost_per_token + from hydra.core.config_store import ConfigStore +# Surface DSPy logging to Hydra and disable verbose to sys.stderr to avoid duplicates +dspy_logger = logging.getLogger("dspy") +dspy_logger.propagate = True +dspy.disable_logging() + @dataclass class ModelConfig: - name: str = "Qwen/Qwen2.5-Coder-7B-Instruct" + name: str = MISSING cache: bool = False text_output: bool = True revision: str | None = None @@ -27,6 +36,7 @@ class GenerateConfig: temperature: float = 0.0 top_p: float = 1.0 top_k: int | None = None + timeout: int | None = 600 cs = ConfigStore.instance() @@ -42,6 +52,7 @@ def get_lm(model: ModelConfig, generate: GenerateConfig) -> dspy.LM: api_base=model.base_url, temperature=generate.temperature, max_tokens=generate.max_new_tokens, + timeout=generate.timeout, ) # Add OpenRouter-specific provider routing: https://openrouter.ai/docs/features/provider-routing @@ -61,6 +72,18 @@ def get_lm(model: ModelConfig, generate: GenerateConfig) -> dspy.LM: if model.text_output: lm.kwargs["reasoning"] = {"exclude": True} # type: ignore[reportArgumentType] + # NOTE: Covers exotic variants like gpt-5.5-pro + if model.name.startswith(("openai/gpt-5.5", "openai/gpt-5.6")): + lm.kwargs["temperature"] = 1.0 + # Choices: "none", "low", "medium" (default), "high", "xhigh", "max" + lm.kwargs["reasoning_effort"] = "high" # type: ignore[reportArgumentType] + + if model.name.startswith("hosted_vllm/zai-org/GLM"): + # Choices: enable_thinking: False, "high", "max" (default) + lm.kwargs["extra_body"] = { # type: ignore[reportArgumentType] + "chat_template_kwargs": {"reasoning_effort": "high"}, + } + return lm @@ -86,6 +109,18 @@ def format_usage(pred: dspy.Prediction) -> str: total_tokens = usage.get("total_tokens") or (prompt_tokens + completion_tokens) cost_usd = usage.get("cost") or usage.get("cost_usd") or usage.get("total_cost") + # Compute costs using litellm + if cost_usd is None and lm_usage: + cost_usd = 0.0 + for model_name, per_lm in lm_usage.items(): + pt = per_lm.get("prompt_tokens") or per_lm.get("input_tokens") or 0 + ct = per_lm.get("completion_tokens") or per_lm.get("output_tokens") or 0 + try: + pc, cc = cost_per_token(model_name, prompt_tokens=pt, completion_tokens=ct) + except Exception: + pc, cc = 0.0, 0.0 + cost_usd += pc + cc + cost = f"${cost_usd:.4f}, " if cost_usd is not None else "" return f"{cost}{total_tokens:,} tok ({prompt_tokens:,} in / {completion_tokens:,} out)" diff --git a/src/ideas/test_symbol.py b/src/ideas/test_symbol.py deleted file mode 100644 index 69d7a33..0000000 --- a/src/ideas/test_symbol.py +++ /dev/null @@ -1,139 +0,0 @@ -# -# Copyright (C) 2026 Intel Corporation -# -# SPDX-License-Identifier: Apache-2.0 -# - -import json -import logging - -import dspy - -from ideas.tools import Crate -from ideas.ast import Symbol -from ideas.init.build import write_main_binding, CodeRust - -logger = logging.getLogger("ideas.test_symbol") - - -class SymbolTester(dspy.Module): - def __init__(self, crate: Crate, symbols: list[Symbol], tests: str): - super().__init__() - self.crate = crate - self.tests = tests - self.main_function: CodeRust | None = None - - for symbol in symbols: - if not (symbol.is_function and symbol.is_definition and symbol.is_global): - continue - if self.crate.is_bin and symbol.spelling == "main": - # main requires special handling because we must bind to it as _main and - # statically create a Rust main that calls it - self.main_function = write_main_binding(crate) - - def test( - self, tests: str, skip: list[str] | None = None - ) -> tuple[bool, dict[str, bool], str]: - rust_src = self.crate.rust_src_path.read_text() - - # Remove forbid unsafe from Rust source - rust_src = rust_src.replace("#![forbid(unsafe_code)]", "") - - # Reference wrapper module in Rust source - WRAPPER_MOD = "pub mod wrapper;" - if WRAPPER_MOD not in rust_src: - rust_src += WRAPPER_MOD + "\n" - wrapper_path = self.crate.rust_src_path.parent / "wrapper.rs" - wrapper_path.touch() - - # Reference binding module in Rust source - BINDING_MOD = "pub mod binding;" - if BINDING_MOD not in rust_src: - rust_src += BINDING_MOD + "\n" - binding_path = self.crate.rust_src_path.parent / "binding.rs" - binding_path.touch() - - self.crate.rust_src_path.write_text(rust_src) - - # Try building the crate to detect if we need to insert a main - builds, feedback = self.crate.cargo_build(fix_E0601=False) - if "error[E0601]" in feedback and self.main_function is not None: - with self.crate.rust_src_path.open("a+") as f: - f.write(str(self.main_function)) - - self.crate.vcs.add(wrapper_path, binding_path, self.crate.rust_src_path) - - # Make sure the crate builds before testing - builds, feedback = self.crate.cargo_build(fix_E0601=False) - if not builds: - raise RuntimeError(f"Crate does not build!\n{feedback}") - - passes, jsonl, error, _ = self.crate.cargo_test( - tests, skip=skip, test_harness="nextest run", message_format="libtest-json" - ) - results = _extract_test_results(jsonl) - return passes, results, error - - def forward(self, symbol: Symbol, skip: list[str] | None = None) -> dspy.Prediction: - logger.info(f"Testing symbol `{symbol.name}` ....") - - # These files are modified by test - binding_path = self.crate.rust_src_path.parent / "binding.rs" - orig_binding_src = binding_path.read_bytes() - orig_rust_src = self.crate.rust_src_path.read_bytes() - - # Run cargo test - passes, results, feedback = self.test(self.tests, skip=skip) - if passes: - msg = f"Tested symbol `{symbol.name}`" - logger.info(msg) - else: - feedback = "Running `cargo test` fails!\n" + feedback - msg = f"Failed to test symbol `{symbol.name}`" - logger.error(msg) - msg += f"\n\n{feedback}" - self.crate.vcs.commit(msg) - - # Restore originals - binding_path.write_bytes(orig_binding_src) - self.crate.rust_src_path.write_bytes(orig_rust_src) - - pred = dspy.Prediction( - success=passes, - output=feedback, - results=results, - feedback="", - ) - - if not passes: - # FIXME: Use test feedback? - pred.feedback = ( - "The current Rust translation in `prior_translation` does not match the behavior of the C `snippet`. " - "Carefully compare `prior_translation` against the C `snippet` and regenerate the Rust `translation` to match the C behavior exactly. " - "Do not assume inputs are well-formed: if the tests exercise malformed, invalid, partial, or adversarial input, preserve the C behavior for those cases too, including error returns, boundary handling, or other observable effects. " - "Make minimal, targeted changes to `prior_translation`, and only modify what is necessary to match the C behavior. " - "Treat the C `snippet` as the source of truth, even if it contains a bug." - ) - - return pred - - -def _extract_test_results(output: str) -> dict[str, bool]: - test_results: dict[str, bool] = {} - - for line in output.splitlines(): - try: - obj = json.loads(line) - except json.JSONDecodeError: - continue - if obj.get("type") != "test": - continue - event = obj.get("event") - if event not in {"ok", "failed", "ignored"}: - continue - name = str(obj.get("name", "")).rsplit("$", 1)[-1].strip() - if name: - # Treat ignored as non-failing for disable-list purposes - test_results[name] = event != "failed" - - return test_results diff --git a/src/ideas/tools.py b/src/ideas/tools.py index e2aaf5e..a111ef7 100644 --- a/src/ideas/tools.py +++ b/src/ideas/tools.py @@ -17,15 +17,9 @@ from tempfile import TemporaryDirectory from pathlib import Path -from .ast import CodeC - - -TestCase = dict[str, None | str | int | float | list[int] | list[str] | list[float]] logger = logging.getLogger("ideas.tools") -DEFAULT_TEST_TIMEOUT = 10.0 # seconds - class VCS: def __init__( @@ -108,39 +102,22 @@ def __call__(self, cmd, *args, **kwargs) -> tuple[bool, str]: return success, output + error -class Workspace: - def __init__( - self, - cargo_toml: Path, - vcs: Literal["none", "git"] = "none", - ): - self.cargo_toml = cargo_toml.resolve() - - workspace_dir = self.cargo_toml.parent - self.vcs = VCS(repo_dir=workspace_dir, vcs=vcs) - - if not self.cargo_toml.exists(): - # Create a new workspace - os.makedirs(workspace_dir, exist_ok=True) - contents = {"workspace": {"resolver": "3"}} - self.cargo_toml.write_text(tomlkit.dumps(contents)) - - # Initialize repository if needed - self.vcs.init(force_init=True) - - class Crate: def __init__( self, cargo_toml: Path, vcs: Literal["none", "git"] = "none", template: Literal["bin", "lib"] | None = None, + reinit: bool = False, ): self.cargo_toml = cargo_toml.resolve() crate_dir = self.cargo_toml.parent self.vcs = VCS(repo_dir=crate_dir, vcs=vcs) + if reinit and self.cargo_toml.exists(): + self.cargo_toml.unlink() + if not self.cargo_toml.exists(): # Create a new crate with specified template, but without VCS if not template: @@ -206,36 +183,55 @@ def bin_targets(self) -> list[dict[str, Any]]: return list(filter(lambda t: "bin" in t["kind"], self.root_package["targets"])) @property - def lib_targets(self) -> list[dict[str, Any]]: - return list(filter(lambda t: "lib" in t["kind"], self.root_package["targets"])) + def lib_target(self) -> dict[str, Any] | None: + all_targets = list(filter(lambda t: "lib" in t["kind"], self.root_package["targets"])) + if len(all_targets) == 1: + return all_targets[0] + if len(all_targets) == 0: + return None + raise ValueError(f"Multiple lib targets found in Cargo.toml: {self.cargo_toml=}") @property - def is_bin(self) -> bool: - if len(self.bin_targets) == 1 and len(self.lib_targets) == 0: - is_bin = True - elif len(self.bin_targets) == 0 and len(self.lib_targets) == 1: - is_bin = False - else: - raise ValueError( - f"Unhandled bin/lib targets configuration in Cargo.toml: {self.bin_targets=} {self.lib_targets=}" + def lib_name(self) -> str | None: + if self.lib_target is None: + return None + return self.lib_target["name"] + + @property + def lib_src_path(self) -> Path | None: + if self.lib_target is None: + return None + return Path(self.lib_target["src_path"]) + + @property + def main_src_path(self) -> Path | None: + if len(self.bin_targets) == 1: + return Path(self.bin_targets[0]["src_path"]) + if len(self.bin_targets) > 1: + raise NotImplementedError( + f"Multiple bin targets found in Cargo.toml: {self.cargo_toml=} {self.bin_targets=}" ) - return is_bin + return None @property - def rust_src_path(self) -> Path: - if len(self.bin_targets) == 1 and len(self.lib_targets) == 0: - rust_src_path = Path(self.bin_targets[0]["src_path"]) - elif len(self.bin_targets) == 0 and len(self.lib_targets) == 1: - rust_src_path = Path(self.lib_targets[0]["src_path"]) - else: + def src_dir(self) -> Path: + src_paths: list[Path] = [] + if self.lib_src_path is not None: + src_paths.append(self.lib_src_path.parent) + if self.main_src_path is not None: + src_paths.append(self.main_src_path.parent) + + if not src_paths: + raise ValueError(f"Crate {self.name} has neither lib.rs nor main.rs!") + if len({path.resolve() for path in src_paths}) > 1: raise ValueError( - f"Unhandled bin/lib targets configuration in Cargo.toml: {self.bin_targets=} {self.lib_targets=}" + f"Crate {self.name} has inconsistent src directories: {src_paths!r}" ) - return rust_src_path + return src_paths[0] @property - def c_src_path(self) -> Path: - return self.rust_src_path.with_suffix(".c") + def name(self) -> str: + return self.root_package["name"] def cargo_add( self, dep: str, section: str | None = None, features: list[str] | None = None @@ -274,6 +270,30 @@ def cargo_feature(self, **features: list[str]) -> None: # Invalidate cached metadata self.invalidate_metadata() + def configure_target( + self, + section: Literal["bin", "lib"], + name: str, + test: bool = True, + doctest: bool = True, + crate_type: list[str] | None = None, + ) -> None: + content = tomlkit.loads(self.cargo_toml.read_text()) + table = tomlkit.table() + table.add("name", name) + if crate_type is not None: + table.add("crate-type", crate_type) + table.add("test", test) + table.add("doctest", doctest) + # A crate can have multiple binaries, so [[bin]] must be an array of tables + if section == "bin": + aot = tomlkit.aot() + aot.append(table) + table = aot + content[section] = table + self.cargo_toml.write_text(tomlkit.dumps(content)) + self.invalidate_metadata() + def add_workspace_dependencies(self, names: list[str]) -> None: if not names: return @@ -332,7 +352,7 @@ def cargo_clean(self) -> None: f"Failed to clean crate at {self.cargo_toml} with error:\n\n{output + error}" ) - def cargo_build(self, fix_E0601: bool = True) -> tuple[bool, str]: + def cargo_build(self) -> tuple[bool, str]: cmd = [ "cargo", "build", @@ -341,15 +361,6 @@ def cargo_build(self, fix_E0601: bool = True) -> tuple[bool, str]: f"--manifest-path={self.cargo_toml}", ] builds, output, error, _ = run_subprocess(cmd) - - # Work around E0601 error "No main function was found in a binary crate." - if fix_E0601 and "error[E0601]" in error: - rust_src = self.rust_src_path.read_text() - with self.rust_src_path.open("a") as f: - f.write('\n\nfn main() {\n println!("Hello, world!");\n}\n') - builds, output, error, _ = run_subprocess(cmd) - self.rust_src_path.write_text(rust_src) - return builds, output + error def cargo_test( @@ -361,6 +372,7 @@ def cargo_test( build_only: bool = False, skip: list[str] | None = None, message_format: str | None = None, + lib: bool = False, ) -> tuple[bool, str, str, int | Literal["timeout"]]: cmd = [ "cargo", @@ -377,8 +389,12 @@ def cargo_test( cmd.append("--quiet") else: raise ValueError(f"Unsupported test harness: {test_harness}") - if name: - cmd.extend(["--test", name]) + if lib: + cmd.append("--lib") + if name: + cmd.append(name) # positional substring filter + elif name: + cmd.extend(["--test", name]) # integration test binary if build_only: cmd.append("--no-run") @@ -398,19 +414,16 @@ def cargo_test( cmd.append("--exact") for test_name in skip: cmd.extend(["--skip", test_name]) - return run_subprocess(cmd, env=env) - def cargo_nextest_config(self, slow: int = 30, terminate_after: int = 4) -> None: - nextest_config_path = self.workspace_root / ".config" / "nextest.toml" - nextest_config_path.parent.mkdir(exist_ok=True) + def cargo_nextest_config( + self, nextest_config_path: Path, slow: int = 30, terminate_after: int = 2 + ) -> None: + nextest_config_path.parent.mkdir(parents=True, exist_ok=True) nextest_config = { "profile": { "default": { "slow-timeout": {"period": f"{slow}s", "terminate-after": terminate_after}, - "final-status-level": "none", - "fail-fast": False, - "failure-output": "never", "test-threads": 1, } } @@ -425,38 +438,6 @@ def write(self, path: Path, data, **kwargs): return path.write_text(data, **kwargs) -def nextest_json_to_libtest(stdout: str) -> str: - """Convert nextest libtest-json output to vanilla `cargo test` text format.""" - lines = [] - summary = {} - for raw in stdout.splitlines(): - obj = json.loads(raw) - - if obj.get("type") == "test": - event = obj.get("event") - if event not in {"ok", "failed", "ignored"}: - continue - - # nextest uses "$" to join binary::suite$test_name - name = obj["name"].rsplit("$", 1)[-1] - status = "FAILED" if event == "failed" else event - lines.append(f"test {name} ... {status}") - - elif obj.get("type") == "suite" and obj.get("event") != "started": - summary = obj - - # Append summary from the suite event (or zeros if missing) - p, f = summary.get("passed", 0), summary.get("failed", 0) - ig, m = summary.get("ignored", 0), summary.get("measured", 0) - fo = summary.get("filtered_out", 0) - result = "FAILED" if f else "ok" - lines.append( - f"test result: {result}. {p} passed; {f} failed; " - f"{ig} ignored; {m} measured; {fo} filtered out" - ) - return "\n".join(lines) + "\n" - - def run_subprocess( cmd: list[str], input: str | None = None, @@ -485,26 +466,6 @@ def run_subprocess( ) -def check_c( - code: CodeC, - *, - flags: list[str] | None = None, -) -> tuple[bool, str]: - cmd = ["clang-21"] - - if flags: - cmd.extend(flags) - else: - cmd.append("-Wall") - - cmd.extend(["-march=native", "-x", "c"]) - cmd.append("-") - cmd.extend(["-o", "/dev/null"]) - - success, output, error, _ = run_subprocess(cmd, input=str(code)) - return success, output + error - - def check_rust( code: str, *, @@ -531,69 +492,5 @@ def rustfmt(path: Path) -> None: run_subprocess(cmd) -def run_test( - executable: Path | str, - test_case: TestCase, - timeout: float | None = DEFAULT_TEST_TIMEOUT, -) -> tuple[bool, str]: - # Turn args into list[str] - args = test_case.get("args", []) or [] - if not isinstance(args, list): - args = [args] - args = [str(arg) for arg in args] - - # Turn stdin into list[str] then join on newlines - stdin = test_case.get("in", []) or [] - if not isinstance(stdin, list): - stdin = [stdin] - stdin = [str(s) for s in stdin] - stdin = "\n".join(stdin) - - # Run test and right-strip output of whitespace - success, output, error, _ = run_subprocess([str(executable), *args], stdin, timeout=timeout) - return success, output + error - - -def check_test( - test_case: TestCase, - stdout: str, -) -> bool: - # Turn out into list[str] then join on newlines - out = test_case["out"] - if not isinstance(out, list): - out = [out] - out = [str(o) for o in out] - if isinstance(out, list): - out = "\n".join(out) - - # Make sure test returned and matches - return out.rstrip() == stdout.rstrip() - - -def run_and_check_test( - executable: Path | str, - test_case: TestCase, - timeout: float | None = DEFAULT_TEST_TIMEOUT, -): - _, stdout = run_test(executable, test_case, timeout=timeout) - return check_test(test_case, stdout) - - -def run_and_check_tests( - executable: Path | str, - test_cases: list[TestCase], - timeout: float | None = DEFAULT_TEST_TIMEOUT, -) -> int: - success = 0 - for test_case in test_cases: - success += 1 if run_and_check_test(executable, test_case, timeout=timeout) else 0 - return success - - -def _in_env(var_name: str, default: bool = True) -> bool: - value = os.getenv(var_name, str(default)) - return value.strip().lower() in {"1", "true", "yes", "on"} - - -LARGE_PROJECT = _in_env("LARGE_PROJECT", default=False) MAX_DEPENDENT_CHARS = int(os.environ.get("MAX_DEPENDENT_CHARS", "20000")) +REDUCED_CONTEXT = os.environ.get("REDUCED_CONTEXT", "1") not in ("0", "", "false", "False") diff --git a/src/ideas/translate.py b/src/ideas/translate.py index 1f856a0..f4b3b96 100644 --- a/src/ideas/translate.py +++ b/src/ideas/translate.py @@ -16,25 +16,28 @@ from hydra.core.hydra_config import HydraConfig from ideas import adapters, model, ModelConfig, GenerateConfig -from ideas import SnippetTranslator, RecurrentTranslator, WrapperGenerator, SymbolTester +from ideas import SnippetTranslator, RecurrentTranslator, WrapperGenerator from ideas import create_translation_unit, extract_info_c -from ideas.init.consolidate import get_symbols_and_dependencies -from .tools import Crate, LARGE_PROJECT +from ideas.consolidate import get_symbols_and_dependencies +from .tools import Crate logger = logging.getLogger("ideas.translate") @dataclass class TranslateConfig: - filename: Path = MISSING model: ModelConfig = field(default_factory=ModelConfig) generate: GenerateConfig = field(default_factory=GenerateConfig) cargo_toml: Path = MISSING - tests: str = MISSING + bindings_cargo_toml: Path = MISSING + tests: str | None = MISSING + template: str = "bin" + deps: list[str] = field(default_factory=list) translator: str = "ChainOfThought" translator_max_iters: int = 5 + wrapper: str = "ChainOfThought" wrapper_max_iters: int = 5 max_iters: int = 3 @@ -45,81 +48,149 @@ class TranslateConfig: cs.store(name="translate", node=TranslateConfig) +def _init_crates(cfg: TranslateConfig) -> tuple[Crate, Crate, Crate]: + sys_crate = Crate(cfg.bindings_cargo_toml, vcs=cfg.vcs) # type: ignore[reportArgumentType] + + # Create fresh -rs (pure Rust translation) crate Cargo.toml so cargo init always runs and registers the crate in workspace.members + rs_cargo_toml = Path(str(cfg.cargo_toml.parent) + "-rs") / "Cargo.toml" + rs_crate = Crate(rs_cargo_toml, vcs=cfg.vcs, template=cfg.template, reinit=True) # type: ignore[reportArgumentType] + # Binary -rs crates always have a lib.rs file with all the functions (including main) translated to safe Rust + if cfg.template == "bin": + assert rs_crate.main_src_path is not None, "Expected main.rs to exist in -rs crate!" + (rs_crate.main_src_path.parent / "lib.rs").touch() + rs_crate.invalidate_metadata() + + for dep in cfg.deps: + rs_crate.cargo_add(dep) + rs_crate.vcs.add(rs_crate.cargo_toml.parent) + workspace_cargo_toml = rs_crate.workspace_root / "Cargo.toml" + if workspace_cargo_toml != rs_crate.cargo_toml and workspace_cargo_toml.exists(): + rs_crate.vcs.add(workspace_cargo_toml) + rs_crate.vcs.commit(f"Created Rust translation crate '{rs_crate.name}'") + + # Create fresh hybrid crate (links -rs + -sys together) Cargo.toml so cargo init always runs and registers the crate in workspace.members + crate = Crate(cfg.cargo_toml, vcs=cfg.vcs, template=cfg.template, reinit=True) # type: ignore[reportArgumentType] + crate.add_workspace_dependencies([rs_crate.name, sys_crate.name]) + for dep in cfg.deps: + crate.cargo_add(dep) + + for dep in sys_crate.root_package["dependencies"]: + if dep["kind"] == "dev": + name_req = dep["name"] + "@" + dep["req"] + crate.cargo_add(name_req, section="dev", features=dep["features"]) + if cfg.template == "bin": + crate.configure_target("bin", name=crate.name, test=False, doctest=False) + # Ensure binary hybrid crates also expose a lib target so wrapper module unit + # tests are discoverable by cargo test/nextest. + assert crate.main_src_path is not None, "Expected main.rs to exist in hybrid crate!" + (crate.main_src_path.parent / "lib.rs").touch() + crate.invalidate_metadata() + elif cfg.template == "lib": + crate.configure_target( + "lib", + name=crate.name.removeprefix("lib"), + test=False, + doctest=False, + crate_type=["lib", "cdylib"], + ) + else: + raise NotImplementedError(f"Unsupported template: {cfg.template!r}") + crate.vcs.add(crate.cargo_toml.parent) + + # Copy the translation test from the -sys crate; skipped entirely when tests=null + if cfg.tests is not None: + sys_test_src = ( + cfg.bindings_cargo_toml.parent / "tests" / f"{cfg.tests}.rs" + ).read_text() + test_path = crate.cargo_toml.parent / "tests" / f"{cfg.tests}.rs" + test_path.parent.mkdir(parents=True, exist_ok=True) + # Prepend `use as _;` so the hybrid crate's `#[export_name]` functions are + # retained by the linker. The translation loop makes each C function extern-only + # (via clang_make_extern_), so if these Rust implementations are dead-code-eliminated + # the test binary will fail to link with unresolved symbol errors. + # FIXME: This could be removed if library tests were portable like binary tests + if cfg.template == "lib": + sys_test_src = f"use {crate.lib_name} as _;\n" + sys_test_src + test_path.write_text(sys_test_src) + crate.vcs.add(test_path) + + workspace_cargo_toml = crate.workspace_root / "Cargo.toml" + if workspace_cargo_toml != crate.cargo_toml and workspace_cargo_toml.exists(): + crate.vcs.add(workspace_cargo_toml) + crate.vcs.commit(f"Created hybrid crate '{crate.name}'") + + return crate, rs_crate, sys_crate + + def _main(cfg: TranslateConfig) -> None: output_dir = Path(HydraConfig.get().runtime.output_dir) logger.info(f"Saving results to {output_dir}") - crate = Crate(cfg.cargo_toml, vcs=cfg.vcs) # type: ignore[reportArgumentType] - - # Save C source since it will be modified by the agent - orig_c_src = crate.c_src_path.read_bytes() - # Make sure Rust source is in known state (i.e., empty) - crate.rust_src_path.write_text("") + crate, rs_crate, sys_crate = _init_crates(cfg) # Get global symbol table - tu = create_translation_unit(cfg.filename) + assert sys_crate.lib_src_path is not None, "Expected lib.rs to exist in -sys crate!" + c_src_path = sys_crate.lib_src_path.with_suffix(".c") + tu = create_translation_unit(c_src_path) asts = [extract_info_c(tu)] symbols, dependencies = get_symbols_and_dependencies( - asts, external_symbol_names=["c:@F@main"] if crate.is_bin else None + asts, external_symbol_names=["c:@F@main"] if cfg.template == "bin" else None ) # Create translation agent model.configure(cfg.model, cfg.generate) dspy.configure(adapter=adapters.ChatAdapter()) translator = getattr(dspy, cfg.translator) - snippet_translator = SnippetTranslator(crate, translator, cfg.translator_max_iters) - symbol_wrapper = WrapperGenerator(crate, cfg.wrapper_max_iters) - symbol_tester = None - if not LARGE_PROJECT: - symbol_tester = SymbolTester(crate, symbols=list(symbols.values()), tests=cfg.tests) + wrapper = getattr(dspy, cfg.wrapper) + cache = crate.workspace_root / "cache.db" + symbol_wrapper = None + tests = None + if cfg.wrapper_max_iters > 0: + symbol_wrapper = WrapperGenerator(wrapper, cfg.wrapper_max_iters, cache=cache) + tests = cfg.tests agent = RecurrentTranslator( - crate, snippet_translator, symbol_wrapper, symbol_tester, cfg.max_iters + sys_crate=sys_crate, + crate=crate, + rs_crate=rs_crate, + symbol_translator=SnippetTranslator(translator, cfg.translator_max_iters, cache=cache), + symbol_wrapper=symbol_wrapper, + tests=tests, + max_iters=cfg.max_iters, ) + msg = f"Initialized translation of {cfg.template} `{crate.name}` ({len(symbols)} symbols)" + logger.info(msg) + crate.vcs.commit(msg) + # Run translation agent and write it to disk try: pred = agent(symbols, dependencies) - crate.rust_src_path.write_text(str(pred.translation)) except Exception as e: logger.exception(e) pred = dspy.Prediction(success=False) - usage = model.format_usage(pred) - if pred.success: - msg = f"Translated `{crate.root_package['name']}` to Rust: {usage}" - logger.info(msg) - else: - # Restore original C code so next agent can use it - crate.c_src_path.write_bytes(orig_c_src) - msg = f"Failed to translate `{crate.root_package['name']}`: {usage}" + usage = model.format_usage(pred) + if not pred.success: + msg = f"Failed to translate `{crate.name}`: {usage}" logger.error(msg) - # Clean up intermediate artifacts produced during translation - _cleanup(crate) + # Force test failures by stubbing out the hybrid crate + if crate.lib_src_path is not None: + crate.lib_src_path.write_text("\n") + crate.vcs.add(crate.lib_src_path) + if crate.main_src_path is not None: + crate.main_src_path.write_text("\n") + crate.vcs.add(crate.main_src_path) + else: + msg = f"Translated {cfg.template} `{crate.name}` to Rust: {usage}" + logger.info(msg) # Commit translation if (output_subdir := HydraConfig.get().output_subdir) is not None: crate.vcs.add(output_dir / output_subdir) - crate.vcs.add(crate.rust_src_path, crate.c_src_path) crate.vcs.commit(msg) -def _cleanup(crate: Crate) -> None: - # Remove bindgen artifacts - crate.vcs.rm( - crate.rust_src_path.parent / "binding", - crate.rust_src_path.parent / "binding.rs", - force=True, - ) - logger.info("Removed bindgen artifacts") - - # For binaries, delete wrappers - if crate.is_bin: - wrapper_dir = crate.rust_src_path.parent / "wrapper" - wrapper_module = crate.rust_src_path.parent / "wrapper.rs" - crate.vcs.rm(wrapper_module, wrapper_dir, force=True) - - @hydra.main(version_base=None, config_name="translate") def main(cfg: TranslateConfig) -> None: try: diff --git a/src/ideas/translate_recurrent.py b/src/ideas/translate_recurrent.py index 4302e54..cdb715c 100644 --- a/src/ideas/translate_recurrent.py +++ b/src/ideas/translate_recurrent.py @@ -4,62 +4,428 @@ # SPDX-License-Identifier: Apache-2.0 # +import re +import math +import json import logging from pathlib import Path +from dataclasses import dataclass, field from collections.abc import Iterable +from textwrap import indent +from typing import Literal import dspy import networkx as nx -from .ast import CodeC, Symbol, TreeResult -from .ast_rust import CodeRust, get_signatures -from .tools import Crate, LARGE_PROJECT, MAX_DEPENDENT_CHARS -from .init.consolidate import create_symbol_lexical_key_fn +from .ast_rust import CodeRust, strip_fns, mangle +from .tools import Crate, MAX_DEPENDENT_CHARS, REDUCED_CONTEXT +from .ast import Symbol, SymbolName, SymbolGroup +from .ast import CodeC, TreeResult, create_symbol_ordering_key_fn +from .ast import clang_make_global_, clang_make_extern_ +from .wrapper import bindgen, generate_unimplemented_function_wrapper +from .wrapper import generate_unimplemented_type_wrapper logger = logging.getLogger("ideas.translate_recurrent") -SymbolName = str -SymbolGroup = tuple[SymbolName, ...] + +WrapperName = str + + +@dataclass(frozen=True) +class TranslationContext: + crate_code: CodeRust + reference_code: CodeRust + dependent_code: CodeC + support_code: CodeC + wrappers: CodeRust + + @classmethod + def build( + cls, + G: nx.DiGraph, + group: SymbolGroup, + groups: list[SymbolGroup], + symbols: dict[SymbolName, Symbol], + translations: dict[SymbolGroup, CodeRust] | None = None, + wrappers: dict[WrapperName, CodeRust] | None = None, + ) -> "TranslationContext": + if translations is None: + translations = {} + if wrappers is None: + wrappers = {} + descendants = nx.descendants(G, group) + already_translated = [g for g in groups if g in descendants] + immediate_already_translated = set(G.successors(group)) + ancestors = nx.ancestors(G, group) + ancestor_deps = {succ for a in ancestors for succ in G.successors(a)} + to_be_translated = [ + g for g in groups if g in ancestors | (ancestor_deps - descendants - {group}) + ] + reference_groups = [g for g in groups if g in translations] + hops = nx.single_source_shortest_path_length(G, group) + return cls( + crate_code=cls._build_crate_code(reference_groups, translations), + reference_code=cls._build_reference_code(reference_groups, translations, hops), + dependent_code=cls._build_dependent_code(to_be_translated, symbols), + support_code=cls._build_support_code( + already_translated, immediate_already_translated, symbols + ), + wrappers=cls._build_wrapper_context(wrappers), + ) + + @staticmethod + def _build_crate_code( + reference_groups: list[SymbolGroup], + translations: dict[SymbolGroup, CodeRust], + ) -> CodeRust: + # Use all unique (dict.fromkeys) translations as the crate's current contents since many symbol names can map to the same translation + return CodeRust.join(dict.fromkeys(translations[g] for g in reference_groups)) + + @staticmethod + def _build_reference_code( + reference_groups: list[SymbolGroup], + translations: dict[SymbolGroup, CodeRust], + hops: dict[SymbolGroup, int], + ) -> CodeRust: + def trim_by_distance(ref_group: SymbolGroup) -> CodeRust: + match hops.get(ref_group): + case 1: + # 1-hop successors keep full function bodies since they are likely to be directly relevant + return translations[ref_group] + case 2: + # 2-hop successors strip top-level function bodies since they are less likely to be directly relevant + return strip_fns(translations[ref_group]) + case _: + # For distant or unreachable groups delete top-level functions but keep types, + # since types may still be needed even when not reachable via static analysis + return strip_fns(translations[ref_group], delete=True) + + return CodeRust.join(dict.fromkeys(trim_by_distance(g) for g in reference_groups)) + + @staticmethod + def _build_support_code( + already_translated: list[SymbolGroup], + immediate_already_translated: set[SymbolGroup], + symbols: dict[SymbolName, Symbol], + ) -> CodeC: + # Gather support code in topological order. + # Reduce C support code context by turning non-immediate symbols into declarations. We keep + # immediate C code in full since they are more likely to be relevant for wrappers. + return CodeC.join( + symbols[name].code + if g in immediate_already_translated + else CodeC(symbols[name].llm_context_declaration) + for g in already_translated + for name in g + ) + + @classmethod + def _build_dependent_code( + cls, + to_be_translated: list[SymbolGroup], + symbols: dict[SymbolName, Symbol], + max_chars: int = MAX_DEPENDENT_CHARS, + ) -> CodeC: + # Gather dependent C code in topological order. + dependent_code = CodeC.join(symbols[name].code for g in to_be_translated for name in g) + if len(str(dependent_code)) > max_chars: + logger.warning(f"Dependent code exceeds max {len(str(dependent_code))}/{max_chars}") + dependent_code = cls._select_c_code(to_be_translated, symbols, max_chars) + return dependent_code + + _MEMORY_PATTERN = re.compile( + r"\b(malloc|calloc|realloc|free|memcpy|memmove|memset|strdup|strndup|fopen|freopen|fclose)\b" + ) + _POINTER_PATTERN = re.compile(r"->|\*|&|\[|\bNULL\b|\bsizeof\b") + + @dataclass(frozen=True) + class _DependentCandidate: + group: SymbolGroup + full: CodeC + full_chars: int + score: float + + @classmethod + def _select_c_code( + cls, + groups: list[SymbolGroup], + symbols: dict[SymbolName, Symbol], + max_chars: int, + ) -> CodeC: + candidates = cls._collect_dependent_candidates(groups, symbols) + chosen: set[SymbolGroup] = set() + total_chars = 0 + + # Select the most informative dependent bodies that fit within the remaining budget. + for candidate in sorted( + candidates, + key=lambda candidate: candidate.score / math.sqrt(max(candidate.full_chars, 1)), + reverse=True, + ): + if total_chars + candidate.full_chars > max_chars: + continue + chosen.add(candidate.group) + total_chars += candidate.full_chars + + return CodeC.join( + candidate.full for candidate in candidates if candidate.group in chosen + ) + + @classmethod + def _collect_dependent_candidates( + cls, + groups: list[SymbolGroup], + symbols: dict[SymbolName, Symbol], + ) -> list: + candidates = [] + for group in groups: + full = CodeC.join(symbols[name].code for name in group) + candidates.append( + cls._DependentCandidate( + group=group, + full=full, + full_chars=len(str(full)), + score=cls._score_dependent_group(group, symbols), + ) + ) + return candidates + + @classmethod + def _score_dependent_group( + cls, group: SymbolGroup, symbols: dict[SymbolName, Symbol] + ) -> float: + score = 0.0 + + # Favor groups with function definitions + if any(symbols[name].is_function and symbols[name].is_definition for name in group): + score += 3.0 + + # Favor groups with memory or pointer-related code patterns + code = "\n".join(str(symbols[name].code) for name in group) + if cls._MEMORY_PATTERN.search(code): + score += 3.0 + if cls._POINTER_PATTERN.search(code): + score += 2.0 + + # Favor smaller groups + return score + 1.0 / math.sqrt(max(len(code), 1)) + + @classmethod + def _build_wrapper_context(cls, wrappers: dict[WrapperName, CodeRust]) -> CodeRust: + context = CodeRust("") + for name, wrapper in wrappers.items(): + if "fn c_to_r" in str(wrapper): + wrapper = cls._build_type_wrapper_context(wrapper) + elif "pub static mut" in str(wrapper): + pass + elif REDUCED_CONTEXT: + wrapper = None + + if wrapper: + context += CodeRust( + f"pub mod {name} {{\n" + indent(str(wrapper), " " * 4) + "\n}" + ) + return context + + @classmethod + def _build_type_wrapper_context(cls, wrapper: CodeRust) -> CodeRust: + # Strip tests from type wrapper + wrapper_src = str(wrapper) + test_idx = wrapper_src.find("#[cfg(test)]") + if test_idx != -1: + wrapper_src = wrapper_src[:test_idx].strip() + wrapper = CodeRust(wrapper_src) + + # Strip functions from wrapper + wrapper = strip_fns(wrapper) + + return wrapper + + +@dataclass +class _State: + c_src: bytes + rust_lib_src: bytes + rust_main_src: bytes | None + hybrid_lib_src: bytes | None + hybrid_main_src: bytes | None + wrappers: dict[Path, bytes] + + +@dataclass +class _TranslationResult: + pred: dspy.Prediction = field(repr=False, compare=False) + + @property + def success(self) -> bool: + return self.pred.success + + @property + def translation(self) -> CodeRust | None: + return self.pred.translation if "translation" in self.pred else None + + @property + def feedback(self) -> str: + return self.pred.feedback if "feedback" in self.pred else "" + + @classmethod + def from_pred(cls, pred: dspy.Prediction) -> "_TranslationResult": + return cls(pred=pred) + + +@dataclass +class _WrapperResult: + pred: dspy.Prediction | None = field(default=None, repr=False, compare=False) + failure: Literal["wrap", "test"] | None = None + failed_tests: set[str] = field(default_factory=set) + + @property + def success(self) -> bool: + return self.failure is None + + @property + def wrapper(self) -> CodeRust | None: + return self.pred.wrapper if self.pred is not None and "wrapper" in self.pred else None + + @property + def feedback(self) -> str: + if self.failure is None: + return "" + + if self.failure == "wrap": + return ( + "It was difficult to generate a C-compatible FFI wrapper for the translation. " + "Regenerate the translation with clear, explicit, wrapper-friendly Rust function boundaries and straightforward ownership, " + "while keeping the translation fully memory-safe and free of unsafe constructs." + ) + + return ( + "The current Rust translation in `prior_translation` does not match the behavior of the C `snippet`. " + "Carefully compare `prior_translation` against the C `snippet` and regenerate the Rust `translation` to match the C behavior exactly. " + "Do not assume inputs are well-formed: if the tests exercise malformed, invalid, partial, or adversarial input, preserve the C behavior for those cases too, including error returns, boundary handling, or other observable effects. " + "Make minimal, targeted changes to `prior_translation`, and only modify what is necessary to match the C behavior. " + "Treat the C `snippet` as the source of truth, even if it contains a bug." + ) + + @classmethod + def from_pred(cls, pred: dspy.Prediction) -> "_WrapperResult": + failure: Literal["wrap", "test"] | None = None if pred.success else "wrap" + return cls(pred=pred, failure=failure) + + +@dataclass +class _Result: + translation_result: _TranslationResult + wrapper_results: dict[WrapperName, _WrapperResult] = field(default_factory=dict) + + @property + def translation(self) -> CodeRust | None: + return self.translation_result.translation + + @property + def failure(self) -> Literal["translate", "wrap", "test"] | None: + if not self.translation_result.success: + return "translate" + for r in self.wrapper_results.values(): + if r.failure is not None: + return r.failure + return None + + @property + def success(self) -> bool: + return self.failure is None + + @property + def failed_tests(self) -> set[str]: + return {t for r in self.wrapper_results.values() for t in r.failed_tests} + + @property + def feedback(self) -> str: + if not self.translation_result.success: + return self.translation_result.feedback + for r in self.wrapper_results.values(): + if not r.success: + return r.feedback + return "" + + @property + def wrappers(self) -> dict[WrapperName, CodeRust]: + return {n: r.wrapper for n, r in self.wrapper_results.items() if r.wrapper is not None} class RecurrentTranslator(dspy.Module): def __init__( self, + sys_crate: Crate, crate: Crate, + rs_crate: Crate, symbol_translator: dspy.Module, - symbol_wrapper: dspy.Module, - symbol_tester: dspy.Module | None = None, + symbol_wrapper: dspy.Module | None, + tests: str | None = None, max_iters: int = 1, ): super().__init__() + assert sys_crate.lib_src_path is not None + self.c_src_path = sys_crate.lib_src_path.with_suffix(".c") self.crate = crate - self.translate_symbol = symbol_translator - self.wrap_symbol = symbol_wrapper - self.test_symbol = symbol_tester + self._translator = symbol_translator + self._wrapper = symbol_wrapper + self._tests = tests self.max_iters = max_iters self._failed_tests: set[str] = set() + self._init_rust_crate(rs_crate) + self._init_hybrid_crate(sys_crate, crate) + + def _init_rust_crate(self, rs_crate: Crate): + if rs_crate.lib_src_path is None: + raise ValueError("Expected lib.rs to exist in -rs crate!") + if rs_crate.lib_name is None: + raise ValueError("Expected a library target in the -rs crate!") + self.rust_crate = rs_crate + rs_crate.lib_src_path.write_text("#![forbid(unsafe_code)]\n\n") + rs_crate.vcs.add(rs_crate.lib_src_path) + if rs_crate.main_src_path is not None: + rs_crate.main_src_path.write_text( + '#![forbid(unsafe_code)]\n\nfn main() { println!("main not yet translated"); }\n' + ) + rs_crate.vcs.add(rs_crate.main_src_path) + + def _init_hybrid_crate(self, sys_crate: Crate, crate: Crate): + if crate.lib_src_path is None and crate.main_src_path is None: + raise ValueError(f"Crate {crate.name} has neither lib.rs nor main.rs!") + + assert sys_crate.lib_name is not None + use_stmt = f"use {sys_crate.lib_name} as _;\n" + + if crate.lib_src_path is not None: + crate.lib_src_path.write_text(use_stmt) + crate.vcs.add(crate.lib_src_path) + if crate.main_src_path is not None: + crate.main_src_path.write_text(f"#![no_main]\n\n{use_stmt}\n\n") + crate.vcs.add(crate.main_src_path) + def forward( self, symbols: dict[SymbolName, Symbol], dependencies: dict[SymbolGroup, Iterable[SymbolGroup]], ast_order: dict[Path, TreeResult] | None = None, ) -> dspy.Prediction: - # We always start with an empty crate - self.crate.rust_src_path.write_text("") self._failed_tests = set() - # Translate symbols in topological order + # Process symbols in topological order G = nx.from_dict_of_lists(dependencies, create_using=nx.DiGraph) - assert isinstance(G, nx.DiGraph) + assert isinstance(G, nx.DiGraph) # create_using guarantees this groups = list( nx.lexicographical_topological_sort( - G.reverse(copy=False), key=create_symbol_lexical_key_fn(symbols, ast_order) + G.reverse(copy=False), key=create_symbol_ordering_key_fn(symbols, ast_order) ) ) + logger.debug(f"Symbol group order: {[' '.join(g) for g in groups]}") snippets: dict[CodeC, SymbolGroup] = {} translations: dict[SymbolGroup, CodeRust] = {} + wrappers: dict[WrapperName, CodeRust] = {} count = len(groups) for i, group in enumerate(groups, start=1): logger.info(f"Translating symbol group `{' '.join(group)}` [{i}/{count}] ...") @@ -74,127 +440,60 @@ def forward( continue snippets[snippet] = group - # FIXME: We could save context here by only including translations of descendants of the current symbol. - # However, one must prompt the LLM to never generate use statements since those could conflict. - # Use all unique (dict.fromkeys) translations as reference code since many symbol names can map to the same translation - already_translated = nx.descendants(G, group) - immediate_already_translated = set(G.successors(group)) - immediate_to_be_translated = set(G.predecessors(group)) + ctx = TranslationContext.build(G, group, groups, symbols, translations, wrappers) - reference_code = CodeRust.join( - dict.fromkeys(translations[g] for g in groups if g in translations) + # Translate and wrap snippet, saving it if it tests + group_result = self._translate_and_wrap_with_retries( + context=ctx, symbols=[symbols[name] for name in group] ) - reference_context = CodeRust.join( - dict.fromkeys( - translations[g] - if g in immediate_already_translated - else get_signatures(translations[g]) - for g in groups - if g in translations - ) - ) - - # Gather support code in topological order - support_code = CodeC.join( - symbols[name].code - if g in immediate_already_translated and LARGE_PROJECT - else CodeC(symbols[name].llm_context_declaration) - for g in groups - if g in already_translated - for name in g - ) - - # Gather dependent code in topological order - dependent_parts: list[CodeC] = [] - total_chars, exceeded = 0, False - for g in groups: - if exceeded: - break - if g in immediate_to_be_translated: - for name in g: - code = symbols[name].code - char_count = len(str(code)) - if LARGE_PROJECT and total_chars + char_count > MAX_DEPENDENT_CHARS: - exceeded = True - break - dependent_parts.append(code) - total_chars += char_count - dependent_code = CodeC.join(dependent_parts) - - # Translate snippet and save it if successful - pred = self.translate_with_retries( - reference_code=reference_code, - reference_context=reference_context, - symbols=[symbols[name] for name in group], - dependent_code=dependent_code, - support_code=support_code, - ) - - if pred.failure == "translate": + if group_result.failure == "translate": # Translate failures (as opposed to wrap/test failures) are fatal break - else: - # Once a test fails, skip it for all future groups in this run - newly_failed_tests = pred.failed_tests - self._failed_tests - if newly_failed_tests: - self._failed_tests.update(newly_failed_tests) - logger.info( - "Disabled the following failing tests: %s", - ", ".join(sorted(newly_failed_tests)), - ) - translations[group] = pred.translation - - # Re-assemble unique (dict.fromkeys) translations in order - translation = CodeRust.join( - dict.fromkeys(translations[group] for group in groups if group in translations) - ) - if not self.crate.is_bin: - translation += CodeRust("pub mod wrapper;") - pred = dspy.Prediction( - translation=translation, success=len(translations) == len(groups) - ) + assert group_result.translation is not None # group_result.failure != translate + translations[group] = group_result.translation + wrappers.update(group_result.wrappers) + + # Once a test fails, skip it for all future groups in this run + newly_failed_tests = group_result.failed_tests - self._failed_tests + if newly_failed_tests: + self._failed_tests.update(newly_failed_tests) + logger.info( + f"Disabled the following failing tests: {', '.join(sorted(newly_failed_tests))}" + ) + pred = dspy.Prediction(success=len(translations) == len(groups)) return pred - def translate_with_retries( + def _translate_and_wrap_with_retries( self, - reference_code: CodeRust, - reference_context: CodeRust, + context: TranslationContext, symbols: list[Symbol], - dependent_code: CodeC, - support_code: CodeC, prior_translation: CodeRust | None = None, - prior_wrappers: dict[str, CodeRust] | None = None, + prior_wrappers: dict[WrapperName, CodeRust] | None = None, feedback: str = "", - ) -> dspy.Prediction: + ) -> _Result: name = " ".join([f"`{s.name}`" for s in symbols]) - pred = dspy.Prediction() num_iters = max(self.max_iters, 1) + for i in range(num_iters): - # Save these in case translation fails - orig_c_src = self.crate.c_src_path.read_bytes() - orig_rust_src = self.crate.rust_src_path.read_bytes() - orig_wrappers_src = self._snapshot_wrappers() + state = self._snapshot() # Attempt translation and exit early on success - pred = self.translate( - reference_code, - reference_context, + result = self._translate_and_wrap( + context, symbols, - dependent_code, - support_code, prior_translation=prior_translation, prior_wrappers=prior_wrappers, feedback=feedback, ) - if pred.success: + if result.success: break # If neither translation nor wrappers differ from previous try, then stop retrying if ( prior_translation is not None - and prior_translation == pred.translation + and prior_translation == result.translation and prior_wrappers is not None - and prior_wrappers == pred.wrappers + and prior_wrappers == result.wrappers ): logger.error( f"Failed to translate symbol(s) {name} due to translation loop ({i + 1}/{num_iters})!" @@ -205,138 +504,459 @@ def translate_with_retries( logger.error(f"Failed to translate symbol(s) {name} ({i + 1}/{num_iters})!") # On failure, restore state based on which stage failed - if i + 1 < num_iters: - # Full restore for next retry since we haven't exhausted retries yet - self.crate.c_src_path.write_bytes(orig_c_src) - self.crate.rust_src_path.write_bytes(orig_rust_src) - self._restore_wrappers(orig_wrappers_src) - elif pred.failure == "translate": - # Full restore since a failure at this stage is fatal - self.crate.c_src_path.write_bytes(orig_c_src) - self.crate.rust_src_path.write_bytes(orig_rust_src) - self._restore_wrappers(orig_wrappers_src) - elif pred.failure == "wrap": + if i + 1 < num_iters or result.failure == "translate": + # Full restore for next retry or if translation failed (graceful exit) + self._restore(state) + elif result.failure == "wrap": # Wrapper restore since they failed but hopefully translation is good # FIXME: What if a wrapper is being tested? Seems fatal? - self._restore_wrappers(orig_wrappers_src) - elif pred.failure == "test": + self._restore(state, wrappers_only=True) + elif result.failure == "test": # Keep wrappers and translation even though they didn't pass tests pass # Create feedback for next iteration - prior_translation = pred.translation - prior_wrappers = pred.wrappers - feedback = pred.feedback - return pred - - def _snapshot_wrappers(self) -> dict[Path, bytes]: - wrappers: dict[Path, bytes] = {} + prior_translation = result.translation + prior_wrappers = result.wrappers + feedback = result.feedback + return result # pyright: ignore[reportPossiblyUnboundVariable] because num_iters is always >= 1 + + def _snapshot(self) -> _State: + def read(path: Path | None) -> bytes | None: + return path.read_bytes() if path else None + + assert self.rust_crate.lib_src_path is not None + wrapper_files = self.crate.src_dir.glob("wrap_*.rs") + return _State( + c_src=self.c_src_path.read_bytes(), + rust_lib_src=self.rust_crate.lib_src_path.read_bytes(), + rust_main_src=read(self.rust_crate.main_src_path), + hybrid_lib_src=read(self.crate.lib_src_path), + hybrid_main_src=read(self.crate.main_src_path), + wrappers={path: path.read_bytes() for path in wrapper_files if path.is_file()}, + ) - path = self.crate.rust_src_path.parent / "wrapper.rs" - if path.exists() and path.is_file(): - wrappers[path] = path.read_bytes() + def _restore(self, state: _State, *, wrappers_only: bool = False): + # Wrappers are always restored + for path, src in state.wrappers.items(): + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(src) - wrapper_dir = self.crate.rust_src_path.parent / "wrapper" - if wrapper_dir.exists(): - for path in wrapper_dir.glob("*.rs"): - if path.exists() and path.is_file(): - wrappers[path] = path.read_bytes() + if wrappers_only: + return - return wrappers + def write(path: Path | None, src: bytes | None): + if path is not None and src is not None: + path.write_bytes(src) - def _restore_wrappers(self, wrappers: dict[Path, bytes]) -> None: - for path, src in wrappers.items(): - path.parent.mkdir(parents=True, exist_ok=True) - path.write_bytes(src) - self.crate.vcs.add(path) + assert self.rust_crate.lib_src_path is not None + self.c_src_path.write_bytes(state.c_src) + self.rust_crate.lib_src_path.write_bytes(state.rust_lib_src) + write(self.rust_crate.main_src_path, state.rust_main_src) + write(self.crate.lib_src_path, state.hybrid_lib_src) + write(self.crate.main_src_path, state.hybrid_main_src) - def translate( + def _translate_and_wrap( self, - reference_code: CodeRust, - reference_context: CodeRust, + context: TranslationContext, symbols: list[Symbol], - dependent_code: CodeC, - support_code: CodeC, prior_translation: CodeRust | None = None, - prior_wrappers: dict[str, CodeRust] | None = None, + prior_wrappers: dict[WrapperName, CodeRust] | None = None, feedback: str = "", - ) -> dspy.Prediction: + ) -> _Result: prior_wrappers = prior_wrappers or {} - # Translate symbols and save it if successful + # Translate snippet and exit early if it fails snippet = CodeC.join(symbol.code for symbol in symbols) - pred = self.translate_symbol( - name=" ".join(symbol.name for symbol in symbols), - reference_code=reference_code, - reference_context=reference_context, + name = " ".join(symbol.name for symbol in symbols) + translation_result = self._translate_snippet( + name=name, + context=context, snippet=snippet, - dependent_code=dependent_code, prior_translation=prior_translation, feedback=feedback, ) - pred.failure = None - pred.wrappers = {} - pred.failed_tests = set() - if not pred.success: - pred.failure = "translate" - return pred - - # Write translation to crate - translation = pred.translation - with self.crate.rust_src_path.open("a") as f: - f.write(str(translation) + "\n") + if not translation_result.success: + return _Result(translation_result=translation_result) + + out = _Result(translation_result=translation_result) + assert out.translation is not None # translation_result.success == True # Generate wrapper for each symbol - wrappers: dict[str, dspy.Prediction] = {} for symbol in symbols: - # We can only hybrid build-test functions and variables - if not (symbol.is_function and symbol.is_definition) and not symbol.is_variable: - continue - # If we can't test symbols, then only wrap globals - if self.test_symbol is None and not symbol.is_global: - continue - - # Wrap function or annotate variable - prior_wrapper = prior_wrappers.get(symbol.name, None) - wrapper = self.wrap_symbol( + wrapper_name: WrapperName = f"wrap_{mangle(symbol.spelling)}" + wrapper_result = self._wrap_symbol( + name=wrapper_name, symbol=symbol, - reference_code=reference_code, - translation=pred.translation, - support_code=support_code + snippet + dependent_code, - prior_wrapper=prior_wrapper, + context=context, + snippet=snippet, + translation=out.translation, + prior_wrapper=prior_wrappers.get(wrapper_name), ) - - # Save function wrappers for next retry and caching - if symbol.is_function and symbol.is_definition and "wrapper" in wrapper: - wrappers[symbol.name] = wrapper - - # If wrapping failed exit early - if not wrapper.success: - pred.success = False - pred.failure = "wrap" - pred.feedback = wrapper.feedback - break - - # Try testing symbol and exit early if it fails - test_symbol = self.test_symbol - if test_symbol is None: + if wrapper_result is None: continue - test = test_symbol(symbol, skip=sorted(self._failed_tests)) - pred.failed_tests.update( - name for name, success in test.results.items() if not success - ) - if not test.success: - pred.success = False - pred.failure = "test" - pred.feedback = test.feedback + out.wrapper_results[wrapper_name] = wrapper_result + if not wrapper_result.success: break # Cache successful translation and wrappers - if pred.success: - self.translate_symbol.write_cache(pred) - for wrapper in wrappers.values(): - self.wrap_symbol.write_cache(wrapper) + if out.success: + self._translator.write_cache(out.translation_result.pred) + for wrapper_result in out.wrapper_results.values(): + if wrapper_result.pred is not None and self._wrapper is not None: + self._wrapper.write_cache(wrapper_result.pred) + return out + + def _translate_snippet( + self, + name: str, + context: TranslationContext, + snippet: CodeC, + prior_translation: CodeRust | None = None, + feedback: str = "", + ) -> _TranslationResult: + assert self.rust_crate.lib_src_path is not None + lib_src_path = self.rust_crate.lib_src_path + base_rust_src = CodeRust(lib_src_path.read_text()) + + def build(translation: CodeRust) -> str: + # Append translation to lib.rs and check if it builds + lib_src_path.write_text(str(base_rust_src + translation)) + self.rust_crate.vcs.add(lib_src_path) + # Import translated `main` symbol from lib.rs for binaries to get build feedback + if self.rust_crate.main_src_path is not None and name == "c:@F@main": + self.rust_crate.main_src_path.write_text( + f"#![forbid(unsafe_code)]\n\nuse {self.rust_crate.lib_name}::*;\n" + ) + self.rust_crate.vcs.add(self.rust_crate.main_src_path) + builds, build_feedback = self.rust_crate.cargo_build() + return "Running `cargo build` fails!\n" + build_feedback if not builds else "" + + def commit(msg: str, pred: dspy.Prediction): + if "reasoning" in pred and pred.reasoning: + msg += f"\n\n# Reasoning\n{indent(pred.reasoning, ' ')}" + if "feedback" in pred and pred.feedback: + msg += f"\n\n# Feedback\n{indent(pred.feedback, ' ')}" + self.rust_crate.vcs.commit(msg) + + pred = self._translator( + name=name, + crate_code=context.crate_code, + reference_code=context.reference_code, + snippet=snippet, + dependent_code=context.dependent_code, + prior_translation=prior_translation, + feedback=feedback, + feedback_fn=build, + on_attempt=commit, + ) + return _TranslationResult.from_pred(pred) - # Return wrappers for next retry - pred.wrappers = {name: wrapper.wrapper for name, wrapper in wrappers.items()} - return pred + def _wrap_symbol( + self, + name: WrapperName, + symbol: Symbol, + context: TranslationContext, + snippet: CodeC, + translation: CodeRust, + prior_wrapper: CodeRust | None, + ) -> _WrapperResult | None: + result = None + + # Main function must always wrapped + if self.crate.main_src_path is not None and symbol.spelling == "main": + result = self._wrap_main() + elif symbol.is_type and symbol.is_definition: + result = self._wrap_type(name, symbol, context, snippet, translation, prior_wrapper) + elif symbol.is_variable: + result = self._wrap_variable(name, symbol) + elif symbol.is_function and symbol.is_definition: + result = self._wrap_function( + name, symbol, context, snippet, translation, prior_wrapper + ) + if result is None: + return None + + # Don't bother testing if the wrapping failed or no tests + if not result.success or self._tests is None: + return result + + # Test the hybrid crate + passes, results = self._test_symbol(symbol, skip=sorted(self._failed_tests)) + result.failed_tests = {name for name, success in results.items() if not success} + if not passes: + result.failure = "test" + return result + + def _test_symbol(self, symbol: Symbol, skip: list[str]) -> tuple[bool, dict[str, bool]]: + assert self._tests is not None + logger.info(f"Testing symbol `{symbol.name}` ...") + + # Make sure the crate builds before testing + builds, feedback = self.crate.cargo_build() + if not builds: + raise RuntimeError(f"Crate does not build!\n{feedback}") + + # Run cargo test + passes, jsonl, feedback, _ = self.crate.cargo_test( + self._tests, skip=skip, test_harness="nextest run", message_format="libtest-json" + ) + results = _extract_test_results(jsonl) + if passes: + msg = f"Tested symbol `{symbol.name}`" + logger.info(msg) + else: + feedback = "Running `cargo test` fails!\n" + feedback + msg = f"Failed to test symbol `{symbol.name}`" + logger.error(msg) + msg += f"\n\n{feedback}" + self.crate.vcs.commit(msg) + return passes, results + + def _wrap_main(self) -> _WrapperResult: + logger.info("Generating wrapper for function `main` ...") + + # Declare C `main` as extern so Rust owns the definition and we avoid + # duplicate entrypoint symbols at link time. + clang_make_extern_(self.c_src_path, "main") + self.crate.vcs.add(self.c_src_path) + + # Import the Rust translation crate from each hybrid root so linker-visible + # symbols stay reachable from the test/build target. + for root_path in (self.crate.lib_src_path, self.crate.main_src_path): + if root_path is None: + continue + root_path.write_text(f"use {self.rust_crate.lib_name}::*;\n") + self.crate.vcs.add(root_path) + + # Fail fast after entrypoint/linkage edits so linker or compile regressions + # are caught before continuing with additional wrapper work. + success, output = self.crate.cargo_build() + if not success: + raise RuntimeError(f"Failed to build crate!\n{output}") + self.crate.vcs.commit("Wrapped function `main`") + return _WrapperResult() + + def _wrap_variable(self, name: WrapperName, symbol: Symbol) -> _WrapperResult | None: + # No point in wrapping non-globals if no tests + if self._tests is None and not symbol.is_global: + return None + + # Emit a Rust module containing FFI variable bindings and persist it + # under src/ so crate-root pub mod declarations can include it. + var_wrapper = bindgen(self.c_src_path, symbol.spelling) + wrapper_path = self.crate.src_dir / f"{name}.rs" + wrapper_path.parent.mkdir(exist_ok=True, parents=True) + wrapper_path.write_text(str(var_wrapper)) + self.crate.vcs.add(wrapper_path) + + # Ensure the C variable has external linkage so the wrapper can resolve + # the symbol at link time and access the same storage across crates. + clang_make_global_(self.c_src_path, symbol.spelling) + self.crate.vcs.add(self.c_src_path) + + # Register the wrapper module in each crate root so tests and callers can + # resolve it by path and rustc includes it in the build graph. + for root_path in (self.crate.lib_src_path, self.crate.main_src_path): + if root_path is None: + continue + with root_path.open("a") as f: + f.write(f"pub mod {name};\n") + self.crate.vcs.add(root_path) + + # Fail fast after wrapper/linkage edits so linker or compile regressions + # are caught before continuing with additional wrapper work. + success, output = self.crate.cargo_build() + if not success: + raise RuntimeError(f"Failed to build crate!\n{output}") + + msg = f"Wrapped variable `{symbol.name}`" + logger.info(msg) + self.crate.vcs.commit(msg) + return _WrapperResult(pred=dspy.Prediction(wrapper=var_wrapper, success=True)) + + def _wrap_type( + self, + name: WrapperName, + symbol: Symbol, + context: TranslationContext, + snippet: CodeC, + translation: CodeRust, + prior_wrapper: CodeRust | None, + ) -> _WrapperResult | None: + if self._wrapper is None: + return None + # Only struct declarations with linkage are wrappable: other type kinds have no + # meaningful field-by-field `c_to_r`/`r_to_c` pair, and structs without linkage + # are unnameable. + if not (symbol.is_struct and symbol.is_global): + return None + + # Seed a concrete wrapper module on disk so the hybrid crate can build + # and the wrapper generator has a stable file path to iteratively replace. + unimplemented_wrapper = generate_unimplemented_type_wrapper( + self.c_src_path, symbol.spelling + ) + wrapper_path = self.crate.src_dir / f"{name}.rs" + wrapper_path.parent.mkdir(exist_ok=True, parents=True) + wrapper_path.write_text(str(unimplemented_wrapper)) + self.crate.vcs.add(wrapper_path) + + # Register the wrapper module in each crate root so tests and callers can + # resolve it by path and rustc includes it in the build graph. + for root_path in (self.crate.lib_src_path, self.crate.main_src_path): + if root_path is None: + continue + with root_path.open("a") as f: + f.write(f"pub mod {name};\n") + self.crate.vcs.add(root_path) + + # Fail fast after wrapper/linkage edits so we surface compiler errors + # before iterative wrapper generation starts. + builds, build_feedback = self.crate.cargo_build() + if not builds: + raise RuntimeError(f"The crate does not build!\n\n{build_feedback}") + + def test(wrapper: CodeRust) -> str: + # Write wrapper to disk and add to VCS so commit can commit it + wrapper_path.write_text(str(wrapper)) + self.crate.vcs.add(wrapper_path) + + # Enforce that both required round-trip test functions are present + missing = [ + fn + for fn in ("round_trip_zeroed", "round_trip_nontrivial") + if f"fn {fn}" not in str(wrapper) + ] + if missing: + return ( + "Required test functions are missing: " + + ", ".join(f"`{fn}`" for fn in missing) + + ". Both `round_trip_zeroed` and `round_trip_nontrivial` must be implemented." + ) + + # Run the round-trip tests: cargo test provides both compilation errors + # and test failure messages, giving richer feedback than cargo build alone. + passes, _, feedback, _ = self.crate.cargo_test(f"{name}::tests", lib=True) + return "" if passes else f"Running `cargo test` fails!\n{feedback}" + + def commit(msg: str, pred: dspy.Prediction): + if "reasoning" in pred and pred.reasoning: + msg += f"\n\n# Reasoning\n{indent(pred.reasoning, ' ')}" + if "build_feedback" in pred and pred.build_feedback: + msg += f"\n\n# Build Feedback\n{indent(pred.build_feedback, ' ')}" + self.crate.vcs.commit(msg) + + pred = self._wrapper( + symbol=symbol, + crate_code=context.crate_code, + translation=translation, + unimplemented_wrapper=unimplemented_wrapper, + wrapper_path=wrapper_path.relative_to(self.crate.cargo_toml.parent), + wrapped_crate=self.rust_crate.lib_name, + other_wrappers=context.wrappers, + prior_wrapper=prior_wrapper, + support_code=context.support_code + snippet + context.dependent_code, + feedback_fn=test, + on_attempt=commit, + ) + return _WrapperResult.from_pred(pred) + + def _wrap_function( + self, + name: WrapperName, + symbol: Symbol, + context: TranslationContext, + snippet: CodeC, + translation: CodeRust, + prior_wrapper: CodeRust | None, + ) -> _WrapperResult | None: + if self._wrapper is None: + return None + # No point in wrapping non-globals if no tests + if self._tests is None and not symbol.is_global: + return None + + # Seed a placeholder wrapper module so the crate can compile immediately + # and the wrapper generator has a stable file to iteratively overwrite. + unimplemented_wrapper = generate_unimplemented_function_wrapper( + self.c_src_path, symbol.spelling + ) + if unimplemented_wrapper is None: + logger.warning(f"Skipping wrap of function symbol `{symbol.name}`") + return _WrapperResult() + wrapper_path = self.crate.src_dir / f"{name}.rs" + wrapper_path.parent.mkdir(exist_ok=True, parents=True) + wrapper_path.write_text(str(unimplemented_wrapper)) + self.crate.vcs.add(wrapper_path) + + # Switch the C function declaration to extern so the Rust wrapper crate + # can provide the callable definition without duplicate symbol ownership. + clang_make_extern_(self.c_src_path, symbol.spelling) + self.crate.vcs.add(self.c_src_path) + + # Register the wrapper module in each crate root so tests and callers can + # resolve it by path and rustc includes it in the build graph. + for root_path in (self.crate.lib_src_path, self.crate.main_src_path): + if root_path is None: + continue + with root_path.open("a") as f: + f.write(f"pub mod {name};\n") + self.crate.vcs.add(root_path) + + # Fail fast after wrapper/linkage edits so linker or compile regressions + # are caught before iterative wrapper generation continues. + builds, build_feedback = self.crate.cargo_build() + if not builds: + raise RuntimeError(f"The crate does not build!\n\n{build_feedback}") + + def build(wrapper: CodeRust) -> str: + wrapper_path.write_text(str(wrapper)) + self.crate.vcs.add(wrapper_path) + builds, build_feedback = self.crate.cargo_build() + return build_feedback if not builds else "" + + def commit(msg: str, pred: dspy.Prediction): + if "reasoning" in pred and pred.reasoning: + msg += f"\n\n# Reasoning\n{indent(pred.reasoning, ' ')}" + if "build_feedback" in pred and pred.build_feedback: + msg += f"\n\n# Build Feedback\n{indent(pred.build_feedback, ' ')}" + if "scope_feedback" in pred and pred.scope_feedback: + msg += f"\n\n# Scope Feedback\n{indent(pred.scope_feedback, ' ')}" + self.crate.vcs.commit(msg) + + pred = self._wrapper( + symbol=symbol, + crate_code=context.crate_code, + translation=translation, + unimplemented_wrapper=unimplemented_wrapper, + wrapper_path=wrapper_path.relative_to(self.crate.cargo_toml.parent), + wrapped_crate=self.rust_crate.lib_name, + other_wrappers=context.wrappers, + prior_wrapper=prior_wrapper, + support_code=context.support_code + snippet + context.dependent_code, + feedback_fn=build, + on_attempt=commit, + ) + return _WrapperResult.from_pred(pred) + + +def _extract_test_results(output: str) -> dict[str, bool]: + test_results: dict[str, bool] = {} + + for line in output.splitlines(): + try: + obj = json.loads(line) + except json.JSONDecodeError: + continue + if obj.get("type") != "test": + continue + event = obj.get("event") + if event not in {"ok", "failed", "ignored"}: + continue + name = str(obj.get("name", "")).rsplit("$", 1)[-1].strip() + if name: + # Treat ignored as non-failing for disable-list purposes + test_results[name] = event != "failed" + + return test_results diff --git a/src/ideas/translate_snippet.py b/src/ideas/translate_snippet.py index dcf7653..2c26a7d 100644 --- a/src/ideas/translate_snippet.py +++ b/src/ideas/translate_snippet.py @@ -7,14 +7,13 @@ import logging import sqlite3 from pathlib import Path -from textwrap import indent +from collections.abc import Callable import dspy from dspy.utils.exceptions import AdapterParseError from dspy.utils.usage_tracker import track_usage from dspy.dsp.utils.settings import settings -from .tools import Crate, LARGE_PROJECT from .ast import CodeC from .ast_rust import CodeRust from .model import format_usage @@ -31,7 +30,7 @@ class SnippetTranslatorSignature(dspy.Signature): - `reference_code`: Existing Rust code the translation must build on. Use it as-is; do not refactor it. - `snippet`: The single C definition to translate. - - `dependent_code`: C code that uses the snippet. Use it only to understand ownership, lifetime, and memory-management requirements; do not translate it. + - `dependent_code`: C code that uses the snippet. Use it to determine concrete types for opaque and void pointers, ownership, lifetimes, and memory-management requirements; do not translate it. - `prior_translation` and `feedback`: If provided, treat the feedback as a critique of the prior translation and address it in the new translation. # Hard constraints @@ -39,11 +38,15 @@ class SnippetTranslatorSignature(dspy.Signature): - The translation must contain no `unsafe` constructs. - Do not include `#![forbid(unsafe_code)]` in the translation since it is included by default. - Do not define any `impl` blocks. + - Define all top-level items (functions, structs, enums, type aliases, constants, statics, unions, traits, and modules) as fully public using plain `pub` (e.g., `pub fn ...`, `pub struct ...`). Also define every field of a top-level struct as `pub`, including unnamed fields in tuple structs (e.g., `pub struct Pair(pub i32, pub i32)`). Do not use restricted visibility such as `pub(crate)` or `pub(super)` anywhere; use only `pub`. - Do not weaken behavior with stubs, fallback defaults, relaxed assertions, or intentionally partial implementations. + - Do not annotate any type with `#[repr(C)]`; the translation does not need C ABI compatibility. + - Do not add `#[derive(...)]` attributes; they generate `impl` blocks and may impose behavior (e.g., `Default`, `Clone`) that does not match C semantics. + - If the snippet references a C type not yet present in `reference_code`, include a correct translation of that type in the output so the translation compiles. Derive the referenced type's translation from `dependent_code` and any context visible in the snippet itself. Its translation will be reused as-is when the type's own snippet is processed later. # Faithfulness to C semantics - The overarching rule: reproduce the C code's observable behavior exactly. Do not "fix", simplify, or second-guess the C code's intent. + The overarching rule: reproduce the C code's runtime behavior exactly. Do not "fix", simplify, or second-guess the C code's intent. For type definitions, use idiomatic Rust types that are semantically equivalent rather than structurally identical. ## Arithmetic and expressions @@ -97,6 +100,22 @@ class SnippetTranslatorSignature(dspy.Signature): - Translate a C function that returns a pointer into a global/static container (e.g., `return &table[i]`) as a function returning `&T` into that container, not an owned clone. - If the container is locked: acquire the lock in the caller and borrow `&T` from the held guard. If the existing accessor locks internally and returns an owned value, the caller must bypass it — lock the container directly, borrow references from the guard, and finish all identity comparisons before releasing. + ## Void pointers (`void *`) + + - A `void *` field or parameter is not inherently polymorphic. Inspect `dependent_code` to find every cast applied to the value. If all casts resolve to the same concrete type, translate the field using that concrete type (e.g., `Option>`). Do not use `Box`, `Box`, or any other type-erasure mechanism unless the pointer is genuinely polymorphic — i.e., cast to multiple structurally unrelated types in different code paths. + - A `void *` used solely to break a forward-declaration cycle is not polymorphic. Resolve the concrete type from the casts and use it directly. + - When the `void *` is nullable in C (compared to `NULL`, initialized to `0`, or conditionally assigned), translate it as `Option>` if the containing struct owns the allocation (evidenced by `free` being called through this field), or as `Option<&T>` / `Option<&mut T>` if it is a non-owning reference. + + ## Pointer fields in structs (`T *`, `T **`) + + Determine ownership from `dependent_code` before choosing a Rust type: + + - If `free(field)` or equivalent is called through the struct, the struct owns the pointee. Use `Box` for a single value or `Vec` for a heap-allocated array. Wrap in `Option<>` if the pointer may be null. + - If the pointer is never freed through the struct (it aliases data owned elsewhere): use `&T` or `&mut T` with an appropriate lifetime. + - If the field stores a heap array whose length is tracked separately (a C dynamic array): use `Vec`, not `Box<[T]>`. + - If the field is a fixed-length C array (`T arr[N]`): use `[T; N]`. + - Do not use raw pointers (`*mut T`, `*const T`) as a shortcut when a safe owned or borrowed type is available. + ## Mutable global state and locking - Never lock the same mutex/`RwLock` more than once in a single expression. @@ -114,10 +133,48 @@ class SnippetTranslatorSignature(dspy.Signature): If the translation introduces an auxiliary data structure (thread-local, `HashMap`, `RefCell>`, etc.) to represent metadata that C tracked via struct fields or raw pointers, every function that conceptually reads or writes that metadata in C must read or write the auxiliary structure in Rust. A stub claiming the data is "not accessible in safe Rust" is never acceptable once such a mechanism exists. + ## C primitive type mappings + + Use the following canonical mappings: + + - `char` (used as integer) → `i8`; `unsigned char` → `u8` + - `short` → `i16`; `unsigned short` → `u16` + - `int` → `i32`; `unsigned int` → `u32` + - `long` → `i64`; `unsigned long` → `u64` + - `long long` → `i64`; `unsigned long long` → `u64` + - `float` → `f32`; `double` → `f64` + - `size_t` → `usize`; `ptrdiff_t` → `isize` + - `intptr_t` → `isize`; `uintptr_t` → `usize` + - `int8_t`/`uint8_t` → `i8`/`u8`; `int16_t`/`uint16_t` → `i16`/`u16`; `int32_t`/`uint32_t` → `i32`/`u32`; `int64_t`/`uint64_t` → `i64`/`u64` + - `char *` used as a string: see "Strings and NUL termination". + + ## C typedefs and forward declarations + + - A typedef that merely names an existing struct (`typedef struct Foo Foo;`) carries no information and should be omitted. + - An anonymous struct typedef (`typedef struct { ... } Foo;`) translates to `pub struct Foo { ... }`. + - A scalar typedef (`typedef unsigned int foo_t;`) translates to `pub type FooT = u32;`. + - A function-pointer typedef (`typedef int (*cmp_fn)(int, int);`) translates to `pub type CmpFn = fn(i32, i32) -> i32;`. + - Forward struct declarations (`struct Foo;`) are not translated; they become concrete when the full definition's snippet is processed. + + ## C enums + + - A C enum whose values are used as integers (assigned to integer variables, used in arithmetic, or used as array indices) translates to a group of `pub const` items with the appropriate integer type (default `i32`). Do not translate such enums as Rust `enum` variants; Rust enums are not freely interchangeable with integers. + - A C enum whose values are used exclusively in switch/pattern-match contexts and never mixed with integers may be translated as a Rust `enum`. + - Anonymous C enums (`enum { A = 0, B, C };`) follow the same rules; omit the type name. + ## Observable output - Reproduce stdout/stderr text, spacing, punctuation, and line breaks exactly. - When the C source contains multi-byte UTF-8 literals (e.g., box-drawing characters), count Unicode scalar values, not bytes. Reproduce the same number of code points. + + # Allowed External Crates + + - `libc`: Raw FFI bindings to platform libraries like libc. + - `openssl`: OpenSSL bindings + - `flate2`: DEFLATE compression and decompression exposed as Read/BufRead/Write streams. Supports miniz_oxide and multiple zlib implementations. Supports zlib, gzip, and raw deflate streams. + - `regex`: An implementation of regular expressions for Rust. This implementation uses finite automata and guarantees linear time matching on all inputs. + + Use functions from these crates, as needed, to translate the C code to equivalent, memory-safe Rust. """ reference_code: CodeRust = dspy.InputField() @@ -128,45 +185,44 @@ class SnippetTranslatorSignature(dspy.Signature): translation: CodeRust = dspy.OutputField() -_crate_dependencies = """ -# Crate dependencies: -The Rust project has visibility into the following crates: -- `flate2` for DEFLATE compression and decompression -- `regex` for regular expression parsing and matching +def _default_feedback_fn(translation: CodeRust) -> str: + return "" -Use functions from these crates as needed to translate the C code to equivalent, memory-safe Rust. -""" + +def _default_on_attempt(msg: str, pred: dspy.Prediction) -> None: + pass class SnippetTranslator(dspy.Module): def __init__( self, - crate: Crate, translator: type[dspy.Module], max_iters: int = 5, + cache: Path | None = None, ): super().__init__() signature = SnippetTranslatorSignature - if LARGE_PROJECT: - signature = signature.with_instructions( - "\n\n".join([signature.instructions, _crate_dependencies]) - ) - - self.crate = crate self._translate = translator(signature) self.max_iters = max_iters - self.cache = _init_cache(crate.workspace_root / "cache.db") + self.cache = _init_cache(cache) def forward( self, name: str, + crate_code: CodeRust, reference_code: CodeRust, - reference_context: CodeRust, snippet: CodeC, dependent_code: CodeC, prior_translation: CodeRust | None = None, feedback: str = "", + feedback_fn: Callable[[CodeRust], str] | None = None, + on_attempt: Callable[[str, dspy.Prediction], None] | None = None, ) -> dspy.Prediction: + if feedback_fn is None: + feedback_fn = _default_feedback_fn + if on_attempt is None: + on_attempt = _default_on_attempt + logger.info(f"Translating snippet `{name}` ...") # Use cache when no prior translation @@ -175,32 +231,28 @@ def forward( translation = CodeRust(f"// Empty snippet `{name}`") elif prior_translation is None: translation = _read_cache(self.cache, name, snippet) + if translation is not None: + translation = _make_public(translation) else: logger.info("Ignoring snippet cache...") translation = None - orig_rust_src = self.crate.rust_src_path.read_bytes() - pred = dspy.Prediction() - builds = False + pred = dspy.Prediction(feedback=feedback) for i in range(max(self.max_iters, 1)): # Use the translation from the prior iteration as feedback for the next iteration if i > 0: prior_translation = translation - # Ensure any translated snippet is safe - rust_src = CodeRust("#![forbid(unsafe_code)]") - rust_src += reference_code - # Use prior translation as the translation on first iteration only. # This allows static translations that violate safety, which will be fixed by the LLM! try: pred = self.translate( - reference_code if not LARGE_PROJECT else reference_context, - snippet, - dependent_code, - prior_translation, - feedback, - translation if i == 0 else None, + reference_code=reference_code, + snippet=snippet, + dependent_code=dependent_code, + prior_translation=prior_translation, + feedback=pred.feedback, + translation=translation if i == 0 else None, ) except AdapterParseError: logger.exception( @@ -214,52 +266,34 @@ def forward( translation = pred.translation assert isinstance(translation, CodeRust) - if translation in reference_code: + if translation in crate_code: translation = CodeRust(f"// duplicate snippet `{name}` detected") if translation == prior_translation: logger.warning("Snippet translation loop detected!") - - # Append translation and check if it builds - rust_src += translation - self.crate.rust_src_path.write_text(str(rust_src)) - self.crate.vcs.add(self.crate.rust_src_path) - # FIXME: Checking name for c:@F@main is brittle but we have no better way here. - # The proper way to fix is to yield the translation back to the caller so it can - # build and tell us whether to translation is successful. - builds, feedback = self.crate.cargo_build(fix_E0601="c:@F@main" not in name) - if not builds: - feedback = "Running `cargo build` fails!\n" + feedback - + pred.name = name + pred.snippet = snippet + pred.crate_code = crate_code + pred.dependent_code = dependent_code + pred.prior_translation = prior_translation or CodeRust() + pred.translation = translation + + parts = [] if CodeRust("#![forbid(unsafe_code)]") in translation: - feedback = "Do not include `#![forbid(unsafe_code)]` in the translation!" - builds = False - - usage = format_usage(pred) - - # Exit early if we build - if builds: - msg = f"Translated snippet `{name}`: {usage}" + parts.append("Do not include `#![forbid(unsafe_code)]` in the translation!") + if feedback := feedback_fn(translation): + parts.append(feedback) + pred.feedback = "\n\n".join(parts) + pred.success = not pred.feedback + + if pred.success: + msg = f"Translated snippet `{name}`: {format_usage(pred)}" logger.info(msg) - if "reasoning" in pred: - msg += f"\n\n# Reasoning\n{indent(pred.reasoning, ' ')}" - self.crate.vcs.commit(msg) + on_attempt(msg, pred) break - - msg = f"Failed to translate snippet `{name}` ({i + 1}/{self.max_iters}): {usage}" - logger.error(msg) - if "reasoning" in pred: - msg += f"\n\n# Reasoning\n{indent(pred.reasoning, ' ')}" - msg += f"\n\n# Feedback\n{indent(feedback, ' ')}" if feedback else "" - self.crate.vcs.commit(msg) - self.crate.rust_src_path.write_bytes(orig_rust_src) - pred.name = name - pred.snippet = snippet - pred.reference_code = reference_code - pred.dependent_code = dependent_code - pred.prior_translation = prior_translation or CodeRust() - pred.feedback = feedback - pred.translation = translation - pred.success = builds + else: + msg = f"Failed to translate snippet `{name}` ({i + 1}/{self.max_iters}): {format_usage(pred)}" + logger.error(msg) + on_attempt(msg, pred) return pred def translate( @@ -271,7 +305,6 @@ def translate( feedback: str, translation: CodeRust | None, ) -> dspy.Prediction: - """Get a prediction for the current iteration.""" parent_usage_tracker = settings.usage_tracker if translation is not None: pred = dspy.Prediction(translation=translation) @@ -310,7 +343,7 @@ def write_cache(self, pred: dspy.Prediction) -> None: self.cache, pred.name, pred.snippet, - pred.reference_code, + pred.crate_code, pred.dependent_code, pred.prior_translation, pred.feedback, @@ -369,11 +402,217 @@ def _read_cache(cache: Path | None, name: str, snippet: CodeC) -> CodeRust | Non return None +def _make_public(translation: CodeRust) -> CodeRust: + source = str(translation) + from typing import Any + from tree_sitter import Language, Parser, Query, QueryCursor + from tree_sitter_rust import language as rust_language + + language = Language(rust_language()) + parser = Parser(language) + tree = parser.parse(source.encode("utf-8")) + query = Query( + language, + """ + (function_item + (visibility_modifier)? @vis + "fn" @fn_kw + ) @item + + (struct_item + (visibility_modifier)? @vis + "struct" @kw + ) @item + + (type_item + (visibility_modifier)? @vis + "type" @kw + ) @item + + (enum_item + (visibility_modifier)? @vis + "enum" @kw + ) @item + + (static_item + (visibility_modifier)? @vis + "static" @kw + ) @item + + (union_item + (visibility_modifier)? @vis + "union" @kw + ) @item + + (const_item + (visibility_modifier)? @vis + "const" @kw + ) @item + + (trait_item + (visibility_modifier)? @vis + "trait" @kw + ) @item + + (mod_item + (visibility_modifier)? @vis + "mod" @kw + ) @item + + (struct_item + body: (field_declaration_list + (field_declaration + (visibility_modifier)? @field_vis + (field_identifier) @field_name + ) @field + ) + ) @struct + + """, + ) + + by_item: dict[tuple[int, int], dict[str, Any]] = {} + by_field: dict[tuple[int, int], dict[str, Any]] = {} + item_types = { + "function_item", + "struct_item", + "type_item", + "enum_item", + "static_item", + "union_item", + "const_item", + "trait_item", + "mod_item", + } + + cursor = QueryCursor(query) + captures = cursor.captures(tree.root_node) + for capture_name, nodes in captures.items(): + for node in nodes: + if capture_name == "item": + key = (node.start_byte, node.end_byte) + by_item.setdefault(key, {"item": node, "vis": None, "kw": None}) + continue + + if capture_name == "field": + key = (node.start_byte, node.end_byte) + by_field.setdefault(key, {"field": node, "vis": None, "name": None}) + continue + + parent = node.parent + while parent is not None and parent.type not in item_types | { + "field_declaration", + }: + parent = parent.parent + if parent is None: + continue + + if parent.type == "field_declaration": + key = (parent.start_byte, parent.end_byte) + entry = by_field.setdefault(key, {"field": parent, "vis": None, "name": None}) + if capture_name == "field_vis": + entry["vis"] = node + elif capture_name == "field_name": + entry["name"] = node + continue + + key = (parent.start_byte, parent.end_byte) + entry = by_item.setdefault(key, {"item": parent, "vis": None, "kw": None}) + if capture_name == "vis": + entry["vis"] = node + elif capture_name in {"fn_kw", "kw"}: + entry["kw"] = node + + edits: list[tuple[int, int, bytes]] = [] + source_bytes = source.encode("utf-8") + + for entry in by_item.values(): + item_node = entry["item"] + vis_node = entry["vis"] + kw_node = entry["kw"] + + # Only rewrite module-level items. + if item_node.parent is None or item_node.parent.type != "source_file": + continue + + if vis_node is None: + if kw_node is None: + continue + edits.append((kw_node.start_byte, kw_node.start_byte, b"pub ")) + continue + + vis_text = source_bytes[vis_node.start_byte : vis_node.end_byte].strip() + if vis_text != b"pub": + edits.append((vis_node.start_byte, vis_node.end_byte, b"pub")) + + for entry in by_field.values(): + field_node = entry["field"] + vis_node = entry["vis"] + name_node = entry["name"] + + # Only rewrite fields of top-level structs. + struct_node = field_node.parent + while struct_node is not None and struct_node.type != "struct_item": + struct_node = struct_node.parent + if struct_node is None or struct_node.parent is None: + continue + if struct_node.parent.type != "source_file": + continue + + if vis_node is None: + if name_node is None: + continue + edits.append((name_node.start_byte, name_node.start_byte, b"pub ")) + continue + + vis_text = source_bytes[vis_node.start_byte : vis_node.end_byte].strip() + if vis_text != b"pub": + edits.append((vis_node.start_byte, vis_node.end_byte, b"pub")) + + # Handle tuple struct fields programmatically: tree-sitter-rust has no + # "ordered_field_declaration" wrapper node; fields are direct children of + # "ordered_field_declaration_list". + for struct_node in tree.root_node.children: + if struct_node.type != "struct_item": + continue + for child in struct_node.children: + if child.type != "ordered_field_declaration_list": + continue + pending_vis = None + for fc in child.children: + if fc.type in ("(", ")", ","): + pending_vis = None + elif fc.type == "visibility_modifier": + pending_vis = fc + elif fc.type == "attribute_item": + pass + else: + # fc is a type node — this is a tuple field + if pending_vis is None: + edits.append((fc.start_byte, fc.start_byte, b"pub ")) + else: + vis_text = source_bytes[ + pending_vis.start_byte : pending_vis.end_byte + ].strip() + if vis_text != b"pub": + edits.append((pending_vis.start_byte, pending_vis.end_byte, b"pub")) + pending_vis = None + break + + if not edits: + return translation + + out = bytearray(source_bytes) + for start, end, replacement in sorted(edits, key=lambda item: item[0], reverse=True): + out[start:end] = replacement + return CodeRust(out.decode("utf-8")) + + def _write_cache( cache: Path | None, name: str, snippet: CodeC, - reference_code: CodeRust, + crate_code: CodeRust, dependent_code: CodeC, prior_translation: CodeRust, feedback: str, @@ -392,7 +631,7 @@ def _write_cache( ( name, str(snippet), - str(reference_code), + str(crate_code), str(dependent_code), str(prior_translation), feedback, diff --git a/src/ideas/wrapper.py b/src/ideas/wrapper.py index a2050a5..81d3a96 100644 --- a/src/ideas/wrapper.py +++ b/src/ideas/wrapper.py @@ -8,46 +8,54 @@ import sqlite3 import logging from pathlib import Path -from textwrap import indent -from collections import OrderedDict +from collections.abc import Callable import dspy from dspy.utils.exceptions import AdapterParseError from dspy.utils.usage_tracker import track_usage from dspy.dsp.utils.settings import settings -from ideas.tools import Crate, check_rust, run_subprocess, LARGE_PROJECT +from ideas.tools import check_rust, run_subprocess +from ideas.ast import CodeC, Symbol, clang_make_bindable_ from ideas.ast_rust import CodeRust, validate_changes, mangle -from ideas.ast import CodeC, Symbol -from ideas.ast import clang_make_global_, clang_make_extern_, clang_make_bindable_ from ideas.model import format_usage logger = logging.getLogger("ideas.wrapper") -class Signature(dspy.Signature): +class FunctionWrapperSignature(dspy.Signature): """ - Generate a C-compatible FFI wrapper for `crate::{symbol_name}`. + Generate a C-compatible FFI wrapper for `{wrapped_crate}::{symbol_name}` in a hybrid C/Rust build where C globals and the Rust port must stay in sync. # Goal - Produce a `wrapper` that callers of the original C symbol can link against unchanged. The implementation of `crate::{symbol_name}` lives in the crate at "{crate_path}"; the wrapper will be written to "{wrapper_path}". + Produce a `wrapper` that callers of the original C symbol can link against unchanged. The implementation of `{wrapped_crate}::{symbol_name}` lives in dependency crate `{wrapped_crate}`; the wrapper will be written to "{wrapper_path}". # Template - Use `example_wrapper` as the template for `wrapper`. Preserve its function signature, attributes, and module structure exactly. - - Replace only the `unimplemented!()` body with an implementation that calls `crate::{symbol_name}`. + - Replace only the `unimplemented!()` body with an implementation that calls `{wrapped_crate}::{symbol_name}`. # Type conversions - Types in `crate::wrapper::` (bindgen-generated, C-compatible layout) are *not* layout-compatible with those in `crate::` (idiomatic Rust). The wrapper must: + Types in `crate::` (bindgen-generated, C-compatible layout) are *not* layout-compatible with those in `{wrapped_crate}::` (idiomatic Rust). The wrapper must: - 1. Copy field values from each `crate::wrapper::` argument into a fresh `crate::` value before the call. - 2. Call `crate::{symbol_name}` with the converted values. - 3. Copy result/output values back from `crate::` types into the `crate::wrapper::` types the C ABI expects. + 1. Copy field values from each `crate::` argument into a fresh `{wrapped_crate}::` value before the call. + 2. Call `{wrapped_crate}::{symbol_name}` with the converted values. + 3. Copy result/output values back from `{wrapped_crate}::` types into the `crate::` types the C ABI expects. Use `support_code` to recover the original C types behind opaque or erased Rust types (notably `void*`, `*mut c_void`, and untyped byte buffers) so each field is converted at its true C type. + When converting a struct argument, check `crate` for a `c_to_r` / `r_to_c` pair generated for that type. If one exists, call `c_to_r(ptr)` (passing a `*const` pointer) instead of converting fields inline. Likewise use `r_to_c(&rust_value, c_out_ptr)` (passing a reference and a `*mut` pointer) when writing a result or out-parameter back to a `crate::` type — `r_to_c` writes in-place and returns nothing. + + # Global synchronization + + If `{wrapped_crate}::{symbol_name}` reads or writes globals, synchronize them in the wrapper: + + - Each C global is exposed in `crate` as a `pub static mut` bearing the variable's name. Locate its full path by inspecting `crate` directly. + - Before the call, copy each readable global from its `pub static mut` in `crate` into the corresponding Rust global in `{wrapped_crate}`. If the global's type has a `c_to_r` function in `crate`, use it for the conversion; otherwise copy field-by-field. + - After the call, copy each writable global from `{wrapped_crate}` back to its `pub static mut` in `crate`. If the global's type has an `r_to_c` function in `crate`, use it for the conversion; otherwise copy field-by-field. + # Raw pointer handling - Null-check every raw pointer parameter before its first dereference. On null, return the same value the C function returns for null input (typically an error code, `false`, `-1`, or `null`) — never dereference and panic. Use `support_code` to determine the correct null-input sentinel. @@ -103,82 +111,7 @@ class Signature(dspy.Signature): wrapper: CodeRust = dspy.OutputField() -class HybridSignature(Signature): - """ - Generate a C-compatible FFI wrapper for `crate::{symbol_name}` in a hybrid C/Rust build where C globals and the Rust port must stay in sync. - - # Goal - - Produce a `wrapper` that callers of the original C symbol can link against unchanged. The implementation of `crate::{symbol_name}` lives in the crate at "{crate_path}"; the wrapper will be written to "{wrapper_path}". - - # Template - - - Use `example_wrapper` as the template for `wrapper`. Preserve its function signature, attributes, and module structure exactly. - - Replace only the `unimplemented!()` body with an implementation that calls `crate::{symbol_name}`. - - # Type conversions - - Types in `crate::wrapper::` (bindgen-generated, C-compatible layout) are *not* layout-compatible with those in `crate::` (idiomatic Rust). The wrapper must: - - 1. Copy field values from each `crate::wrapper::` argument into a fresh `crate::` value before the call. - 2. Call `crate::{symbol_name}` with the converted values. - 3. Copy result/output values back from `crate::` types into the `crate::wrapper::` types the C ABI expects. - - Use `support_code` to recover the original C types behind opaque or erased Rust types (notably `void*`, `*mut c_void`, and untyped byte buffers) so each field is converted at its true C type. - - # Global synchronization - - If `crate::{symbol_name}` reads or writes globals, synchronize them in the wrapper: - - - Before the call, copy each readable global from the bindgen-generated extern `crate::wrapper::{{var_name}}::{{var_name}}` into the Rust global `crate::{{var_name}}`. - - After the call, copy each writable global from `crate::{{var_name}}` back to `crate::wrapper::{{var_name}}::{{var_name}}`. - - # Raw pointer handling - - - Null-check every raw pointer parameter before its first dereference. On null, return the same value the C function returns for null input (typically an error code, `false`, `-1`, or `null`) — never dereference and panic. Use `support_code` to determine the correct null-input sentinel. - - Treat every mutable raw pointer parameter as in-out unless `support_code` clearly proves it is read-only. - - For every out / in-out pointer parameter, write the converted result back through the raw pointer after the Rust call. Struct and array out-parameters require full field/element write-back. - - Do not route pointer-identity comparisons through detached clones/copies; identity must survive the boundary. - - # Mutable `char*` / byte buffers - - Reproduce C buffer-mutation semantics exactly: - - - If the Rust function computes a normalized or truncated value, write it back into the caller's buffer. - - Classify each pointer+length input by C semantics before conversion: if the C code treats it as a string (`strcmp`, `strlen`, `%s`, token parsing, command dispatch, pattern matching), normalize at ingress by truncating at the first `\0`; if it is a fixed-length or binary buffer, preserve embedded `\0` bytes and use the explicit length. - - Apply the same normalization policy to all operands in the same logical operation. Do not compare a length-decoded string that still contains trailing `\0` against a C-string-decoded operand that was truncated at `\0`. - - When both pointer and length are present for string-style data, use length only as a safety bound for reads; derive semantic content from C string termination and stop at the first `\0`. - - Preserve NUL termination wherever C expects it; never write past the C-implied capacity. - - When C truncates a buffer by writing `'\0'` at a position found by a search (e.g., `strstr`, `strchr`, or a manual scan), the wrapper must re-derive that same offset from the raw buffer and write the NUL byte explicitly — even if the Rust function has internalized the truncation and does not expose the offset. Use `support_code` to recover the exact search function, offset, and capacity assumptions the C original relied on. - - Treat in-place buffer mutations as **primary observable effects**. Callers (and tests) assert them directly; omitting them silently fails every assertion on the buffer regardless of the parsed return values. - - # Panic safety at the ABI boundary - - Wrap every call into Rust code that could panic in `std::panic::catch_unwind`. On a caught panic, return the appropriate C error value for the return type (`0`, `false`, `-1`, `null`, ...). Letting a panic cross an `extern "C"` boundary is undefined behavior and aborts the process in practice. Omit `catch_unwind` only when the called Rust code provably cannot panic. - - # Calling libc / system functions - - The crate already depends on the `libc` crate. When the wrapper needs to call a C standard library or POSIX function (e.g. `fdopen`, `close`, `malloc`, `free`, `memcpy`, `strlen`, `open`, `read`, `write`, `fopen`, `fclose`, ...), call it through the `libc` crate (`unsafe {{ ::libc::fdopen(fd, mode) }}`). - - - **Do not** emit `extern "C" {{ ... }}` (or `unsafe extern "C" {{ ... }}`) blocks declaring libc / POSIX / system functions, and do not emit `#[link(name = "c")]` (or similar) link attributes for libc symbols. - - The only `extern "C"` items permitted in generated wrapper code are those already present in `example_wrapper`; do not introduce any new `extern "C"` items. - - If a needed symbol is not available in `libc`, prefer a safe Rust equivalent from `std` (e.g. `std::ptr`, `std::ffi::CStr`, `std::fs`, `std::io`). If neither is available, do not declare a new foreign function; explain the limitation in your reasoning. - - # Hard constraints - - - Do not relax behavior, skip write-backs, or use placeholder/stub logic. - - Do not declare libc / POSIX functions in `extern "C"` blocks; call them via the `libc` crate. - - # Inputs and feedback - - - `support_code`: the original C source that was translated to Rust. - - `prior_wrapper`: a previous attempt to fix, if any. - - `build_feedback`: errors from `cargo build`. Address them. - - `scope_feedback`: deviations from the `example_wrapper` template. Address them. - """ - - -def generate_unimplemented_wrapper(path: Path, symbol_name: str) -> CodeRust: +def generate_unimplemented_function_wrapper(path: Path, symbol_name: str) -> CodeRust | None: # unsafe extern "C" { # #[link_name = "\u{1}match"] # pub fn match_( @@ -215,6 +148,12 @@ def generate_unimplemented_wrapper(path: Path, symbol_name: str) -> CodeRust: success, output = check_rust( unimplemented_wrapper, flags=["--crate-type", "lib", "--emit", "metadata"] ) + + # Rust does not support C-compatible variadic functions on stable Rust (they require the nightly-only `c_variadic` feature): + # https://github.com/rust-lang/rust/issues/44930 + if "error[E0658]: C-variadic functions are unstable" in output: + return None + if not success: raise ValueError( f"Failed to validate wrapper template for `{symbol_name}`!\nWrapper:\n{unimplemented_wrapper}\nError:\n{output}" @@ -222,100 +161,186 @@ def generate_unimplemented_wrapper(path: Path, symbol_name: str) -> CodeRust: return CodeRust(unimplemented_wrapper) +class TypeWrapperSignature(dspy.Signature): + """ + Generate `c_to_r` and `r_to_c` conversion functions for `{wrapped_crate}::{type_name}` in a hybrid C/Rust build. + + # Goal + + Produce a `wrapper` containing two unsafe Rust synchronization functions: + - `c_to_r`: reads from a C-layout allocation (via `*const`) and produces an idiomatic Rust value. Does **not** take ownership of C memory. + - `r_to_c`: writes an idiomatic Rust value back into an **existing** C-layout allocation (via `*mut`). Does **not** allocate a new C value — it writes in-place. + + These functions will be written to "{wrapper_path}" and used as FFI glue between the C-layout types exposed by `bindgen` and the safe Rust types in `{wrapped_crate}`. + + For pointer fields that form a linked structure (e.g. a linked list), `r_to_c` must walk the Rust chain and the existing C chain in parallel: + - If both sides have a node: write data in-place and recurse. + - If Rust has a node but C does not (Rust function added a node): allocate a new C node via `::libc::malloc(std::mem::size_of::())` — this uses the same allocator as C's `malloc`, so C can safely `free()` it later. Do **not** use `Box::into_raw` for this; `Box` uses Rust's allocator which C cannot `free()`. + - If C has a node but Rust does not (Rust function removed a node): `::libc::free()` the C node and write null. + + # Template + + - Use `example_wrapper` as the starting template for `wrapper`. + - The `()` placeholder types in the `c_to_r` and `r_to_c` signatures **must** be replaced with the actual idiomatic Rust type from `{wrapped_crate}::{type_name}`. Keeping `()` is not a valid implementation. + - Replace the `todo!()` bodies with field-by-field conversion implementations. + - Do **not** add, remove, or rename functions beyond what the template contains. You may add `use` items if needed. + + # Type mapping + + Types in `crate::` (bindgen-generated, C-compatible layout) are structurally equivalent to their C originals but are **not** the same types as those in `{wrapped_crate}::` (idiomatic Rust). Use `support_code` (the original C source) and `crate` (the existing Rust translations) to determine the correct field-by-field mapping. + + - For each field in `crate::{type_name}`, locate the corresponding field in `{wrapped_crate}::{type_name}` and convert its value. + - Primitive numeric fields (`c_int`, `c_uint`, `c_long`, etc.) map to their Rust integer equivalents (`i32`, `u32`, `i64`, etc.) with an `as` cast. + - `*const c_char` / `*mut c_char` fields that represent strings map to `String` or `Option` in `{wrapped_crate}::` — use `std::ffi::CStr` for the conversion. + - Raw pointer fields (`*mut T`, `*const T`) that are nullable map to `Option<...>` in `{wrapped_crate}::` — use `ptr::NonNull` or a null check. + - Nested struct fields: call the corresponding `c_to_r` / `r_to_c` for that field's type if one exists in `other_wrappers`; otherwise convert field-by-field inline. + - Array fields: convert element-by-element. + + # Round-trip correctness + + The two functions must satisfy: given a valid C-layout input `cs`, after calling `r_to_c(&c_to_r(&cs), &mut cs)` the fields of `cs` must equal their original values. Pay special attention to: + - Fields that are zero-initialized in the C layout but have `Default` values in Rust. + - Pointer fields: `null` must round-trip to `null`. For non-null pointers, the **data** at the pointed address must be preserved; the pointer address itself is preserved naturally because `r_to_c` writes in-place. Do **not** use global state (provenance tables, `OnceLock`, `Mutex`, etc.) to track pointer addresses — that is a design smell indicating the wrong approach. + - String fields where the C layout stores a pointer (document if a true round-trip is impossible without re-allocation). + + # Hard constraints + + - `unsafe` is permitted and expected when crossing the C/Rust boundary (e.g., dereferencing raw pointers, reading C-allocated memory). These functions ARE the FFI bridge — the no-unsafe constraint belongs to the safe Rust translation, not here. + - Do not add `extern "C"` items or FFI exports — these are conversion functions only, not ABI entry points. + - Do not use placeholder/stub logic or `todo!()` / `unimplemented!()` in the final output. + + # Round-trip tests + + After the conversion functions, fill in the `#[cfg(test)]` skeleton from `example_wrapper` with at least two tests: + + - `round_trip_zeroed`: construct the simplest valid C-layout input (all primitive fields 0, all pointers null if null is a valid input). Call `let rs = unsafe {{ c_to_r(&cs) }}; unsafe {{ r_to_c(&rs, &mut cs) }};` and assert each field of `cs` equals its original value with field-by-field `assert_eq!`. If a zeroed instance would cause undefined behavior inside the conversion (e.g. a null pointer would be dereferenced), construct instead the simplest valid input and document why zeroed is unsafe. + - `round_trip_nontrivial`: construct an instance with representative non-zero field values that exercise the real conversion path — non-null pointers (via `Box::leak`, a stack address cast, or a small allocation), non-zero integers, and non-trivial sizes. Call `let rs = unsafe {{ c_to_r(&cs) }}; unsafe {{ r_to_c(&rs, &mut cs) }};` and assert each field of `cs` equals its original value with field-by-field `assert_eq!`. This test is especially important for pointer-bearing types where `round_trip_zeroed` does not exercise the pointer path. + - Tests may use `unsafe`. The `todo!()` in the skeleton is not a valid test body — replace it with a real implementation. + + # Inputs and feedback + + - `support_code`: the original C source that was translated to Rust. + - `prior_wrapper`: a previous attempt, if any. + - `build_feedback`: errors from `cargo build`. Address them. + - `scope_feedback`: deviations from the `example_wrapper` template. Address them. + """ + + crate: CodeRust = dspy.InputField() + support_code: CodeC = dspy.InputField() + example_wrapper: CodeRust = dspy.InputField() + prior_wrapper: CodeRust = dspy.InputField() + build_feedback: str = dspy.InputField() + scope_feedback: str = dspy.InputField() + + wrapper: CodeRust = dspy.OutputField() + + +def generate_unimplemented_type_wrapper(path: Path, type_name: str) -> CodeRust: + # bindgen matches `--allowlist-item` against the mangled Rust name and emits that + # name in its output, so the template must reference the mangled name rather than + # the raw C spelling (they differ for names colliding with Rust keywords). + rust_name = mangle(type_name) + + # Get the C-layout struct definition from bindgen so the LLM sees the exact + # field names and types. The () placeholder marks where the LLM should fill + # in the idiomatic Rust type from the wrapped crate. + # Unlike functions/variables, struct types are already visible to bindgen + # without clang_make_bindable_, so we call bindgen directly. + ok, binding, error, _ = run_subprocess( + [ + "bindgen", + "--disable-header-comment", + "--no-doc-comments", + "--no-layout-tests", + "--sort-semantically", + str(path), + "--allowlist-item", + rust_name, + ] + ) + if not ok: + raise ValueError(f"Bindgen failed for `{type_name}` in '{path}'!\nError:\n{error}") + binding = binding.strip() + if not binding: + raise ValueError(f"Bindgen failed to generate a binding for `{type_name}` in '{path}'!") + + # todo!() is intentional here (not unimplemented!()): validate_changes uses the + # presence of unimplemented!() as a marker for allowed-change nodes. Since the + # LLM must replace the () placeholder *signatures* as well as the bodies, we + # use todo!() so that validate_changes returns no scope constraints, letting + # build_feedback from cargo test enforce correctness instead. + template = ( + f"{binding}\n\n" + f"pub unsafe fn c_to_r(_cs: *const {rust_name}) -> () {{\n todo!()\n}}\n\n" + f"pub unsafe fn r_to_c(_rs: &(), _cs: *mut {rust_name}) {{\n todo!()\n}}\n\n" + f"#[cfg(test)]\nmod tests {{\n use super::*;\n\n" + f" #[test]\n fn round_trip_zeroed() {{\n todo!()\n }}\n\n" + f" #[test]\n fn round_trip_nontrivial() {{\n todo!()\n }}\n}}\n" + ) + + ok, template, error, _ = run_subprocess(["rustfmt"], input=template) + if not ok: + raise ValueError(f"rustfmt failed for type wrapper template `{type_name}`!\n{error}") + + success, output = check_rust(template, flags=["--crate-type", "lib", "--emit", "metadata"]) + if not success: + raise ValueError( + f"Failed to validate type wrapper template for `{type_name}`!\nTemplate:\n{template}\nError:\n{output}" + ) + return CodeRust(template) + + +def _default_feedback_fn(wrapper: CodeRust) -> str: + return "" + + +def _default_on_attempt(msg: str, pred: dspy.Prediction) -> None: + pass + + class WrapperGenerator(dspy.Module): def __init__( self, - crate: Crate, + wrapper: type[dspy.Module] = dspy.ChainOfThought, max_iters: int = 5, + cache: Path | None = None, ) -> None: super().__init__() - self.crate = crate + self._wrapper = wrapper self.max_iters = max_iters - self.cache = _init_cache(crate.workspace_root / "cache.db") - - # Make sure wrapper module is in known state (i.e., empty) - self.wrapper_path = crate.rust_src_path.parent / "wrapper.rs" - self.wrapper_path.write_text("") + self.cache = _init_cache(cache) def forward( self, symbol: Symbol, - reference_code: CodeRust, + crate_code: CodeRust, translation: CodeRust, + unimplemented_wrapper: CodeRust, + wrapper_path: Path, + wrapped_crate: str, + other_wrappers: CodeRust = CodeRust(), prior_wrapper: CodeRust | None = None, support_code: CodeC | None = None, + feedback_fn: Callable[[CodeRust], str] | None = None, + on_attempt: Callable[[str, dspy.Prediction], None] | None = None, ) -> dspy.Prediction: - if symbol.is_function and symbol.is_definition: - return self.wrap_function( - symbol, - reference_code, - translation, - prior_wrapper=prior_wrapper, - support_code=support_code, - ) - elif symbol.is_variable: - self.wrap_variable_(symbol) - return dspy.Prediction(success=True) + if feedback_fn is None: + feedback_fn = _default_feedback_fn + if on_attempt is None: + on_attempt = _default_on_attempt + + # Dynamically select signature based on the symbol kind + if symbol.is_type: + sig_cls = TypeWrapperSignature + elif symbol.is_function: + sig_cls = FunctionWrapperSignature else: - raise NotImplementedError - - def wrap_variable_(self, symbol: Symbol): - logger.info(f"Generating wrapper for variable `{symbol.name}` ...") - - # Variable wrappers are just bindings to C symbols - rust_spelling = mangle(symbol.spelling) - wrapper = bindgen(self.crate.c_src_path, symbol.spelling) - symbol_wrapper_path = self.wrapper_path.parent / "wrapper" / f"{rust_spelling}.rs" - symbol_wrapper_path.parent.mkdir(exist_ok=True, parents=True) - symbol_wrapper_path.write_text(str(wrapper)) - self.crate.vcs.add(symbol_wrapper_path) - - success, output = self._build(symbol) - if not success: - raise RuntimeError(f"Failed to build crate!\n{output}") - - # Permanently make variable global - clang_make_global_(self.crate.c_src_path, symbol.spelling) - self.crate.vcs.add(self.crate.c_src_path) - - # Reference symbol wrapper in wrapper module. - with self.wrapper_path.open("a") as f: - f.write(f"pub mod {rust_spelling};\n") - self.crate.vcs.add(self.wrapper_path) + raise ValueError( + f"WrapperGenerator only supports function and type symbols, " + f"got `{symbol.kind}` for `{symbol.name}`" + ) - msg = f"Wrapped variable `{symbol.name}`" - logger.info(msg) - self.crate.vcs.commit(msg) - - def wrap_function( - self, - symbol: Symbol, - reference_code: CodeRust, - translation: CodeRust, - prior_wrapper: CodeRust | None = None, - support_code: CodeC | None = None, - ) -> dspy.Prediction: - # Don't bother wrapping main in binary crates - if symbol.spelling == "main" and self.crate.is_bin: - # Permanently make main function extern - clang_make_extern_(self.crate.c_src_path, symbol.spelling) - self.crate.vcs.add(self.crate.c_src_path) - self.crate.vcs.commit(f"Made function `{symbol.name}` extern") - return dspy.Prediction(success=True) - - logger.info(f"Generating wrapper for function `{symbol.name}` ...") - - # Use bindgen to generate unimplemented wrapper and write to disk to make sure we can actually build - unimplemented_wrapper = generate_unimplemented_wrapper( - self.crate.c_src_path, symbol.spelling - ) - rust_spelling = mangle(symbol.spelling) - symbol_wrapper_path = self.wrapper_path.parent / "wrapper" / f"{rust_spelling}.rs" - symbol_wrapper_path.parent.mkdir(exist_ok=True, parents=True) - symbol_wrapper_path.write_text(str(unimplemented_wrapper)) - success, build_feedback = self._build(symbol) - if not success: - raise RuntimeError(f"The crate does not build!\n\n{build_feedback}") + logger.info(f"Generating wrapper for `{symbol.name}` ...") # Use cache when no prior wrapper if prior_wrapper is None: @@ -325,25 +350,20 @@ def wrap_function( wrapper = None # Generate dynamic signature and module for symbol - signature_class = HybridSignature if not LARGE_PROJECT else Signature - signature = signature_class.with_instructions( - signature_class.instructions.format( + signature = sig_cls.with_instructions( + sig_cls.instructions.format( symbol_name=symbol.spelling, - crate_path=self.crate.rust_src_path.relative_to(self.crate.cargo_toml.parent), - wrapper_path=symbol_wrapper_path.relative_to(self.crate.cargo_toml.parent), + type_name=symbol.spelling, + wrapped_crate=wrapped_crate, + wrapper_path=wrapper_path, ) ) - generate_wrapper = dspy.ChainOfThought(signature) + generate_wrapper = self._wrapper(signature) - # Construct crate context for generate_wrapper and format it - crate = ( - reference_code + translation + self.gather_wrappers(exclude_wrapper=rust_spelling) - ) + # Construct crate context for generate_wrapper + crate = crate_code + translation + other_wrappers - # Try generating wrapper up to max_iter times - msg = "" - success, build_feedback, scope_feedback = False, "", "" - pred = dspy.Prediction() + pred = dspy.Prediction(build_feedback="", scope_feedback="") for i in range(max(self.max_iters, 1)): # Use the wrapper from the prior iteration as feedback for the next iteration if i > 0: @@ -356,8 +376,8 @@ def wrap_function( support_code, unimplemented_wrapper, prior_wrapper, - build_feedback, - scope_feedback, + pred.build_feedback, + pred.scope_feedback, wrapper if i == 0 else None, ) except AdapterParseError: @@ -370,7 +390,7 @@ def wrap_function( # Otherwise attempt again before any build logic continue - # Reset scope feedback + # Scope validation if "wrapper" not in pred or not isinstance(pred.wrapper, CodeRust): wrapper = unimplemented_wrapper scope_feedback = "No wrapper was generated. You must respect the template and instructions **exactly**!" @@ -380,59 +400,29 @@ def wrap_function( scope_feedback = "\n\n".join( validate_changes(wrapper, unimplemented_wrapper).values() ) - # TODO: Check for a single crate function call in scope - - # Write wrapper to disk and check if we build with unsafe code since wrappers can use unsafe code - symbol_wrapper_path.write_text(str(wrapper)) - self.crate.vcs.add(symbol_wrapper_path) - success, build_feedback = self._build(symbol) - success = success and not build_feedback and not scope_feedback - - usage = format_usage(pred) - # Exit early if we build - if success: - # Permanently make function extern - clang_make_extern_(self.crate.c_src_path, symbol.spelling) - self.crate.vcs.add(self.crate.c_src_path) + pred.name = symbol.spelling + pred.bindgen_template = unimplemented_wrapper + pred.prior_wrapper = prior_wrapper or CodeRust() + pred.wrapper = wrapper + pred.scope_feedback = scope_feedback + pred.build_feedback = feedback_fn(wrapper) + pred.success = not pred.scope_feedback and not pred.build_feedback - # Reference successful symbol wrapper in wrapper module - with self.wrapper_path.open("a") as f: - f.write(f"pub mod {rust_spelling};\n") - self.crate.vcs.add(self.wrapper_path) - - # Log and commit success - msg = f"Wrapped function `{symbol.name}`: {usage}" + if pred.success: + msg = f"Wrapped `{symbol.name}`: {format_usage(pred)}" logger.info(msg) - if "reasoning" in pred: - msg += f"\n\n# Reasoning\n{indent(pred.reasoning, ' ')}" - self.crate.vcs.commit(msg) + on_attempt(msg, pred) break - - # Log and commit failure - msg = f"Failed to wrap function `{symbol.name}` ({i + 1}/{self.max_iters}): {usage}" - logger.error(msg) - if "reasoning" in pred: - msg += f"\n\n# Reasoning\n{indent(pred.reasoning, ' ')}" - msg += f"\n\n# Build Feedback\n{indent(build_feedback, ' ')}" - msg += f"\n\n# Scope Feedback\n{indent(scope_feedback, ' ')}" - self.crate.vcs.commit(msg) - - pred.success = success - pred.name = symbol.spelling - pred.wrapper = wrapper - pred.bindgen_template = unimplemented_wrapper - pred.prior_wrapper = prior_wrapper or CodeRust() - pred.build_feedback = build_feedback - pred.scope_feedback = scope_feedback - if not success: - # Feedback for translator - pred.feedback = "It was difficult to generate a C-compatible FFI wrapper for the translation. Regenerate the translation with clear, explicit, wrapper-friendly Rust function boundaries and straightforward ownership, while keeping the translation fully memory-safe and free of unsafe constructs." + else: + msg = f"Failed to wrap `{symbol.name}` ({i + 1}/{self.max_iters}): {format_usage(pred)}" + logger.error(msg) + on_attempt(msg, pred) return pred def generate( self, - generate_wrapper: dspy.ChainOfThought, + generate_wrapper: dspy.Module, crate: CodeRust, support_code: CodeC | None, example_wrapper: CodeRust, @@ -441,7 +431,6 @@ def generate( scope_feedback: str, wrapper: CodeRust | None, ) -> dspy.Prediction: - """Generate a wrapper prediction, using cached wrapper or calling the LLM.""" parent_usage_tracker = settings.usage_tracker if wrapper is not None: pred = dspy.Prediction(wrapper=wrapper) @@ -473,72 +462,6 @@ def generate( parent_usage_tracker.add_usage(lm_name, usage_entry) return pred - def gather_wrappers(self, exclude_wrapper: str = "") -> CodeRust: - wrapper_dir = self.wrapper_path.parent / "wrapper" - if not wrapper_dir.is_dir(): - return CodeRust() - - modules: OrderedDict[str, str] = OrderedDict() - for symbol_wrapper_path in sorted(wrapper_dir.glob("*.rs")): - rust_spelling = symbol_wrapper_path.stem - if exclude_wrapper and rust_spelling == exclude_wrapper: - continue - if rust_spelling in modules: - continue - - wrapper_src = symbol_wrapper_path.read_text().strip() - if not wrapper_src: - continue - - modules[rust_spelling] = ( - f"pub mod {rust_spelling} {{\n" + indent(wrapper_src, " ") + "\n}" - ) - - if not modules: - return CodeRust() - - return CodeRust( - "pub mod wrapper {\n" + indent("\n\n".join(modules.values()), " ") + "\n}\n" - ) - - def _build(self, symbol: Symbol) -> tuple[bool, str]: - orig_c_src = self.crate.c_src_path.read_bytes() - orig_rust_src = self.crate.rust_src_path.read_bytes() - orig_wrapper_src = self.wrapper_path.read_bytes() - - if symbol.is_function: - # Make C function extern so that we use the Rust function definition - clang_make_extern_(self.crate.c_src_path, symbol.spelling) - elif symbol.is_variable: - # Make C variable global so we can reference it in the Rust wrapper - clang_make_global_(self.crate.c_src_path, symbol.spelling) - else: - raise NotImplementedError - self.crate.vcs.add(self.crate.c_src_path) - - # Remove forbid unsafe from Rust source - rust_src = orig_rust_src.decode().replace("#![forbid(unsafe_code)]", "") - - # Reference wrapper module in Rust source - rust_src += "pub mod wrapper;\n" - self.crate.rust_src_path.write_text(rust_src) - self.crate.vcs.add(self.crate.rust_src_path) - - # Reference symbol wrapper module to wrapper module - with self.wrapper_path.open("a") as f: - f.write(f"pub mod {mangle(symbol.spelling)};\n") - self.crate.vcs.add(self.wrapper_path) - - # Check whether all of the changes compile and commit them - success, feedback = self.crate.cargo_build() - - # Restore original source - self.crate.c_src_path.write_bytes(orig_c_src) - self.crate.rust_src_path.write_bytes(orig_rust_src) - self.wrapper_path.write_bytes(orig_wrapper_src) - - return success, feedback - def write_cache(self, pred: dspy.Prediction) -> None: # If prediction was not generated by an LM then don't write it to cache if not pred.get_lm_usage(): diff --git a/test/fixtures/ast/formatting.c.i b/test/fixtures/ast/formatting.c.i deleted file mode 100644 index c075e30..0000000 --- a/test/fixtures/ast/formatting.c.i +++ /dev/null @@ -1,23 +0,0 @@ -# 0 "/home/some/path.c" -# 0 "" - -extern int a; extern int b; -float c[7] = {1.0, 2.0, -3.0, 4.0, 5.0, - 6.0, - 7.0}; - -# 1 "some/other/path.c" 1 3 4 - -void foo() { - int x = 10; int y = 20; - // A comment - int z = 20; - - - /* A comment - block */ - if (z > 15) { z += 5; } else { - z -= 5; - } -} diff --git a/test/fixtures/compile/echo_123 b/test/fixtures/compile/echo_123 deleted file mode 100755 index 83e3b23..0000000 --- a/test/fixtures/compile/echo_123 +++ /dev/null @@ -1,2 +0,0 @@ -#!/bin/sh -echo 1 2 3 diff --git a/test/fixtures/compile/echo_stdin b/test/fixtures/compile/echo_stdin deleted file mode 100755 index 0e96ad1..0000000 --- a/test/fixtures/compile/echo_stdin +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/bash -while IFS='' read -r line || [ "$line" ]; do - echo $line -done diff --git a/test/fixtures/compile/hello_world_bad.c b/test/fixtures/compile/hello_world_bad.c deleted file mode 100644 index 3f7add8..0000000 --- a/test/fixtures/compile/hello_world_bad.c +++ /dev/null @@ -1,6 +0,0 @@ -#include - -int main() { - pprintf("Hello, World!\n"); - return 0; -} diff --git a/test/fixtures/compile/hello_world_bad.rs b/test/fixtures/compile/hello_world_bad.rs deleted file mode 100644 index 933247f..0000000 --- a/test/fixtures/compile/hello_world_bad.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn main() { - pprintln!("Hello, world!"); -} diff --git a/test/fixtures/compile/hello_world_good.c b/test/fixtures/compile/hello_world_good.c deleted file mode 100644 index 0923337..0000000 --- a/test/fixtures/compile/hello_world_good.c +++ /dev/null @@ -1,6 +0,0 @@ -#include - -int main() { - printf("Hello, World!\n"); - return 0; -} diff --git a/test/fixtures/compile/hello_world_good.rs b/test/fixtures/compile/hello_world_good.rs deleted file mode 100644 index e7a11a9..0000000 --- a/test/fixtures/compile/hello_world_good.rs +++ /dev/null @@ -1,3 +0,0 @@ -fn main() { - println!("Hello, world!"); -} diff --git a/test/fixtures/text_processor/Cargo.toml b/test/fixtures/text_processor/Cargo.toml deleted file mode 100644 index 5223392..0000000 --- a/test/fixtures/text_processor/Cargo.toml +++ /dev/null @@ -1,15 +0,0 @@ -[package] -name = "text_processor" -version = "0.1.0" -edition = "2021" - -[[bin]] -name = "text_processor" -path = "src/main.rs" - -[dependencies] - -[dev-dependencies] -assert_cmd = "2.0.17" -predicates = "3.1.3" -ntest = "0.9.3" diff --git a/test/fixtures/text_processor/json_test_cases/test1.json b/test/fixtures/text_processor/json_test_cases/test1.json deleted file mode 100644 index f151d98..0000000 --- a/test/fixtures/text_processor/json_test_cases/test1.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "argv": ["upper"], - "stdin": "Hello World!\n", - "stdout": {"pattern": "HELLO WORLD!\n"}, - "stderr": {"pattern": ""} -} diff --git a/test/fixtures/text_processor/json_test_cases/test2.json b/test/fixtures/text_processor/json_test_cases/test2.json deleted file mode 100644 index 675b609..0000000 --- a/test/fixtures/text_processor/json_test_cases/test2.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "argv": [], - "stdin": "Hello World!\n", - "rc": 1, - "stdout": {"pattern": ""}, - "stderr": {"pattern": "Error: Missing required arguments\nModes: upper, lower, reverse, count\n"} -} diff --git a/test/fixtures/text_processor/json_test_cases/test3.json b/test/fixtures/text_processor/json_test_cases/test3.json deleted file mode 100644 index 7d51156..0000000 --- a/test/fixtures/text_processor/json_test_cases/test3.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "argv": ["upper", " | ", "extra"], - "stdin": "hello\n", - "rc": 1, - "stdout": {"pattern": ""}, - "stderr": {"pattern": "Error: Too many arguments (expected 1-2, got 3)\n"} -} diff --git a/test/fixtures/text_processor/json_test_cases/test4.json b/test/fixtures/text_processor/json_test_cases/test4.json deleted file mode 100644 index c0fd7da..0000000 --- a/test/fixtures/text_processor/json_test_cases/test4.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "argv": ["upper"], - "stdin": "line1\nline2\nline3\n", - "rc": 0, - "stdout": {"pattern": "LINE1\nLINE2\nLINE3\n"}, - "stderr": {"pattern": ""} -} diff --git a/test/fixtures/text_processor/json_test_cases/test5.json b/test/fixtures/text_processor/json_test_cases/test5.json deleted file mode 100644 index 6330aa2..0000000 --- a/test/fixtures/text_processor/json_test_cases/test5.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "argv": ["count"], - "stdin": "", - "stdout": {"pattern": "Words: 0, Characters: 0\n"}, - "stderr": {"pattern": ""} -} diff --git a/test/fixtures/text_processor/json_test_cases/test6.json b/test/fixtures/text_processor/json_test_cases/test6.json deleted file mode 100644 index e6b189b..0000000 --- a/test/fixtures/text_processor/json_test_cases/test6.json +++ /dev/null @@ -1,7 +0,0 @@ -{ - "argv": ["count"], - "stdin": "", - "stdout": {"pattern": "Words: 0, Characters: 0\n"}, - "stderr": {"pattern": ""}, - "has_ub": "This test should be completely skipped" -} diff --git a/test/fixtures/text_processor/src/main.rs b/test/fixtures/text_processor/src/main.rs deleted file mode 100644 index 09a92f2..0000000 --- a/test/fixtures/text_processor/src/main.rs +++ /dev/null @@ -1,62 +0,0 @@ -use std::env; -use std::io::{self, Read}; -use std::process; - -fn main() { - let args: Vec = env::args().collect(); - - // Skip the program name, we need exactly 1 or 2 arguments - let cmd_args = &args[1..]; - - if cmd_args.is_empty() { - eprintln!("Error: Missing required arguments"); - eprintln!("Modes: upper, lower, reverse, count"); - process::exit(1); - } - - if cmd_args.len() > 2 { - eprintln!("Error: Too many arguments (expected 1-2, got {})", cmd_args.len()); - process::exit(1); - } - - let mode = &cmd_args[0]; - let separator = if cmd_args.len() == 2 { - &cmd_args[1] - } else { - "\n" - }; - - // Read from stdin - let mut input = String::new(); - match io::stdin().read_to_string(&mut input) { - Ok(_) => {}, - Err(e) => { - eprintln!("Error reading from stdin: {}", e); - process::exit(1); - } - } - - // Remove trailing newline if present - if input.ends_with('\n') { - input.pop(); - } - - // Process based on mode - let result = match mode.as_str() { - "upper" => input.to_uppercase(), - "lower" => input.to_lowercase(), - "reverse" => input.chars().rev().collect(), - "count" => { - let word_count = input.split_whitespace().count(); - let char_count = input.chars().count(); - format!("Words: {}, Characters: {}", word_count, char_count) - }, - _ => { - eprintln!("Error: Unknown mode '{}'", mode); - eprintln!("Available modes: upper, lower, reverse, count"); - process::exit(1); - } - }; - - print!("{}{}", result, separator); -} diff --git a/test/fixtures/text_processor/tests/test_cases.rs b/test/fixtures/text_processor/tests/test_cases.rs deleted file mode 100644 index 77d30de..0000000 --- a/test/fixtures/text_processor/tests/test_cases.rs +++ /dev/null @@ -1 +0,0 @@ -unimplemented!() diff --git a/test/integration/conftest.py b/test/integration/conftest.py new file mode 100644 index 0000000..5fc1668 --- /dev/null +++ b/test/integration/conftest.py @@ -0,0 +1,252 @@ +# +# Copyright (C) 2026 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +import os +import subprocess +from pathlib import Path + +import dspy +import pytest +from dspy.utils.dummies import DummyLM + +from ideas import adapters +from ideas.translate import TranslateConfig +import ideas.model as model_mod +import ideas.translate as translate_mod + +REPO_ROOT = Path(__file__).resolve().parents[2] +IDEAS_MK = REPO_ROOT / "IDEAS.mk" +TRANSLATION_DIR = "translation.test" +CRATE = "driver" + +BUILD_TARGET = "bear" +INIT_TARGET = "init" + +BUILD_ENV = {**os.environ, "CARGO_NET_OFFLINE": "true", "RUSTFLAGS": "-Awarnings"} +MAKE_ENV = {**os.environ, "UV_PROJECT": str(REPO_ROOT)} + +_MINI_H = """\ +#ifndef MINI_H +#define MINI_H +int add(int a, int b); +int sub(int a, int b); +#endif +""" + +_MINI_C_LIB = """\ +#include "mini.h" + +int add(int a, int b) { + return a + b; +} + +int sub(int a, int b) { + return a - b; +} +""" + +_MINI_C_BIN = """\ +#include "mini.h" + +int add(int a, int b) { + return a + b; +} + +int sub(int a, int b) { + return a - b; +} + +int main(void) { + return add(2, 3) - sub(5, 0); +} +""" + +_CMAKE_LIB = """\ +cmake_minimum_required(VERSION 3.19) +project({crate} C) +add_library({crate} SHARED src/mini.c) +target_include_directories({crate} PUBLIC ${{CMAKE_CURRENT_SOURCE_DIR}}/include) +""" + +_CMAKE_BIN = """\ +cmake_minimum_required(VERSION 3.19) +project({crate} C) +add_executable({crate} src/mini.c) +target_include_directories({crate} PUBLIC ${{CMAKE_CURRENT_SOURCE_DIR}}/include) +""" + +_TRANSLATION = { + "add": "pub fn add(a: i32, b: i32) -> i32 {\n a + b\n}\n", + "sub": "pub fn sub(a: i32, b: i32) -> i32 {\n a - b\n}\n", + "main": "pub fn main() {\n let _ = add(2, 3) - sub(5, 0);\n}\n", +} + + +def get_crate_name(template: str) -> str: + return f"{CRATE}" if template == "bin" else f"lib{CRATE}" + + +@pytest.fixture +def instrumented_workspace(tmp_path): + """ + cmake -> init -> tests/smoke.rs + """ + + def _factory(template: str) -> Path: + instrumented = tmp_path / template + instrumented.mkdir() + _write_project(instrumented, template) + _make(instrumented, BUILD_TARGET) + _make(instrumented, INIT_TARGET) + _make( + instrumented, + f"{TRANSLATION_DIR}/{get_crate_name(template)}-sys/tests/smoke.rs", + ) + return instrumented / TRANSLATION_DIR + + return _factory + + +def _write_project(instrumented: Path, template: str) -> None: + test_case = instrumented / "test_case" + (test_case / "src").mkdir(parents=True, exist_ok=True) + (test_case / "include").mkdir(parents=True, exist_ok=True) + (test_case / "include" / "mini.h").write_text(_MINI_H) + if template == "lib": + (test_case / "src" / "mini.c").write_text(_MINI_C_LIB) + (test_case / "CMakeLists.txt").write_text(_CMAKE_LIB.format(crate=CRATE)) + elif template == "bin": + (test_case / "src" / "mini.c").write_text(_MINI_C_BIN) + (test_case / "CMakeLists.txt").write_text(_CMAKE_BIN.format(crate=CRATE)) + else: + raise ValueError(template) + + +def _make(instrumented: Path, goal: str) -> None: + cmd = [ + "make", + "-C", + str(instrumented), + "-f", + str(IDEAS_MK), + f"TRANSLATION_DIR={TRANSLATION_DIR}", + goal, + ] + proc = subprocess.run(cmd, env=MAKE_ENV, capture_output=True, text=True) + if proc.returncode != 0: + raise RuntimeError( + f"make target {goal!r} failed (rc={proc.returncode}):\n" + f"--- stdout ---\n{proc.stdout}\n--- stderr ---\n{proc.stderr}" + ) + + +def build_ok(workspace: Path, crate: str, *extra: str) -> bool: + cmd = [ + "cargo", + "build", + "--quiet", + "--manifest-path", + str(workspace / "Cargo.toml"), + "-p", + crate, + *extra, + ] + return subprocess.run(cmd, env=BUILD_ENV, capture_output=True, text=True).returncode == 0 + + +def _snippet_field(content: str) -> str: + # DummyLM keys match anywhere in the prompt, but `dependent_code` also carries the + # C code of sibling symbols, so every translation prompt contains every definition. + # Narrow matching to the `snippet` field so each key selects exactly one symbol. + _, marker, rest = content.partition("[[ ## snippet ## ]]") + if not marker: + return content + return rest.partition("[[ ## ")[0] + + +class _SnippetKeyedLM(DummyLM): + def __call__(self, prompt=None, messages=None, **kwargs): + if messages: + last = messages[-1] + messages = [ + *messages[:-1], + {**last, "content": _snippet_field(last["content"])}, + ] + return super().__call__(prompt=prompt, messages=messages, **kwargs) + + +def success_lm() -> DummyLM: + return _SnippetKeyedLM( + { + "return a + b;": { + "reasoning": "trivial translation", + "translation": _TRANSLATION["add"], + }, + "return a - b;": { + "reasoning": "trivial translation", + "translation": _TRANSLATION["sub"], + }, + "int main": { + "reasoning": "trivial translation", + "translation": _TRANSLATION["main"], + }, + }, + adapter=adapters.ChatAdapter(), + ) + + +def error_lm() -> DummyLM: + # The empty key matches every prompt, so every call returns a field the signature + # never declares -> the adapter raises AdapterParseError, which the pipeline treats + # as an irrecoverable DSPy error. + return DummyLM( + {"": {"unexpected_field": "not a translation"}}, adapter=adapters.ChatAdapter() + ) + + +def make_config(workspace: Path, template: str): + crate_name = get_crate_name(template) + return TranslateConfig( + cargo_toml=workspace / crate_name / "Cargo.toml", + bindings_cargo_toml=workspace / f"{crate_name}-sys" / "Cargo.toml", + tests="smoke", + template=template, + translator="ChainOfThought", + translator_max_iters=1, + wrapper="ChainOfThought", + wrapper_max_iters=0, + max_iters=1, + vcs="none", + ) + + +def install_mock(monkeypatch, workspace: Path, lm: dspy.LM, template: str) -> None: + def fake_configure(model, generate): + dspy.configure(lm=lm, track_usage=True) + + monkeypatch.setattr(model_mod, "configure", fake_configure) + output_dir = workspace / get_crate_name(template) + + class _Runtime: + def __init__(self, path: Path): + self.output_dir = str(path) + + class _HydraCfg: + def __init__(self, path: Path): + self.runtime = _Runtime(path) + self.output_subdir = None + + class _HydraConfig: + @staticmethod + def get(): + return _HydraCfg(output_dir) + + monkeypatch.setattr(translate_mod, "HydraConfig", _HydraConfig) + + +def run_translate(monkeypatch, workspace: Path, template: str, lm: dspy.LM) -> None: + install_mock(monkeypatch, workspace, lm, template) + translate_mod._main(make_config(workspace, template)) diff --git a/test/integration/test_final.py b/test/integration/test_final.py new file mode 100644 index 0000000..33c0ce3 --- /dev/null +++ b/test/integration/test_final.py @@ -0,0 +1,63 @@ +# +# Copyright (C) 2026 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +""" +The two terminal states of a translation run: + +* exit after a whole translation with no irrecoverable error, and +* exit after an irrecoverable DSPy error (the hybrid crate is stubbed out). +""" + +import pytest +from conftest import build_ok, error_lm, get_crate_name, run_translate, success_lm + + +@pytest.mark.parametrize("template", ["lib", "bin"]) +def test_crates_buildable_on_exit_after_success(instrumented_workspace, monkeypatch, template): + ws = instrumented_workspace(template) + + run_translate(monkeypatch, ws, template, success_lm()) + + crate_name = get_crate_name(template) + assert build_ok(ws, crate_name) + assert build_ok(ws, f"{crate_name}-rs") + if template == "bin": + # Translating a binary migrates the C `main` to Rust + # so the -sys reference binary (`#![no_main]`) can no longer link + assert not build_ok(ws, f"{crate_name}-sys") + else: + assert build_ok(ws, f"{crate_name}-sys") + assert build_ok( + ws, f"{crate_name}-sys", "--no-default-features", "--features", "cc_ubsan" + ) + assert build_ok( + ws, f"{crate_name}-sys", "--no-default-features", "--features", "cc_asan" + ) + assert build_ok( + ws, f"{crate_name}-sys", "--no-default-features", "--features", "cc_coverage" + ) + + +@pytest.mark.parametrize("template", ["lib", "bin"]) +def test_crates_buildable_on_exit_after_error(instrumented_workspace, monkeypatch, template): + ws = instrumented_workspace(template) + + run_translate(monkeypatch, ws, template, error_lm()) + + crate_name = get_crate_name(template) + if template == "bin": + # Any error leaves the hybrid crate without a buildable binary + # because the app overwrites `main.rs` with an empty file + assert not build_ok(ws, crate_name) + else: + assert build_ok(ws, crate_name) + assert build_ok(ws, f"{crate_name}-rs") + assert build_ok(ws, f"{crate_name}-sys") + assert build_ok(ws, f"{crate_name}-sys", "--no-default-features", "--features", "cc_ubsan") + assert build_ok(ws, f"{crate_name}-sys", "--no-default-features", "--features", "cc_asan") + assert build_ok( + ws, f"{crate_name}-sys", "--no-default-features", "--features", "cc_coverage" + ) diff --git a/test/integration/test_init.py b/test/integration/test_init.py new file mode 100644 index 0000000..ce78650 --- /dev/null +++ b/test/integration/test_init.py @@ -0,0 +1,79 @@ +# +# Copyright (C) 2026 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +import subprocess +import textwrap + +import pytest +from conftest import get_crate_name, build_ok + +from ideas.tools import Crate +from ideas.agents.utils import write_instrumentation_script + + +@pytest.mark.parametrize("template", ["lib", "bin"]) +def test_sys_crate_buildable_after_init(instrumented_workspace, template): + ws = instrumented_workspace(template) + + crate_name = get_crate_name(template) + assert build_ok(ws, f"{crate_name}-sys") + assert build_ok(ws, f"{crate_name}-sys", "--no-default-features", "--features", "cc_ubsan") + assert build_ok(ws, f"{crate_name}-sys", "--no-default-features", "--features", "cc_asan") + assert build_ok( + ws, f"{crate_name}-sys", "--no-default-features", "--features", "cc_coverage" + ) + + +def test_bin_stripped_sys_crate_links_with_sanitizer(instrumented_workspace): + ws = instrumented_workspace("bin") + crate_name = get_crate_name("bin") + src = (ws / f"{crate_name}-sys" / "Cargo.toml").parent / "src" + + # Strip out the library target completely and link to C `main` + (src / "lib.rs").unlink() + (src / "main.rs").write_text("#![no_main]\n") + + assert build_ok(ws, f"{crate_name}-sys") + assert build_ok(ws, f"{crate_name}-sys", "--no-default-features", "--features", "cc_ubsan") + assert build_ok(ws, f"{crate_name}-sys", "--no-default-features", "--features", "cc_asan") + assert build_ok( + ws, f"{crate_name}-sys", "--no-default-features", "--features", "cc_coverage" + ) + + +@pytest.mark.parametrize("template", ["lib", "bin"]) +def test_instrumentation_runs(instrumented_workspace, template): + ws = instrumented_workspace(template) + sys_crate = Crate(ws / f"{get_crate_name(template)}-sys" / "Cargo.toml") + + test_collect = sys_crate.cargo_toml.parent / "tests" / "test_collect.rs" + test_collect.parent.mkdir(parents=True, exist_ok=True) + test_collect.write_text( + textwrap.dedent( + """ + #[test] + fn always_pass() { + assert_eq!(1 + 1, 2); + } + """ + ).strip() + ) + + script = write_instrumentation_script( + sys_crate.cargo_toml.parent / "instrument.sh", features=["cc_asan", "cc_ubsan"] + ) + + proc = subprocess.run( + ["bash", script.name], + cwd=script.parent, + capture_output=True, + text=True, + ) + # The instrumentation script should run successfully + assert proc.returncode == 0, ( + f"instrumentation script failed (rc={proc.returncode}):\n" + f"--- stdout ---\n{proc.stdout}\n--- stderr ---\n{proc.stderr}" + ) diff --git a/test/test_bear.py b/test/test_bear.py new file mode 100644 index 0000000..b7c8cf3 --- /dev/null +++ b/test/test_bear.py @@ -0,0 +1,407 @@ +# +# Copyright (C) 2026 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +from pathlib import Path + +from ideas.bear import BuildDatabase, CompileCommand, LinkCommand + + +def test_resolve_targets_sphincs_build(): + """ + Models the sphincs-plus build where utils.c is compiled twice: + - For sphincs_obj (app/): with -DBLAKE_TR=1 + - For blake (lib/blake/): without -DBLAKE_TR=1 + The driver links against both, so resolve_targets must dedup utils.c and + pick the sphincs version (first encountered in link order). + """ + build = Path("/build/test_case") + src_app = Path("/src/test_case/app/src") + src_blake = Path("/src/test_case/lib/blake/src") + + utils_c = src_app / "utils.c" + sign_c = src_app / "sign.c" + rng_c = src_app / "rng.c" + pqcgen_c = src_app / "PQCgenKAT_sign.c" + hash_blake_c = src_blake / "hash_blake.c" + + sphincs_dir = build / "app/CMakeFiles/sphincs_obj.dir/src" + blake_dir = build / "lib/blake/CMakeFiles/blake.dir" + driver_dir = build / "app/CMakeFiles/driver.dir/src" + + sphincs_flags = ["-DPARAMS=sphincs-blake-128f", "-DBLAKE_TR=1", "-w", "-O3", "-std=gnu99"] + blake_flags = ["-DPARAMS=sphincs-blake-128f", "-w", "-O3", "-std=gnu99"] + + libblake = build / "lib/blake/libblake.so" + libsphincs = build / "app/libsphincs_core_det.so" + driver = build / "app/driver" + + compile_commands = [ + # sphincs_obj objects — compiled with -DBLAKE_TR=1 + CompileCommand.from_arguments( + ["clang", *sphincs_flags, "-c", str(sign_c), "-o", str(sphincs_dir / "sign.c.o")], + working_dir=build, + ), + CompileCommand.from_arguments( + ["clang", *sphincs_flags, "-c", str(utils_c), "-o", str(sphincs_dir / "utils.c.o")], + working_dir=build, + ), + CompileCommand.from_arguments( + ["clang", *sphincs_flags, "-c", str(rng_c), "-o", str(sphincs_dir / "rng.c.o")], + working_dir=build, + ), + # blake objects — compiled WITHOUT -DBLAKE_TR=1 + CompileCommand.from_arguments( + [ + "clang", + *blake_flags, + "-c", + str(hash_blake_c), + "-o", + str(blake_dir / "src/hash_blake.c.o"), + ], + working_dir=build, + ), + CompileCommand.from_arguments( + [ + "clang", + *blake_flags, + "-c", + str(utils_c), + "-o", + str(blake_dir / "__/__/app/src/utils.c.o"), + ], + working_dir=build, + ), + # driver object + CompileCommand.from_arguments( + [ + "clang", + *blake_flags, + "-c", + str(pqcgen_c), + "-o", + str(driver_dir / "PQCgenKAT_sign.c.o"), + ], + working_dir=build, + ), + ] + + link_commands = [ + LinkCommand.from_arguments( + [ + "clang", + "-shared", + "-o", + str(libblake), + str(blake_dir / "src/hash_blake.c.o"), + str(blake_dir / "__/__/app/src/utils.c.o"), + ], + working_dir=build, + ), + LinkCommand.from_arguments( + [ + "clang", + "-shared", + "-o", + str(libsphincs), + str(sphincs_dir / "sign.c.o"), + str(sphincs_dir / "utils.c.o"), + str(sphincs_dir / "rng.c.o"), + ], + working_dir=build, + ), + LinkCommand.from_arguments( + [ + "clang", + "-o", + str(driver), + str(driver_dir / "PQCgenKAT_sign.c.o"), + str(libsphincs), + str(libblake), + "-lcrypto", + ], + working_dir=build, + ), + ] + + db = BuildDatabase( + compile_commands=[c for c in compile_commands if c is not None], + link_commands=[lc for lc in link_commands if lc is not None], + ) + targets = db.resolve_targets() + + assert set(targets) == {"libblake.so", "libsphincs_core_det.so", "driver"} + + # libblake.so: blake sources only, no -DBLAKE_TR=1 on utils.c + blake_sources = [e.source for e in targets["libblake.so"].entries] + assert blake_sources == [hash_blake_c, utils_c] + assert "-DBLAKE_TR=1" not in next( + e.arguments for e in targets["libblake.so"].entries if e.source == utils_c + ) + assert targets["libblake.so"].link_libs == [] + + # libsphincs_core_det.so: sphincs sources, utils.c has -DBLAKE_TR=1 + sphincs_sources = [e.source for e in targets["libsphincs_core_det.so"].entries] + assert sphincs_sources == [sign_c, utils_c, rng_c] + assert "-DBLAKE_TR=1" in next( + e.arguments for e in targets["libsphincs_core_det.so"].entries if e.source == utils_c + ) + assert targets["libsphincs_core_det.so"].link_libs == [] + + # driver: transitive sources in order; utils.c appears exactly once (sphincs version) + driver_sources = [e.source for e in targets["driver"].entries] + assert driver_sources == [pqcgen_c, sign_c, utils_c, rng_c, hash_blake_c] + assert driver_sources.count(utils_c) == 1 + assert "-DBLAKE_TR=1" in next( + e.arguments for e in targets["driver"].entries if e.source == utils_c + ) + assert targets["driver"].link_libs == ["crypto"] + + +def test_compile_command_strips_dep_tracking_flags(): + # CMake injects -MD, -MF, -MT (and -MMD, -MP) into every compile command. + # These must be stripped so libclang doesn't try to write .d files at + # relative paths that don't exist during analysis. + build = Path("/build") + src = Path("/src/foo.c") + obj = build / "foo.c.o" + dep = build / "foo.c.o.d" + + cmd = CompileCommand.from_arguments( + [ + "clang", + "-I/usr/include", + "-DFOO=1", + "-MD", + "-MMD", + "-MP", + "-MG", + "-MF", + str(dep), + "-MT", + str(obj), + "-MQ", + str(obj), + "-c", + str(src), + "-o", + str(obj), + ], + working_dir=build, + ) + + assert cmd is not None + assert cmd.source == src + assert cmd.output == obj + # None of the dep-tracking flags or their arguments should survive + assert "-MD" not in cmd.arguments + assert "-MMD" not in cmd.arguments + assert "-MP" not in cmd.arguments + assert "-MG" not in cmd.arguments + assert "-MF" not in cmd.arguments + assert "-MT" not in cmd.arguments + assert "-MQ" not in cmd.arguments + assert str(dep) not in cmd.arguments + # Unrelated flags are preserved + assert "-I/usr/include" in cmd.arguments + assert "-DFOO=1" in cmd.arguments + + +def test_compile_command_versioned_compiler_recognized(): + build = Path("/build") + src = Path("/src/foo.c") + for compiler in ("clang-21", "gcc-13", "clang-3.8"): + cmd = CompileCommand.from_arguments( + [compiler, "-c", str(src), "-o", str(build / "foo.c.o")], + working_dir=build, + ) + assert cmd is not None, f"{compiler} should be recognized" + + +def test_compile_command_non_compiler_returns_none(): + build = Path("/build") + for executable in ("cmake", "make", "sh", "python3", "ar"): + result = CompileCommand.from_arguments( + [executable, "-c", "/src/foo.c", "-o", "/build/foo.c.o"], + working_dir=build, + ) + assert result is None, f"{executable} should not be recognized as a compiler" + + +def test_compile_command_no_output_flag_returns_none(): + build = Path("/build") + result = CompileCommand.from_arguments( + ["clang", "-c", "/src/foo.c"], # no -o + working_dir=build, + ) + assert result is None + + +def test_compile_command_unrecognized_source_extension_returns_none(): + build = Path("/build") + result = CompileCommand.from_arguments( + ["clang", "-c", "/src/foo.txt", "-o", str(build / "foo.txt.o")], + working_dir=build, + ) + assert result is None + + +def test_link_command_so_version_stripped(): + build = Path("/build") + for versioned, expected in [ + ("libfoo.so.1.2.3", "libfoo.so"), + ("libbar.so.0", "libbar.so"), + ("libbaz.so", "libbaz.so"), # no version suffix — unchanged + ]: + cmd = LinkCommand.from_arguments( + ["clang", "-shared", "-o", str(build / versioned), str(build / "foo.c.o")], + working_dir=build, + ) + assert cmd is not None + assert cmd.target == expected, f"{versioned} → expected {expected}, got {cmd.target}" + + +def test_link_command_compile_only_returns_none(): + build = Path("/build") + result = LinkCommand.from_arguments( + ["clang", "-c", "/src/foo.c", "-o", str(build / "foo.c.o")], + working_dir=build, + ) + assert result is None + + +def test_link_command_no_object_inputs_returns_none(): + # Pure -l link with no .o files — not captured as a link command + build = Path("/build") + result = LinkCommand.from_arguments( + ["clang", "-o", str(build / "mybin"), "-lfoo", "-lbar"], + working_dir=build, + ) + assert result is None + + +def test_link_command_versioned_linker_recognized(): + build = Path("/build") + for linker in ("clang-21", "gcc-13", "ld.bfd", "ld.lld"): + cmd = LinkCommand.from_arguments( + [linker, "-o", str(build / "mybin"), str(build / "foo.c.o")], + working_dir=build, + ) + assert cmd is not None, f"{linker} should be recognized as a linker" + + +def test_resolve_targets_static_archive_traversed(): + # A .a archive in linked_binary_inputs should be transitively traversed + # just like a .so, as long as it has a corresponding link command. + build = Path("/build") + src = Path("/src") + + lib_c_o = build / "lib.c.o" + libfoo_a = build / "libfoo.a" + main_c_o = build / "main.c.o" + mybin = build / "mybin" + + compile_cmds = [ + CompileCommand.from_arguments( + ["clang", "-c", str(src / "lib.c"), "-o", str(lib_c_o)], + working_dir=build, + ), + CompileCommand.from_arguments( + ["clang", "-c", str(src / "main.c"), "-o", str(main_c_o)], + working_dir=build, + ), + ] + link_cmds = [ + LinkCommand.from_arguments( + ["clang", "-r", "-o", str(libfoo_a), str(lib_c_o)], + working_dir=build, + ), + LinkCommand.from_arguments( + ["clang", "-o", str(mybin), str(main_c_o), str(libfoo_a)], + working_dir=build, + ), + ] + db = BuildDatabase( + compile_commands=[c for c in compile_cmds if c is not None], + link_commands=[lc for lc in link_cmds if lc is not None], + ) + targets = db.resolve_targets() + assert "mybin" in targets + assert [e.source for e in targets["mybin"].entries] == [src / "main.c", src / "lib.c"] + + +def test_resolve_targets_external_so_silently_skipped(): + # A .so passed directly to the linker but not produced by any link command + # in the database (e.g. libssl.so) should be skipped without error. + build = Path("/build") + src = Path("/src") + + main_o = build / "main.c.o" + external_so = Path("/usr/lib/libssl.so") + + compile_cmds = [ + CompileCommand.from_arguments( + ["clang", "-c", str(src / "main.c"), "-o", str(main_o)], + working_dir=build, + ), + ] + link_cmds = [ + LinkCommand.from_arguments( + ["clang", "-o", str(build / "mybin"), str(main_o), str(external_so)], + working_dir=build, + ), + ] + db = BuildDatabase( + compile_commands=[c for c in compile_cmds if c is not None], + link_commands=[lc for lc in link_cmds if lc is not None], + ) + targets = db.resolve_targets() + assert "mybin" in targets + assert [e.source for e in targets["mybin"].entries] == [src / "main.c"] + + +def test_resolve_targets_versioned_so_traversed(): + # When a library is produced as libfoo.so.1.2.3 but a consuming target links + # against libfoo.so (the unversioned symlink name), resolve_targets should still + # traverse the library's sources transitively. + build = Path("/build") + src = Path("/src") + + lib_c_o = build / "lib.c.o" + libfoo_versioned = build / "libfoo.so.1.2.3" # actual linker output + libfoo_unversioned = build / "libfoo.so" # symlink name used by consumer + main_c_o = build / "main.c.o" + + compile_cmds = [ + CompileCommand.from_arguments( + ["clang", "-c", str(src / "lib.c"), "-o", str(lib_c_o)], + working_dir=build, + ), + CompileCommand.from_arguments( + ["clang", "-c", str(src / "main.c"), "-o", str(main_c_o)], + working_dir=build, + ), + ] + link_cmds = [ + # Produces libfoo.so.1.2.3 — output_path key in binary_source_map + LinkCommand.from_arguments( + ["clang", "-shared", "-o", str(libfoo_versioned), str(lib_c_o)], + working_dir=build, + ), + # Links against libfoo.so — unversioned symlink name + LinkCommand.from_arguments( + ["clang", "-o", str(build / "driver"), str(main_c_o), str(libfoo_unversioned)], + working_dir=build, + ), + ] + db = BuildDatabase( + compile_commands=[c for c in compile_cmds if c is not None], + link_commands=[lc for lc in link_cmds if lc is not None], + ) + targets = db.resolve_targets() + + assert "driver" in targets + assert [e.source for e in targets["driver"].entries] == [src / "main.c", src / "lib.c"] diff --git a/test/test_cargo_test.py b/test/test_cargo_test.py deleted file mode 100644 index 00a85e7..0000000 --- a/test/test_cargo_test.py +++ /dev/null @@ -1,138 +0,0 @@ -# -# Copyright (C) 2026 Intel Corporation -# -# SPDX-License-Identifier: Apache-2.0 -# - -from pathlib import Path - -import pytest - -from ideas import tools - - -@pytest.fixture -def tmp_crate(tmp_path: Path): - """Create a minimal lib crate with passing and failing integration tests.""" - crate_dir = tmp_path / "test_crate" - crate = tools.Crate(cargo_toml=crate_dir / "Cargo.toml", vcs="none", template="lib") - - (crate_dir / "src" / "lib.rs").write_text("pub fn add(a: i32, b: i32) -> i32 { a + b }\n") - - tests_dir = crate_dir / "tests" - tests_dir.mkdir(exist_ok=True) - (tests_dir / "test_pass.rs").write_text( - "use test_crate::add;\n" - "#[test] fn pass_one() { assert_eq!(add(1, 2), 3); }\n" - "#[test] fn pass_two() { assert_eq!(add(0, 0), 0); }\n" - ) - (tests_dir / "test_mixed.rs").write_text( - "use test_crate::add;\n" - "#[test] fn mixed_pass() { assert_eq!(add(1, 1), 2); }\n" - "#[test] fn mixed_fail() { assert_eq!(add(1, 1), 99); }\n" - ) - (tests_dir / "test_ignored.rs").write_text( - "#[test] fn runs() { assert!(true); }\n" - '#[test] #[ignore] fn skipped() { panic!("should not run"); }\n' - ) - (tests_dir / "test_stdout.rs").write_text( - '#[test] fn noisy() { println!("hello from test"); assert!(true); }\n' - ) - - return crate - - -def _sorted_output(output: str) -> str: - """Sort test result lines for deterministic comparison (nextest runs in parallel).""" - lines = output.splitlines() - test_lines = sorted(line for line in lines if line.startswith("test ") and "..." in line) - rest = [line for line in lines if not (line.startswith("test ") and "..." in line)] - return "\n".join(test_lines + rest) + "\n" - - -# --- cargo nextest run harness --- - - -def test_passing_json(tmp_crate): - success, stdout, _, rc = tmp_crate.cargo_test( - name="test_pass", message_format="libtest-json" - ) - assert success is True - assert rc == 0 - assert _sorted_output(tools.nextest_json_to_libtest(stdout)) == ( - "test pass_one ... ok\n" - "test pass_two ... ok\n" - "test result: ok. 2 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out\n" - ) - - -def test_failing_json(tmp_crate): - success, stdout, _, rc = tmp_crate.cargo_test( - name="test_mixed", message_format="libtest-json" - ) - assert success is False - assert rc == 100 - assert _sorted_output(tools.nextest_json_to_libtest(stdout)) == ( - "test mixed_fail ... FAILED\n" - "test mixed_pass ... ok\n" - "test result: FAILED. 1 passed; 1 failed; 0 ignored; 0 measured; 0 filtered out\n" - ) - - -def test_ignored_json(tmp_crate): - success, stdout, _, rc = tmp_crate.cargo_test( - name="test_ignored", message_format="libtest-json" - ) - assert success is True - assert rc == 0 - assert _sorted_output(tools.nextest_json_to_libtest(stdout)) == ( - "test runs ... ok\n" - "test result: ok. 1 passed; 0 failed; 1 ignored; 0 measured; 0 filtered out\n" - ) - - -def test_stdout_not_in_output(tmp_crate): - success, stdout, _, rc = tmp_crate.cargo_test( - name="test_stdout", message_format="libtest-json" - ) - assert success is True - assert rc == 0 - assert _sorted_output(tools.nextest_json_to_libtest(stdout)) == ( - "test noisy ... ok\n" - "test result: ok. 1 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out\n" - ) - - -# --- cargo test harness --- - - -def test_cargo_test_passing(tmp_crate): - success, stdout, _, rc = tmp_crate.cargo_test( - name="test_pass", test_harness="test", quiet=False - ) - assert success is True - assert rc == 0 - assert "test pass_one ... ok" in stdout - assert "test pass_two ... ok" in stdout - assert "2 passed; 0 failed" in stdout - - -def test_cargo_test_failing(tmp_crate): - success, stdout, _, rc = tmp_crate.cargo_test( - name="test_mixed", test_harness="test", quiet=False - ) - assert success is False - assert rc == 101 - assert "test mixed_pass ... ok" in stdout - assert "test mixed_fail ... FAILED" in stdout - assert "1 passed; 1 failed" in stdout - - -def test_cargo_test_ignored(tmp_crate): - success, stdout, _, rc = tmp_crate.cargo_test( - name="test_ignored", test_harness="test", quiet=False - ) - assert success is True - assert rc == 0 - assert "test runs ... ok" in stdout - assert "1 passed; 0 failed; 1 ignored" in stdout diff --git a/test/test_cmake.py b/test/test_cmake.py deleted file mode 100644 index 348e0ba..0000000 --- a/test/test_cmake.py +++ /dev/null @@ -1,113 +0,0 @@ -# -# Copyright (C) 2026 Intel Corporation -# -# SPDX-License-Identifier: Apache-2.0 -# - -import json -from pathlib import Path - -from ideas.cmake import _normalize_isystem - - -def test_normalize_isystem_command_space_separated(tmp_path: Path): - """Replaces '-isystem /path' with '-I /path' in command strings.""" - db = [ - {"directory": "/build", "command": "cc -isystem /usr/include -c foo.c", "file": "foo.c"} - ] - p = tmp_path / "compile_commands.json" - p.write_text(json.dumps(db)) - - _normalize_isystem(p) - - result = json.loads(p.read_text()) - assert result[0]["command"] == "cc -I /usr/include -c foo.c" - - -def test_normalize_isystem_command_no_space(tmp_path: Path): - """Replaces '-isystem/path' with '-I/path' in command strings.""" - db = [ - {"directory": "/build", "command": "cc -isystem/usr/include -c foo.c", "file": "foo.c"} - ] - p = tmp_path / "compile_commands.json" - p.write_text(json.dumps(db)) - - _normalize_isystem(p) - - result = json.loads(p.read_text()) - assert result[0]["command"] == "cc -I/usr/include -c foo.c" - - -def test_normalize_isystem_arguments_space_separated(tmp_path: Path): - """Replaces '-isystem' followed by path in arguments array.""" - db = [ - { - "directory": "/build", - "arguments": ["cc", "-isystem", "/usr/include", "-c", "foo.c"], - "file": "foo.c", - } - ] - p = tmp_path / "compile_commands.json" - p.write_text(json.dumps(db)) - - _normalize_isystem(p) - - result = json.loads(p.read_text()) - assert result[0]["arguments"] == ["cc", "-I", "/usr/include", "-c", "foo.c"] - - -def test_normalize_isystem_arguments_joined(tmp_path: Path): - """Replaces '-isystem/path' in arguments array.""" - db = [ - { - "directory": "/build", - "arguments": ["cc", "-isystem/usr/include", "-c", "foo.c"], - "file": "foo.c", - } - ] - p = tmp_path / "compile_commands.json" - p.write_text(json.dumps(db)) - - _normalize_isystem(p) - - result = json.loads(p.read_text()) - assert result[0]["arguments"] == ["cc", "-I/usr/include", "-c", "foo.c"] - - -def test_normalize_isystem_multiple_entries(tmp_path: Path): - """Handles multiple entries and multiple -isystem flags per entry.""" - db = [ - { - "directory": "/build", - "command": "cc -isystem /a -isystem /b -c foo.c", - "file": "foo.c", - }, - {"directory": "/build", "command": "cc -I/c -c bar.c", "file": "bar.c"}, - ] - p = tmp_path / "compile_commands.json" - p.write_text(json.dumps(db)) - - _normalize_isystem(p) - - result = json.loads(p.read_text()) - assert result[0]["command"] == "cc -I /a -I /b -c foo.c" - assert result[1]["command"] == "cc -I/c -c bar.c" - - -def test_normalize_isystem_no_isystem(tmp_path: Path): - """No-op when there are no -isystem flags.""" - db = [{"directory": "/build", "command": "cc -I/usr/include -c foo.c", "file": "foo.c"}] - p = tmp_path / "compile_commands.json" - p.write_text(json.dumps(db)) - - _normalize_isystem(p) - - result = json.loads(p.read_text()) - assert result[0]["command"] == "cc -I/usr/include -c foo.c" - - -def test_normalize_isystem_missing_file(tmp_path: Path): - """No-op when compile_commands.json does not exist.""" - p = tmp_path / "compile_commands.json" - _normalize_isystem(p) # should not raise - assert not p.exists() diff --git a/test/test_consolidate.py b/test/test_consolidate.py index 7118360..c843c55 100644 --- a/test/test_consolidate.py +++ b/test/test_consolidate.py @@ -4,18 +4,31 @@ # SPDX-License-Identifier: Apache-2.0 # +import pytest + from pathlib import Path from textwrap import dedent +from itertools import permutations import json from ideas import ast -from ideas.init.consolidate import ( - create_ast_order, - create_symbol_lexical_key_fn, - init as consolidate_init, -) -from ideas.tools import check_c +from ideas.ast import CodeC, create_symbol_lexical_key_fn +from ideas.consolidate import create_ast_order, analyze, consolidate +from ideas.tools import run_subprocess + + +def compile_c(code: CodeC, flags: list[str]) -> tuple[bool, str]: + cmd = ["clang-21", *flags, "-march=native", "-c", "-x", "c", "-", "-o", "/dev/null"] + success, output, error, _ = run_subprocess(cmd, input=str(code)) + return success, output + error + + +def consolidate_init( + compile_commands: Path, source_priority: list[Path] | None = None +) -> CodeC: + symbols, symbol_order = analyze(compile_commands, source_priority or []) + return consolidate(symbols, symbol_order) def _usr_by_spelling(symbols: dict[str, ast.Symbol], spelling: str) -> str: @@ -43,6 +56,41 @@ def _write_compile_commands(tmp_path: Path, c_files: list[Path], extra_flags: st return compile_commands +def _build_tus( + base: Path, sources: dict[str, str], cflags: list[str] | None = None +) -> list[Path]: + """Materialize `sources` under `base`; return its translation units, sorted by name. + + Each .c is also checked to compile standalone. Those checks are preconditions on the + test *inputs*: they say nothing about consolidation and are invariant under + compile-command order, so they belong here rather than in each parametrized test. A + broken input then surfaces as an ERROR rather than a FAILURE. + """ + for name, code in sources.items(): + path = base / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(dedent(code)) + + tus = sorted((base / n for n in sources if n.endswith(".c")), key=lambda p: p.name) + for tu in tus: + ok, out, err, _ = run_subprocess( + [ + "clang-21", + "-Wall", + "-Werror", + f"-I{base}", + *(cflags or []), + "-c", + str(tu), + "-o", + str(tu.with_suffix(".o")), + ] + ) + assert ok, f"{tu.name} does not compile standalone:\n{out}{err}" + + return tus + + def _ast_order_from_symbols( symbols: dict[str, ast.Symbol], source_priority: list[Path] | None = None ) -> dict[Path, ast.TreeResult]: @@ -206,7 +254,7 @@ def test_consolidation_places_typedef_before_struct_definition(tmp_path: Path): types_h.write_text( dedent( """\ - typedef struct X X; + typedef struct not_renamed not_renamed; """ ) ) @@ -215,12 +263,12 @@ def test_consolidation_places_typedef_before_struct_definition(tmp_path: Path): """\ #include "types.h" - struct X { - int (*notify)(X *self, int status); + struct not_renamed { + int (*notify)(not_renamed *self, int status); void *payload; }; - int X_init(X *out); + int not_renamed_init(not_renamed *out); """ ) ) @@ -229,7 +277,7 @@ def test_consolidation_places_typedef_before_struct_definition(tmp_path: Path): """\ #include "thing.h" - int X_init(X *out) { + int not_renamed_init(not_renamed *out) { out->notify = 0; out->payload = 0; return 0; @@ -238,152 +286,164 @@ def test_consolidation_places_typedef_before_struct_definition(tmp_path: Path): ) ) + # Fully compile the TU to an object file (resolves thing.h/types.h). + ok, out, err, _ = run_subprocess( + [ + "clang-21", + "-Wall", + "-Werror", + f"-I{tmp_path}", + "-c", + str(thing_c), + "-o", + str(tmp_path / "thing.o"), + ] + ) + assert ok, f"thing.c failed to compile:\n{out}{err}" + # Parse the C file (which transitively includes types.h via thing.h) compile_commands = _write_compile_commands(tmp_path, [thing_c]) consolidated = consolidate_init(compile_commands, source_priority=[]) - success, error = check_c(consolidated, flags=["-fsyntax-only", "-Wall"]) + text = str(consolidated) + assert "not_renamed_init" in text + assert "thing_not_renamed" not in text and "types_not_renamed" not in text + + success, error = compile_c(consolidated, flags=["-Wall", "-Werror"]) assert success, ( f"Consolidated code does not compile:\n{error}\n\nConsolidated output:\n{consolidated}" ) -def test_consolidation_typedef_before_struct_cross_tu(tmp_path: Path): - types_h = tmp_path / "types.h" - node_h = tmp_path / "node.h" - api_c = tmp_path / "api.c" - internal_c = tmp_path / "internal.c" - - types_h.write_text( - dedent( - """\ - typedef struct Node Node; - """ - ) - ) - node_h.write_text( - dedent( - """\ - struct Node { - int val; - struct Node *next; - }; - """ - ) +@pytest.fixture(scope="module") +def cross_tu_typedef_tus(tmp_path_factory: pytest.TempPathFactory) -> list[Path]: + return _build_tus( + tmp_path_factory.mktemp("cross_tu_typedef"), + { + "types.h": """\ + typedef struct not_renamed not_renamed; + """, + "node.h": """\ + struct not_renamed { + int val; + struct not_renamed *next; + }; + """, + "a.c": """\ + #include "node.h" + + int node_get_val(struct not_renamed *n) { + return n->val; + } + """, + "b.c": """\ + #include "types.h" + #include "node.h" + + not_renamed *node_create(int val) { + (void)val; + return (not_renamed *)0; + } + """, + }, ) - api_c.write_text( - dedent( - """\ - #include "types.h" - #include "node.h" - Node *node_create(int val) { - (void)val; - return (Node *)0; - } - """ - ) - ) - internal_c.write_text( - dedent( - """\ - #include "node.h" - int node_get_val(struct Node *n) { - return n->val; - } - """ - ) - ) +@pytest.mark.parametrize("order", permutations(["a.c", "b.c"]), ids="-".join) +def test_consolidation_typedef_before_struct_cross_tu( + cross_tu_typedef_tus: list[Path], order: tuple[str, ...] +): + # After merge_symbols, the struct may retain its cursor from one TU and the typedef + # from another, making cross-TU location comparison undefined. The result must not + # depend on the order the TUs appear in compile_commands.json. + a_c, b_c = cross_tu_typedef_tus + by_name = {tu.name: tu for tu in cross_tu_typedef_tus} - # Parse both TUs — after merge_symbols, the struct may retain its cursor from - # one TU and the typedef from another, making cross-TU location comparison undefined. - compile_commands = _write_compile_commands(tmp_path, [internal_c, api_c]) + compile_commands = _write_compile_commands(a_c.parent, [by_name[n] for n in order]) consolidated = consolidate_init( - compile_commands, source_priority=[internal_c.resolve(), api_c.resolve()] + compile_commands, source_priority=[a_c.resolve(), b_c.resolve()] ) - # The typedef must appear before usages of 'Node' as a bare type name. - success, error = check_c(consolidated, flags=["-fsyntax-only", "-Wall"]) + text = str(consolidated) + assert "not_renamed" in text + assert "a_not_renamed" not in text and "b_not_renamed" not in text, consolidated + + success, error = compile_c(consolidated, flags=["-Wall", "-Werror"]) assert success, ( f"Consolidated code does not compile:\n{error}\n\nConsolidated output:\n{consolidated}" ) -def test_consolidation_mutual_cross_tu_typedefs(tmp_path: Path): +@pytest.fixture(scope="module") +def mutual_typedef_tus(tmp_path_factory: pytest.TempPathFactory) -> list[Path]: + return _build_tus( + tmp_path_factory.mktemp("mutual_typedef"), + { + "a_types.h": """\ + typedef struct not_renamed_a not_renamed_a; + """, + "b_types.h": """\ + typedef struct not_renamed_b not_renamed_b; + """, + "a.c": """\ + #include "a_types.h" + #include "b_types.h" + + struct not_renamed_a { + not_renamed_b *ref; + int val; + }; + + not_renamed_a *create_a(void) { + return (not_renamed_a *)0; + } + """, + "b.c": """\ + #include "a_types.h" + #include "b_types.h" + + struct not_renamed_b { + not_renamed_a *ref; + int val; + }; + + not_renamed_b *create_b(void) { + return (not_renamed_b *)0; + } + """, + }, + ) + + +@pytest.mark.parametrize("order", permutations(["a.c", "b.c"]), ids="-".join) +def test_consolidation_mutual_cross_tu_typedefs( + mutual_typedef_tus: list[Path], order: tuple[str, ...] +): """ Mutual cross-references create a cycle that merges symbols from different TUs into one SCC: - - a_types.h: typedef struct A A; - - b_types.h: typedef struct B B; - - a.c: includes both, defines struct A { B *ref; }; + function using A - - b.c: includes both, defines struct B { A *ref; }; + function using B - - After merge: struct A (from a.c) → typedef B → struct B (from b.c) → typedef A → struct A - All 4 in one SCC with cross-TU cursors. clang_isBeforeInTranslationUnit is - undefined across TUs, so the comparator must still produce compilable output. + - a_types.h: typedef struct not_renamed_a not_renamed_a; + - b_types.h: typedef struct not_renamed_b not_renamed_b; + - a.c: includes both, defines struct not_renamed_a { not_renamed_b *ref; }; + - b.c: includes both, defines struct not_renamed_b { not_renamed_a *ref; }; + + After merge the two structs and two typedefs form one SCC with cross-TU cursors. + clang_isBeforeInTranslationUnit is undefined across TUs, so the comparator must + still produce compilable output. Neither type collides, so nothing is renamed. """ - a_types_h = tmp_path / "a_types.h" - b_types_h = tmp_path / "b_types.h" - a_c = tmp_path / "a.c" - b_c = tmp_path / "b.c" + a_c, b_c = mutual_typedef_tus + by_name = {tu.name: tu for tu in mutual_typedef_tus} - a_types_h.write_text( - dedent( - """\ - typedef struct A A; - """ - ) - ) - b_types_h.write_text( - dedent( - """\ - typedef struct B B; - """ - ) - ) - a_c.write_text( - dedent( - """\ - #include "a_types.h" - #include "b_types.h" - - struct A { - B *ref; - int val; - }; - - A *create_a(void) { - return (A *)0; - } - """ - ) - ) - b_c.write_text( - dedent( - """\ - #include "a_types.h" - #include "b_types.h" - - struct B { - A *ref; - int val; - }; - - B *create_b(void) { - return (B *)0; - } - """ - ) - ) - - compile_commands = _write_compile_commands(tmp_path, [a_c, b_c]) + compile_commands = _write_compile_commands(a_c.parent, [by_name[n] for n in order]) consolidated = consolidate_init( compile_commands, source_priority=[a_c.resolve(), b_c.resolve()] ) - # Both typedefs must appear before the struct definitions that reference them. - success, error = check_c(consolidated, flags=["-fsyntax-only", "-Wall"]) + text = str(consolidated) + assert "create_a" in text and "create_b" in text + assert "a_not_renamed_a" not in text and "b_not_renamed_b" not in text, consolidated + + success, error = compile_c(consolidated, flags=["-Wall", "-Werror"]) assert success, ( f"Consolidated code does not compile:\n{error}\n\nConsolidated output:\n{consolidated}" ) @@ -398,10 +458,10 @@ def test_macro_wrapped_declaration(tmp_path: Path): """\ #define LIB_EXPORT(type) extern type - typedef struct my_object my_object; + typedef struct not_renamed not_renamed; - LIB_EXPORT(void) my_free(my_object *obj); - LIB_EXPORT(int) my_get_value(my_object *obj); + LIB_EXPORT(void) my_free(not_renamed *obj); + LIB_EXPORT(int) my_get_value(not_renamed *obj); """ ) ) @@ -412,19 +472,19 @@ def test_macro_wrapped_declaration(tmp_path: Path): """\ #include "api.h" - struct my_object { + struct not_renamed { int value; int refcount; }; - void my_free(my_object *obj) + void my_free(not_renamed *obj) { if (obj && my_get_value(obj) < 0) { /* free */ } } - int my_get_value(my_object *obj) + int my_get_value(not_renamed *obj) { my_free(obj); return obj->value; @@ -433,190 +493,267 @@ def test_macro_wrapped_declaration(tmp_path: Path): ) ) + # Fully compile the TU to an object file (resolves api.h). + ok, out, err, _ = run_subprocess( + [ + "clang-21", + "-Wall", + "-Werror", + f"-I{tmp_path}", + "-c", + str(impl_c), + "-o", + str(tmp_path / "impl.o"), + ] + ) + assert ok, f"impl.c failed to compile:\n{out}{err}" + compile_commands = _write_compile_commands(tmp_path, [impl_c]) consolidated = consolidate_init(compile_commands, source_priority=[]) + text = str(consolidated) + assert "not_renamed" in text + assert "impl_not_renamed" not in text + # The consolidated output must not contain the unexpanded macro assert "LIB_EXPORT" not in str(consolidated), ( f"Consolidated output contains unexpanded macro 'LIB_EXPORT':\n{consolidated}" ) - success, error = check_c(consolidated, flags=["-fsyntax-only", "-Wall"]) + success, error = compile_c(consolidated, flags=["-Wall", "-Werror"]) assert success, ( f"Consolidated code does not compile:\n{error}\n\nConsolidated output:\n{consolidated}" ) -def test_typedef_after_struct_cross_tu_three_tus(tmp_path: Path): - """ - - types.h: typedef struct X X; (forward-declares struct X via typedef) - - TU1 (a.c): #include "types.h", defines struct X { X *self; int val; }; - The struct body uses the typedef name 'X' → creates cycle: - struct X → typedef X → struct X - - TU2 (b.c): #include "types.h" only, uses X* in a function signature - - TU3 (c.c): #include "types.h", defines struct Y { X *member; }; and a - function returning Y* +@pytest.fixture(scope="module") +def three_tu_typedef_tus(tmp_path_factory: pytest.TempPathFactory) -> list[Path]: + return _build_tus( + tmp_path_factory.mktemp("three_tu_typedef"), + { + "types.h": """\ + typedef struct not_renamed not_renamed; + """, + "a.c": """\ + #include "types.h" - After merge_symbols with asts=[TU2, TU3, TU1]: - - typedef X cursor retained from TU2 (first encounter, definition) - - struct X cursor from TU1 (only TU with full definition) + struct not_renamed { + not_renamed *self; + int val; + }; - With ast_order=[a.c, b.c, c.c]: - - struct X from a.c → rank 0 - - typedef X from b.c → rank 1 + not_renamed *create_x(int v) { + (void)v; + return (not_renamed *)0; + } + """, + "b.c": """\ + #include "types.h" - They share an SCC (mutual dependency via X *self in struct body), so - _merge_pure_type_declaration_sccs sorts them by symbol_lexical_key which - uses ast_order ranks. struct X (rank 0) sorts before typedef X (rank 1). + void consume_x(not_renamed *p) { + (void)p; + } + """, + "c.c": """\ + #include "types.h" - Result: consolidated output places struct X { X *self; ... } BEFORE - typedef struct X X; → 'X' is unknown at that point → compilation failure. - """ - types_h = tmp_path / "types.h" - a_c = tmp_path / "a.c" - b_c = tmp_path / "b.c" - c_c = tmp_path / "c.c" + struct Y { + not_renamed *member; + int id; + }; - types_h.write_text( - dedent( - """\ - typedef struct X X; - """ - ) + struct Y *alloc_y(void) { + return (struct Y *)0; + } + """, + }, ) - a_c.write_text( - dedent( - """\ - #include "types.h" - struct X { - X *self; - int val; - }; - X *create_x(int v) { - (void)v; - return (X *)0; - } - """ - ) - ) - b_c.write_text( - dedent( - """\ - #include "types.h" +@pytest.mark.parametrize("order", permutations(["a.c", "b.c", "c.c"]), ids="-".join) +def test_typedef_after_struct_cross_tu_three_tus( + three_tu_typedef_tus: list[Path], order: tuple[str, ...] +): + """ + - types.h: typedef struct not_renamed not_renamed; (forward-declares the struct) + - TU1 (a.c): #include "types.h", defines struct not_renamed { not_renamed *self; }; + The struct body uses the typedef name -> creates cycle: + struct not_renamed -> typedef not_renamed -> struct not_renamed + - TU2 (b.c): #include "types.h" only, uses not_renamed* in a function signature + - TU3 (c.c): #include "types.h", defines struct Y { not_renamed *member; }; and a + function returning Y* - void consume_x(X *p) { - (void)p; - } - """ - ) - ) - c_c.write_text( - dedent( - """\ - #include "types.h" + After merge_symbols with asts=[TU2, TU3, TU1]: + - typedef not_renamed cursor retained from TU2 (first encounter, definition) + - struct not_renamed cursor from TU1 (only TU with full definition) - struct Y { - X *member; - int id; - }; + With ast_order=[a.c, b.c, c.c]: + - struct not_renamed from a.c -> rank 0 + - typedef not_renamed from b.c -> rank 1 - struct Y *alloc_y(void) { - return (struct Y *)0; - } - """ - ) - ) + They share an SCC (mutual dependency via not_renamed *self in the struct body), + so _merge_pure_type_declaration_sccs sorts them by symbol_lexical_key which uses + ast_order ranks. struct not_renamed (rank 0) must not sort before the typedef. - # Parse TUs — process b first so merge_symbols retains typedef X cursor from b.c - # In init(), get_asts processes in compile_commands order, so list b first. - # ast_order is derived from source_priority: a.c first so struct X gets rank 0. - compile_commands = _write_compile_commands(tmp_path, [b_c, c_c, a_c]) + 'not_renamed' is a single shared entity, so it must not be renamed; the typedef + must still precede the struct definition that uses it as a bare type name. + """ + a_c, b_c, c_c = three_tu_typedef_tus + by_name = {tu.name: tu for tu in three_tu_typedef_tus} + + compile_commands = _write_compile_commands(a_c.parent, [by_name[n] for n in order]) consolidated = consolidate_init( compile_commands, source_priority=[a_c.resolve(), b_c.resolve(), c_c.resolve()] ) - # The typedef MUST appear before the struct definition that uses 'X' as a - # bare type name in its body. If the cross-TU lexical key comparison - # incorrectly places struct X before typedef X, this will fail. - success, error = check_c(consolidated, flags=["-fsyntax-only", "-Wall"]) + text = str(consolidated) + assert "create_x" in text + assert ( + "a_not_renamed" not in text + and "b_not_renamed" not in text + and "types_not_renamed" not in text + ), consolidated + + success, error = compile_c(consolidated, flags=["-Wall", "-Werror"]) assert success, ( f"Consolidated code does not compile:\n{error}\n\nConsolidated output:\n{consolidated}" ) -def test_static_function_and_static_variable_same_name_renamed(tmp_path: Path): - a_c = tmp_path / "a.c" - b_c = tmp_path / "b.c" +@pytest.fixture(scope="module") +def static_fn_and_var_tus(tmp_path_factory: pytest.TempPathFactory) -> list[Path]: + return _build_tus( + tmp_path_factory.mktemp("static_fn_and_var"), + { + "a.c": """\ + static int renamed(int x) { return x; } - a_c.write_text( - dedent( - """\ - static int some(int x) { return x; } + int use_a(void) { return renamed(42); } + """, + "b.c": """\ + static int renamed; - int use_a(void) { return some(42); } - """ - ) + int use_b(void) { return renamed; } + """, + }, ) - b_c.write_text( - dedent( - """\ - static int some; - int use_b(void) { return some; } - """ - ) - ) - compile_commands = _write_compile_commands(tmp_path, [a_c, b_c]) +@pytest.mark.parametrize("order", permutations(["a.c", "b.c"]), ids="-".join) +def test_static_function_and_static_variable_same_name_renamed( + static_fn_and_var_tus: list[Path], order: tuple[str, ...] +): + a_c, b_c = static_fn_and_var_tus + by_name = {tu.name: tu for tu in static_fn_and_var_tus} + + compile_commands = _write_compile_commands(a_c.parent, [by_name[n] for n in order]) consolidated = consolidate_init( compile_commands, source_priority=[a_c.resolve(), b_c.resolve()] ) - success, error = check_c(consolidated, flags=["-fsyntax-only", "-Wall"]) + text = str(consolidated) + assert "a_renamed" in text and "b_renamed" in text, consolidated + + success, error = compile_c(consolidated, flags=["-Wall", "-Werror"]) assert success, ( f"Consolidated code does not compile (missing rename for static name collision):\n" f"{error}\n\nConsolidated output:\n{consolidated}" ) -def test_static_variable_tentative_defs_same_name_renamed(tmp_path: Path): - a_c = tmp_path / "a.c" - b_c = tmp_path / "b.c" +@pytest.fixture(scope="module") +def static_tentative_def_tus(tmp_path_factory: pytest.TempPathFactory) -> list[Path]: + return _build_tus( + tmp_path_factory.mktemp("static_tentative_def"), + { + "a.c": """\ + static int renamed; - a_c.write_text( - dedent( - """\ - static int count; + int get_a(void) { renamed += 2; return renamed; } + """, + "b.c": """\ + static char renamed; - int get_a(void) { count += 2; return count; } - """ - ) + int get_b(void) { return (int)renamed + 1; } + """, + }, ) - b_c.write_text( - dedent( - """\ - static char count; - int get_b(void) { return (int)count + 1; } - """ - ) - ) - compile_commands = _write_compile_commands(tmp_path, [a_c, b_c]) +@pytest.mark.parametrize("order", permutations(["a.c", "b.c"]), ids="-".join) +def test_static_variable_tentative_defs_same_name_renamed( + static_tentative_def_tus: list[Path], order: tuple[str, ...] +): + a_c, b_c = static_tentative_def_tus + by_name = {tu.name: tu for tu in static_tentative_def_tus} + + compile_commands = _write_compile_commands(a_c.parent, [by_name[n] for n in order]) consolidated = consolidate_init( compile_commands, source_priority=[a_c.resolve(), b_c.resolve()] ) - success, error = check_c(consolidated, flags=["-fsyntax-only", "-Wall"]) + text = str(consolidated) + assert "a_renamed" in text and "b_renamed" in text, consolidated + + success, error = compile_c(consolidated, flags=["-Wall", "-Werror"]) assert success, ( f"Consolidated code does not compile (missing rename for static variable collision):\n" f"{error}\n\nConsolidated output:\n{consolidated}" ) -def test_isystem_inline_function_dependency_not_lost(tmp_path: Path): +@pytest.fixture(scope="module") +def isystem_tus(tmp_path_factory: pytest.TempPathFactory) -> list[Path]: + base = tmp_path_factory.mktemp("isystem") + return _build_tus( + base, + { + "util/alloc.h": """\ + #include + static inline void *not_renamed(size_t len) { + return malloc(len); + } + """, + "ext/bridge.h": """\ + #include "alloc.h" + #define ext_malloc(x) not_renamed(x) + """, + "ext/a.c": """\ + #include "bridge.h" + + typedef struct { int val; } item_t; + + static item_t *make_item(int val) { + item_t *p; + if (!(p = (item_t *)ext_malloc(sizeof(item_t)))) + return (void *)0; + p->val = val; + return p; + } + + int do_work(int x) { + item_t *item = make_item(x); + if (item) return item->val; + return -1; + } + """, + "util/b.c": """\ + #include "alloc.h" + + void *my_calloc(size_t n, size_t sz) { + void *p = not_renamed(n * sz); + return p; + } + """, + }, + cflags=[f"-I{base / 'util'}", f"-I{base / 'ext'}"], + ) + + +@pytest.mark.parametrize("order", permutations(["a.c", "b.c"]), ids="-".join) +def test_isystem_inline_function_dependency_not_lost( + isystem_tus: list[Path], order: tuple[str, ...] +): """ When a header is included via -isystem in one TU but via -I in another, clang generates different USRs for the same static inline function @@ -627,156 +764,94 @@ def test_isystem_inline_function_dependency_not_lost(tmp_path: Path): This manifests as the inline function definition being placed AFTER its caller in the consolidated output, causing: - error: call to undeclared function 'my_alloc'; ISO C99 and later do not + error: call to undeclared function 'not_renamed'; ISO C99 and later do not support implicit function declarations """ - # alloc.h in util/ with a static inline function - util_dir = tmp_path / "util" - util_dir.mkdir() - alloc_h = util_dir / "alloc.h" - alloc_h.write_text( - dedent( - """\ - #include - static inline void *my_alloc(size_t len) { - return malloc(len); - } - """ - ) - ) - - # bridge.h wraps my_alloc in a macro - ext_dir = tmp_path / "ext" - ext_dir.mkdir() - bridge_h = ext_dir / "bridge.h" - bridge_h.write_text( - dedent( - """\ - #include "alloc.h" - #define ext_malloc(x) my_alloc(x) - """ - ) - ) - - # caller.c in ext/ - calls my_alloc via ext_malloc macro - caller_c = ext_dir / "caller.c" - caller_c.write_text( - dedent( - """\ - #include "bridge.h" - - typedef struct { int val; } item_t; - - static item_t *make_item(int val) { - item_t *p; - if (!(p = (item_t *)ext_malloc(sizeof(item_t)))) - return (void *)0; - p->val = val; - return p; - } - - int do_work(int x) { - item_t *item = make_item(x); - if (item) return item->val; - return -1; - } - """ - ) - ) - - # user.c in util/ - calls my_alloc directly - user_c = util_dir / "user.c" - user_c.write_text( - dedent( - """\ - #include "alloc.h" - - void *my_calloc(size_t n, size_t sz) { - void *p = my_alloc(n * sz); - return p; - } - """ - ) - ) + a_c, b_c = isystem_tus + ext_dir, util_dir = a_c.parent, b_c.parent # Keep mixed include modes across TUs to exercise the USR normalization path. - compile_commands = tmp_path / "compile_commands.json" - compile_commands.write_text( - json.dumps( - [ - { - "directory": str(tmp_path), - "file": str(caller_c), - "command": f"cc -isystem {util_dir} -I{ext_dir} -c {caller_c}", - }, - { - "directory": str(tmp_path), - "file": str(user_c), - "command": f"cc -I{util_dir} -c {user_c}", - }, - ] - ) - ) + entries = { + "a.c": { + "directory": str(ext_dir.parent), + "file": str(a_c), + "command": f"cc -isystem {util_dir} -I{ext_dir} -c {a_c}", + }, + "b.c": { + "directory": str(ext_dir.parent), + "file": str(b_c), + "command": f"cc -I{util_dir} -c {b_c}", + }, + } + compile_commands = ext_dir.parent / "compile_commands.json" + compile_commands.write_text(json.dumps([entries[n] for n in order])) consolidated = consolidate_init( - compile_commands, source_priority=[caller_c.resolve(), user_c.resolve()] + compile_commands, source_priority=[a_c.resolve(), b_c.resolve()] ) - success, error = check_c(consolidated, flags=["-fsyntax-only", "-Wall"]) + + text = str(consolidated) + assert "not_renamed" in text + assert "alloc_not_renamed" not in text and "b_not_renamed" not in text, consolidated + success, error = compile_c(consolidated, flags=["-Wall", "-Werror"]) assert success, ( f"Consolidated code does not compile (isystem USR mismatch lost dependency):\n" f"{error}\n\nConsolidated output:\n{consolidated}" ) -def test_static_inline_in_scc_emitted_before_caller(tmp_path: Path): - # header.h: static inline helper reads extern vtable - header_h = tmp_path / "header.h" - header_h.write_text( - dedent( - """\ - struct vtable_t { int (*fn)(int); }; - extern struct vtable_t vtable; - static inline int helper(int x) { - return vtable.fn(x); - } - """ - ) +@pytest.fixture(scope="module") +def scc_static_inline_tus(tmp_path_factory: pytest.TempPathFactory) -> list[Path]: + return _build_tus( + tmp_path_factory.mktemp("scc_static_inline"), + { + "header.h": """\ + struct vtable_t { int (*fn)(int); }; + extern struct vtable_t vtable; + static inline int not_renamed(int x) { + return vtable.fn(x); + } + """, + # a.c (rank 0): defines compute() which calls not_renamed() + "a.c": """\ + #include "header.h" + int compute(int x) { + return not_renamed(x) + 1; + } + """, + # b.c (rank 1): includes header.h, defines vtable referencing compute + "b.c": """\ + #include "header.h" + int compute(int x); + struct vtable_t vtable = { .fn = compute }; + """, + }, ) - # caller.c (rank 0): defines compute() which calls helper() - caller_c = tmp_path / "caller.c" - caller_c.write_text( - dedent( - """\ - #include "header.h" - int compute(int x) { - return helper(x) + 1; - } - """ - ) - ) - # state.c (rank 1): includes header.h, defines vtable referencing compute - state_c = tmp_path / "state.c" - state_c.write_text( - dedent( - """\ - #include "header.h" - int compute(int x); - struct vtable_t vtable = { .fn = compute }; - """ - ) - ) +@pytest.mark.parametrize("order", permutations(["a.c", "b.c"]), ids="-".join) +def test_static_inline_in_scc_emitted_before_caller( + scc_static_inline_tus: list[Path], order: tuple[str, ...] +): + a_c, b_c = scc_static_inline_tus + base = a_c.parent + by_name = {tu.name: tu for tu in scc_static_inline_tus} compile_commands = _write_compile_commands( - tmp_path, [state_c, caller_c], extra_flags=f"-I{tmp_path}" + base, [by_name[n] for n in order], extra_flags=f"-I{base}" ) consolidated = consolidate_init( - compile_commands, source_priority=[caller_c.resolve(), state_c.resolve()] + compile_commands, source_priority=[a_c.resolve(), b_c.resolve()] ) - success, error = check_c(consolidated, flags=["-fsyntax-only", "-Wall"]) + + text = str(consolidated) + assert "not_renamed" in text and "compute" in text + assert "a_compute" not in text and "b_compute" not in text, consolidated + assert "header_not_renamed" not in text, consolidated + success, error = compile_c(consolidated, flags=["-Wall", "-Werror"]) assert success, ( - f"Consolidated code fails (static inline in SCC emitted after caller due to TU rank):\n" + f"Consolidated code fails " + f"(static inline in SCC emitted after caller due to TU rank):\n" f"{error}\n\nConsolidated output:\n{consolidated}" ) @@ -788,7 +863,7 @@ def test_system_macro_double_expansion(tmp_path: Path): """\ #include - void setup_signal(void) { + void not_renamed(void) { struct sigaction ign_handler; ign_handler.sa_handler = SIG_IGN; } @@ -796,11 +871,19 @@ def test_system_macro_double_expansion(tmp_path: Path): ) ) + # Fully compile the TU to an object file (resolves ). + ok, out, err, _ = run_subprocess( + ["clang-21", "-Wall", "-Werror", "-c", str(main_c), "-o", str(tmp_path / "main.o")] + ) + assert ok, f"main.c failed to compile:\n{out}{err}" + compile_commands = _write_compile_commands(tmp_path, [main_c]) consolidated = consolidate_init(compile_commands, source_priority=[]) - # The consolidated code must compile without double macro expansion - success, error = check_c(consolidated, flags=["-fsyntax-only"]) + text = str(consolidated) + assert "not_renamed" in text + assert "main_not_renamed" not in text + success, error = compile_c(consolidated, flags=["-Wall", "-Werror"]) assert success, ( f"Consolidated code does not compile (system macro double expansion):\n{error}\n\n" f"Consolidated output:\n{consolidated}" @@ -818,7 +901,7 @@ def test_cc_defines_preserved_in_consolidation(tmp_path: Path): #include #include - int count_env(void) { + int not_renamed(void) { int count = 0; char **kv; for (kv = environ; *kv; kv++) @@ -850,6 +933,22 @@ def test_cc_defines_preserved_in_consolidation(tmp_path: Path): ) ) + # Fully compile the TU to an object file (needs the same -D feature macros). + ok, out, err, _ = run_subprocess( + [ + "clang-21", + "-Wall", + "-Werror", + "-D_GNU_SOURCE", + "-DPCRE2_CODE_UNIT_WIDTH=8", + "-c", + str(main_c), + "-o", + str(tmp_path / "main.o"), + ] + ) + assert ok, f"main.c failed to compile:\n{out}{err}" + compile_commands = _write_compile_commands( tmp_path, [main_c], extra_flags="-D_GNU_SOURCE -DPCRE2_CODE_UNIT_WIDTH=8" ) @@ -860,7 +959,11 @@ def test_cc_defines_preserved_in_consolidation(tmp_path: Path): f"Consolidated output is missing '{sym}' usage:\n{consolidated}" ) - success, error = check_c(consolidated, flags=["-fsyntax-only"]) + text = str(consolidated) + assert "not_renamed" in text + assert "main_not_renamed" not in text and "main_sort_with_context" not in text + + success, error = compile_c(consolidated, flags=["-Wall", "-Werror"]) assert success, ( f"Consolidated code does not compile without -D_GNU_SOURCE and -DPCRE2_CODE_UNIT_WIDTH=8 " f"(macros lost during consolidation):\n" @@ -883,7 +986,7 @@ def test_posix_c_source_preserved_in_consolidation(tmp_path: Path): return ts.tv_sec * 1000000000L + ts.tv_nsec; } - char *duplicate(const char *s) { + char *not_renamed(const char *s) { return strdup(s); } @@ -894,6 +997,22 @@ def test_posix_c_source_preserved_in_consolidation(tmp_path: Path): ) ) + # Fully compile the TU to an object file (needs the same -std/-D feature macros). + ok, out, err, _ = run_subprocess( + [ + "clang-21", + "-Wall", + "-Werror", + "-std=c11", + "-D_POSIX_C_SOURCE=200809L", + "-c", + str(main_c), + "-o", + str(tmp_path / "main.o"), + ] + ) + assert ok, f"main.c failed to compile:\n{out}{err}" + compile_commands = _write_compile_commands( tmp_path, [main_c], extra_flags="-std=c11 -D_POSIX_C_SOURCE=200809L" ) @@ -905,7 +1024,11 @@ def test_posix_c_source_preserved_in_consolidation(tmp_path: Path): f"Consolidated output is missing '{sym}' usage:\n{consolidated}" ) - success, error = check_c(consolidated, flags=["-std=c11", "-fsyntax-only"]) + text = str(consolidated) + assert "not_renamed" in text and "next_token" in text + assert "main_not_renamed" not in text and "main_next_token" not in text + + success, error = compile_c(consolidated, flags=["-std=c11", "-Wall", "-Werror"]) assert success, ( f"Consolidated code does not compile with -std=c11 without _POSIX_C_SOURCE " f"(clock_gettime/strdup/strtok_r undeclared — feature-test macro lost):\n" @@ -926,7 +1049,7 @@ def test_system_macro_undefs_preserve_benign_macros(tmp_path: Path): #include #include - int run(void) { + int not_renamed(void) { char *p = NULL; if (p == NULL) return EXIT_FAILURE; @@ -957,10 +1080,20 @@ def test_system_macro_undefs_preserve_benign_macros(tmp_path: Path): ) ) + # Fully compile the TU to an object file (resolves the C stdlib headers). + ok, out, err, _ = run_subprocess( + ["clang-21", "-Wall", "-Werror", "-c", str(main_c), "-o", str(tmp_path / "main.o")] + ) + assert ok, f"main.c failed to compile:\n{out}{err}" + compile_commands = _write_compile_commands(tmp_path, [main_c]) consolidated = consolidate_init(compile_commands, source_priority=[]) - success, error = check_c(consolidated, flags=["-fsyntax-only"]) + text = str(consolidated) + assert "not_renamed" in text and "compute" in text and "check" in text + assert "main_not_renamed" not in text and "main_compute" not in text + + success, error = compile_c(consolidated, flags=["-Wall", "-Werror"]) assert success, ( f"Consolidated code does not compile (benign macros broken):\n{error}\n\n" f"Consolidated output:\n{consolidated}" @@ -983,7 +1116,7 @@ def test_source_level_defines_required_by_system_headers_preserved(tmp_path: Pat #include #include - int use_pcre(const char *pat) { + int not_renamed(const char *pat) { (void)pat; return 0; } @@ -997,74 +1130,765 @@ def test_source_level_defines_required_by_system_headers_preserved(tmp_path: Pat ) ) + # Fully compile the TU to an object file (in-source #defines gate ). + ok, out, err, _ = run_subprocess( + ["clang-21", "-Wall", "-Werror", "-c", str(main_c), "-o", str(tmp_path / "main.o")] + ) + assert ok, f"main.c failed to compile:\n{out}{err}" + compile_commands = _write_compile_commands(tmp_path, [main_c]) consolidated = consolidate_init(compile_commands, source_priority=[]) - success, error = check_c(consolidated, flags=["-fsyntax-only", "-Werror"]) + text = str(consolidated) + assert "not_renamed" in text and "get_err" in text + assert "main_not_renamed" not in text and "main_get_err" not in text + + success, error = compile_c(consolidated, flags=["-Wall", "-Werror"]) assert success, ( f"Consolidated code does not compile (source-level defines dropped):\n{error}\n\n" f"Consolidated output:\n{consolidated}" ) -def test_header_defined_gnu_source_with_default_source_flag(tmp_path: Path): +@pytest.fixture(scope="module") +def shadowing_local_tus(tmp_path_factory: pytest.TempPathFactory) -> list[Path]: + return _build_tus( + tmp_path_factory.mktemp("shadowing_local"), + { + "a.c": """\ + static int renamed(int x) { return x * 2; } + + int use_a(int val) { + int a_renamed = val + 1; + return renamed(a_renamed); + } + """, + "b.c": """\ + static int renamed(int x) { return x + 1; } + + int use_b(void) { return renamed(42); } + """, + }, + ) + + +@pytest.mark.parametrize("order", permutations(["a.c", "b.c"]), ids="-".join) +def test_renamed_symbol_shadowed_by_local_variable( + shadowing_local_tus: list[Path], order: tuple[str, ...] +): """ - Compile commands have -D_DEFAULT_SOURCE, but _GNU_SOURCE is defined inside - a project header (config.h/first.h). The consolidator preserves - _DEFAULT_SOURCE (from -D flags) but loses _GNU_SOURCE (from the header). - Code using the GNU strerror_r (returns char*) then fails with: - error: incompatible integer to pointer conversion + rename_conflicting_symbols_ builds candidate names from the TU stem plus the + original spelling (e.g. "a_renamed" for "renamed" in "a.c"), but only checks + used_spellings against top-level symbol names. Local variable names inside + function bodies are never added to used_spellings, so the generated name can + collide with a local variable in the same TU. + + Scenario: + a.c – static renamed(int x) + use_a() which declares int a_renamed = ... + and calls renamed(a_renamed). + b.c – conflicting static renamed(int x). + + After renaming, a.c's "renamed" → "a_renamed". The call site becomes: + return a_renamed(a_renamed); + where the first "a_renamed" now resolves to the local int, not the function, + producing: "called object type 'int' is not a function or function pointer". + """ + a_c, b_c = shadowing_local_tus + by_name = {tu.name: tu for tu in shadowing_local_tus} - Uses two TUs to exercise the union: both include config.h which defines - _GNU_SOURCE, so the union should emit it exactly once. + compile_commands = _write_compile_commands(a_c.parent, [by_name[n] for n in order]) + consolidated = consolidate_init( + compile_commands, source_priority=[a_c.resolve(), b_c.resolve()] + ) + + text = str(consolidated) + assert "b_renamed" in text, consolidated + assert "_a_renamed" in text, consolidated + + success, error = compile_c(consolidated, flags=["-Wall", "-Werror"]) + assert success, ( + f"Consolidated code does not compile (renamed symbol shadowed by local variable):\n" + f"{error}\n\nConsolidated output:\n{consolidated}" + ) + + +@pytest.fixture(scope="module") +def shared_header_rename_tus(tmp_path_factory: pytest.TempPathFactory) -> list[Path]: + return _build_tus( + tmp_path_factory.mktemp("shared_header_rename"), + { + "header.h": """\ + typedef struct renamed { + int a; + } renamed; + """, + "a.c": """\ + #include "header.h" + + static void local_push(void) { + renamed *spec; + (void)spec; + } + + int call_local_push(void) { local_push(); return 0; } + """, + "b.c": """\ + #include "header.h" + + static void push_spec_rref_cmp(void) { + const renamed *push_spec_a; + (void)push_spec_a; + } + + int call_rref_cmp(void) { push_spec_rref_cmp(); return 0; } + """, + "c.c": """\ + static void renamed(void) { return; } + + int use_push(void) { renamed(); return 0; } + """, + }, + ) + + +@pytest.mark.parametrize("order", permutations(["a.c", "b.c", "c.c"]), ids="-".join) +def test_rename_conflict_in_shared_header_across_tus( + shared_header_rename_tus: list[Path], order: tuple[str, ...] +): """ - config_h = tmp_path / "config.h" - config_h.write_text( - dedent( - """\ - #define _GNU_SOURCE - """ - ) + When a typedef in a shared header conflicts with a same-named symbol in + another TU, _apply_renames renames the typedef and updates header.h in + modified_sources while processing the first TU that includes it. Parsing a + subsequent TU that includes the same header then fails: the header no longer + defines the original spelling, but the TU's own source still references it, + causing a TranslationUnitLoadError. + + Scenario: + header.h – typedef struct renamed { int a; } renamed; + a.c – #include "header.h", uses renamed + b.c – #include "header.h", uses renamed (also has a local named push_spec_a) + c.c – static void renamed(void) { ... } ← conflicts with the typedef + + c.c's renamed function forces the typedef to be renamed. The rename for + a.c modifies header.h in modified_sources. When b.c is subsequently parsed + with that modified header, b.c's references to 'renamed' become + unresolved and the parse fails. + """ + a_c, b_c, c_c = shared_header_rename_tus + base = a_c.parent + by_name = {tu.name: tu for tu in shared_header_rename_tus} + + compile_commands = _write_compile_commands( + base, [by_name[n] for n in order], extra_flags=f"-I{base}" + ) + consolidated = consolidate_init( + compile_commands, + source_priority=[a_c.resolve(), b_c.resolve(), c_c.resolve()], ) - main_c = tmp_path / "main.c" - main_c.write_text( - dedent( - """\ - #include "config.h" - #include + text = str(consolidated) + assert "header_renamed" in text and "c_renamed" in text, consolidated - const char *get_err(int errnum) { - static char buf[256]; - const char *errstr = strerror_r(errnum, buf, sizeof(buf)); - return errstr; - } - """ - ) + success, error = compile_c(consolidated, flags=["-Wall", "-Werror"]) + assert success, ( + f"Consolidated code does not compile " + f"(rename of shared header breaks subsequent TU parse):\n" + f"{error}\n\nConsolidated output:\n{consolidated}" ) - util_c = tmp_path / "util.c" - util_c.write_text( - dedent( - """\ - #include "config.h" - #include - const char *get_err2(int errnum) { - static char buf[128]; - const char *errstr = strerror_r(errnum, buf, sizeof(buf)); - return errstr; - } - """ - ) +@pytest.fixture(scope="module") +def forward_typedef_tus(tmp_path_factory: pytest.TempPathFactory) -> list[Path]: + return _build_tus( + tmp_path_factory.mktemp("forward_typedef"), + { + "common.h": """\ + typedef struct not_renamed not_renamed; + + struct Bar { + not_renamed *hideset; + }; + """, + "a.c": """\ + #include "common.h" + + int use_a(not_renamed *f) { + (void)f; + return 0; + } + """, + "b.c": """\ + #include "common.h" + + typedef struct not_renamed not_renamed; + struct not_renamed { + not_renamed *next; + char *name; + }; + + int use_b(void) { + not_renamed *h = 0; + (void)h; + return 0; + } + """, + }, + ) + + +@pytest.mark.parametrize("order", permutations(["a.c", "b.c"]), ids="-".join) +def test_forward_typedef_completed_in_one_tu_rename_conflict( + forward_typedef_tus: list[Path], order: tuple[str, ...] +): + """ + A forward typedef declared in a shared header but completed (full struct + definition) in exactly one TU triggers conflicting rename edits on the + header token. + + Scenario: + common.h – typedef struct not_renamed not_renamed; (forward decl only) + struct Bar { not_renamed *hideset; }; + a.c – #include "common.h", global fn referencing not_renamed + b.c – #include "common.h", re-declares `typedef struct not_renamed not_renamed;` + AND completes `struct not_renamed { ... };`, global fn referencing not_renamed + + Because a same-spelled definition of the `not_renamed` typedef exists in more than + one presumed path, consolidation renames it. The new spelling is derived + from each representative symbol's presumed_path in _get_conflicting_symbols: + in a.c the representative resolves to common.h -> `common_not_renamed`, while in b.c + (where the completing struct lives) it resolves to b.c -> `b_not_renamed`. Both map + the same USR `c:common.h@T@Foo`, so _apply_renames emits two different + replacements for the same header token and raises + `ValueError: Conflicting rename edits ...`. + """ + a_c, b_c = forward_typedef_tus + base = a_c.parent + by_name = {tu.name: tu for tu in forward_typedef_tus} + + compile_commands = _write_compile_commands( + base, [by_name[n] for n in order], extra_flags=f"-I{base}" + ) + consolidated = consolidate_init( + compile_commands, + source_priority=[a_c.resolve(), b_c.resolve()], + ) + + text = str(consolidated) + assert "not_renamed" in text + assert "common_not_renamed" not in text and "b_not_renamed" not in text, consolidated + + success, error = compile_c(consolidated, flags=["-Wall", "-Werror"]) + assert success, ( + f"Consolidated code does not compile " + f"(forward typedef completed in one TU yields conflicting header rename):\n" + f"{error}\n\nConsolidated output:\n{consolidated}" + ) + + +@pytest.fixture(scope="module") +def odr_struct_tus(tmp_path_factory: pytest.TempPathFactory) -> list[Path]: + # Each TU compiles fine in isolation — the ODR violation only surfaces when + # the two definitions are combined into a single translation unit. + return _build_tus( + tmp_path_factory.mktemp("odr_struct"), + { + "common.h": """\ + struct token; + """, + "a.c": """\ + #include "common.h" + + struct token { int type; int line; }; + + int token_line(struct token *t) { + return t->line; + } + """, + "b.c": """\ + #include "common.h" + + struct token { int type; char *text; }; + + const char *token_text(struct token *t) { + return t->text; + } + """, + }, + ) + + +@pytest.mark.xfail(reason="UB due to conflicting struct definitions in different TUs") +@pytest.mark.parametrize("order", permutations(["a.c", "b.c"]), ids="-".join) +def test_struct_forward_decl_completed_differently_in_two_tus( + odr_struct_tus: list[Path], order: tuple[str, ...] +): + """ + A struct forward-declared in a shared header but completed with *different* + field layouts in two TUs is an ODR violation. Both completions may share + the same USR (from the canonical forward-declaration location in common.h), + so the USR-identity check in _get_conflicting_symbols must not skip renaming + here — the code bodies differ, so this is a genuine conflict. + + Scenario: + common.h – struct token; (forward declaration only) + a.c – #include "common.h", completes struct token { int type; int line; }; + and a function that only uses it locally + b.c – #include "common.h", completes struct token { int type; char *text; }; + and a function that only uses it locally + + Expected: the two incompatible definitions are renamed (a_token / b_token) + and the consolidated output compiles without errors. + """ + a_c, b_c = odr_struct_tus + base = a_c.parent + by_name = {tu.name: tu for tu in odr_struct_tus} + + compile_commands = _write_compile_commands( + base, [by_name[n] for n in order], extra_flags=f"-I{base}" + ) + consolidated = consolidate_init( + compile_commands, + source_priority=[a_c.resolve(), b_c.resolve()], + ) + + text = str(consolidated) + # The two incompatible struct definitions must be renamed so they can coexist, + # and all usages within each TU must be updated consistently. + assert "struct a_token {\n int type;\n int line;\n};\n" in text + assert "int token_line(struct a_token *t) {\n return t->line;\n}\n" in text + assert "struct b_token {\n int type;\n char *text;\n};\n" in text + assert "const char *token_text(struct b_token *t) {\n return t->text;\n}\n" in text + + success, error = compile_c(consolidated, flags=["-Wall", "-Werror"]) + assert success, ( + f"Consolidated code does not compile:\n{error}\n\nConsolidated output:\n{consolidated}" ) + +@pytest.fixture(scope="module") +def odr_typedef_struct_tus(tmp_path_factory: pytest.TempPathFactory) -> list[Path]: + # Each TU compiles fine in isolation — the ODR violation only surfaces when + # the two definitions are combined into a single translation unit. + return _build_tus( + tmp_path_factory.mktemp("odr_typedef_struct"), + { + "common.h": """\ + typedef struct token token; + """, + "a.c": """\ + #include "common.h" + + struct token { int type; int line; }; + + int token_line(token *t) { + return t->line; + } + """, + "b.c": """\ + #include "common.h" + + struct token { int type; char *text; }; + + const char *token_text(token *t) { + return t->text; + } + """, + }, + ) + + +@pytest.mark.xfail( + reason="Shared typedef bound to a struct with conflicting completions cannot " + "be renamed to two spellings, leaving 'token' as an incomplete type." +) +@pytest.mark.parametrize("order", permutations(["a.c", "b.c"]), ids="-".join) +def test_typedef_struct_forward_decl_completed_differently_in_two_tus( + odr_typedef_struct_tus: list[Path], order: tuple[str, ...] +): + """ + Same ODR-violation scenario as + test_struct_forward_decl_completed_differently_in_two_tus, but the struct is + reached through a typedef declared in the shared header: + + common.h – typedef struct token token; (forward typedef) + a.c – #include "common.h", completes struct token { int type; int line; }; + and a function that refers to the type as the bare typedef `token` + b.c – #include "common.h", completes struct token { int type; char *text; }; + and a function that refers to the type as the bare typedef `token` + + Renaming the two incompatible struct completions apart (a_token / b_token) + would require the shared `typedef struct token token;` in common.h to become + two different typedefs as well. Because that single token span can only carry + one spelling, the shared edit is dropped, leaving `token` pointing at an + (now undefined) `struct token` — so dereferencing `t->line` no longer compiles. + """ + a_c, b_c = odr_typedef_struct_tus + base = a_c.parent + by_name = {tu.name: tu for tu in odr_typedef_struct_tus} + compile_commands = _write_compile_commands( - tmp_path, [main_c, util_c], extra_flags=f"-D_DEFAULT_SOURCE -I{tmp_path}" + base, [by_name[n] for n in order], extra_flags=f"-I{base}" + ) + consolidated = consolidate_init( + compile_commands, + source_priority=[a_c.resolve(), b_c.resolve()], + ) + + text = str(consolidated) + # Desired behavior: the typedef is split alongside the struct so each TU keeps + # a complete type under its own spelling. + assert "struct a_token {\n int type;\n int line;\n};\n" in text + assert "int token_line(a_token *t) {\n return t->line;\n}\n" in text + assert "struct b_token {\n int type;\n char *text;\n};\n" in text + assert "const char *token_text(b_token *t) {\n return t->text;\n}\n" in text + + success, error = compile_c(consolidated, flags=["-Wall", "-Werror"]) + assert success, ( + f"Consolidated code does not compile:\n{error}\n\nConsolidated output:\n{consolidated}" + ) + + +@pytest.fixture(scope="module") +def genuine_conflict_tus(tmp_path_factory: pytest.TempPathFactory) -> list[Path]: + return _build_tus( + tmp_path_factory.mktemp("genuine_conflict"), + { + "common.h": """\ + typedef struct renamed renamed; + + struct holder { + renamed *p; + }; + """, + "a.c": """\ + #include "common.h" + + int use_a(renamed *f) { + (void)f; + return 0; + } + """, + "b.c": """\ + #include "common.h" + + typedef struct renamed renamed; + struct renamed { + renamed *next; + int x; + }; + + int use_b(void) { + renamed *h = 0; + (void)h; + return 0; + } + """, + "c.c": """\ + static int renamed(void) { + return 0; + } + + int use_c(void) { return renamed(); } + """, + }, + ) + + +@pytest.mark.xfail(reason="Known limitation") +@pytest.mark.parametrize("order", permutations(["a.c", "b.c", "c.c"]), ids="-".join) +def test_shared_header_entity_inside_genuine_conflict_gets_single_spelling( + genuine_conflict_tus: list[Path], order: tuple[str, ...] +): + """ + A shared-header entity whose presumed path diverges across TUs must receive + ONE canonical new spelling even when it is caught up in a *genuine* naming + conflict with an unrelated same-spelled symbol. + + Scenario: + common.h – typedef struct renamed renamed; struct holder { renamed *p; }; + a.c – #include "common.h", uses renamed (USR-A, presumed common.h) + b.c – #include "common.h", REDECLARES+completes renamed, uses renamed + (USR-A, presumed b.c) + c.c – static int renamed(void) { ... } (USR-B, a genuine clash) + + c.c's `renamed` supplies the second distinct USR so the group is a real conflict; + the shared typedef (USR-A) must still be renamed consistently for both a.c + and b.c so _apply_renames does not raise on the common.h token. + + The spelling must also come from the header that declares the entity, not from + whichever TU leads compile_commands.json: a first-seen-wins rule would yield + `b_renamed` for every order in which b.c precedes a.c. + """ + base = genuine_conflict_tus[0].parent + by_name = {tu.name: tu for tu in genuine_conflict_tus} + ordered = [by_name[n] for n in order] + + compile_commands = _write_compile_commands(base, ordered, extra_flags=f"-I{base}") + consolidated = consolidate_init( + compile_commands, source_priority=[tu.resolve() for tu in ordered] + ) + + text = str(consolidated) + assert "common_renamed" in text and "c_renamed" in text, consolidated + assert "a_renamed" not in text and "b_renamed" not in text, consolidated + + success, error = compile_c(consolidated, flags=["-Wall", "-Werror"]) + assert success, ( + f"Consolidated code does not compile " + f"(shared-header entity in a genuine conflict got inconsistent renames):\n" + f"{error}\n\nConsolidated output:\n{consolidated}" + ) + + +@pytest.fixture(scope="module") +def cv_qualified_alias_tus(tmp_path_factory: pytest.TempPathFactory) -> list[Path]: + # Each TU compiles fine in isolation — the ODR violation only surfaces when + # the two definitions are combined into a single translation unit. + return _build_tus( + tmp_path_factory.mktemp("cv_qualified_alias"), + { + "common.h": """\ + typedef const struct token const_token; + typedef volatile struct token volatile_token; + """, + "a.c": """\ + #include "common.h" + + struct token { int type; int line; }; + + int token_line(const_token *t) { + return t->line; + } + + int token_tick(volatile_token *t) { + return t->type; + } + """, + "b.c": """\ + #include "common.h" + + struct token { int type; char *text; }; + + const char *token_text(const_token *t) { + return t->text; + } + + int token_kind(volatile_token *t) { + return t->type; + } + """, + }, + ) + + +@pytest.mark.xfail( + reason="A cv-qualified typedef is a type of its own, so it cannot adopt the split " + "tag's spelling and is rejected rather than silently losing its qualifier.", +) +@pytest.mark.parametrize("order", permutations(["a.c", "b.c"]), ids="-".join) +def test_cv_qualified_typedef_completed_differently_in_two_tus( + cv_qualified_alias_tus: list[Path], order: tuple[str, ...] +): + """ + Same ODR-violation scenario as + test_typedef_struct_forward_decl_completed_differently_in_two_tus, but each typedef + applies a cv-qualifier on top of the tag: + + common.h – typedef const struct token const_token; + typedef volatile struct token volatile_token; + a.c – #include "common.h", completes struct token { int type; int line; }; + b.c – #include "common.h", completes struct token { int type; char *text; }; + + Each typedef still names the tag *directly*, so this isolates qualifiers from any + question of typedef chains. + + A qualifier belongs to the alias, not to the tag, so splitting the tag apart must + carry it across. Note that a lost qualifier would still compile — dropping `const` + only widens the type — so the qualifiers are asserted on the emitted text rather + than left for the compiler to catch. + """ + a_c, b_c = cv_qualified_alias_tus + base = a_c.parent + by_name = {tu.name: tu for tu in cv_qualified_alias_tus} + + compile_commands = _write_compile_commands( + base, [by_name[n] for n in order], extra_flags=f"-I{base}" + ) + consolidated = consolidate_init( + compile_commands, + source_priority=[a_c.resolve(), b_c.resolve()], + ) + + text = str(consolidated) + # Desired behavior: each typedef is split alongside the struct, keeping its qualifier + # and a spelling of its own. + assert "struct a_token {\n int type;\n int line;\n};\n" in text + assert "struct b_token {\n int type;\n char *text;\n};\n" in text + assert "typedef const struct a_token a_const_token;" in text + assert "typedef volatile struct a_token a_volatile_token;" in text + assert "typedef const struct b_token b_const_token;" in text + assert "typedef volatile struct b_token b_volatile_token;" in text + + # Each use keeps the alias it was written with, qualifier intact. + assert "int token_line(a_const_token *t) {\n return t->line;\n}\n" in text + assert "int token_tick(a_volatile_token *t) {\n return t->type;\n}\n" in text + assert "const char *token_text(b_const_token *t) {\n return t->text;\n}\n" in text + assert "int token_kind(b_volatile_token *t) {\n return t->type;\n}\n" in text + + success, error = compile_c(consolidated, flags=["-Wall", "-Werror"]) + assert success, ( + f"Consolidated code does not compile:\n{error}\n\nConsolidated output:\n{consolidated}" + ) + + +@pytest.fixture(scope="module") +def chained_alias_tus(tmp_path_factory: pytest.TempPathFactory) -> list[Path]: + # Each TU compiles fine in isolation — the ODR violation only surfaces when + # the two definitions are combined into a single translation unit. + return _build_tus( + tmp_path_factory.mktemp("chained_alias"), + { + "common.h": """\ + typedef struct token token; + typedef token token_ref; + """, + "a.c": """\ + #include "common.h" + + struct token { int type; int line; }; + + int token_line(token_ref *t) { + return t->line; + } + """, + "b.c": """\ + #include "common.h" + + struct token { int type; char *text; }; + + const char *token_text(token_ref *t) { + return t->text; + } + """, + }, + ) + + +@pytest.mark.xfail( + reason="A typedef of a typedef must be rebuilt on the link it names rather than " + "collapsed onto the tag, so it is rejected instead of flattening the chain.", +) +@pytest.mark.parametrize("order", permutations(["a.c", "b.c"]), ids="-".join) +def test_chained_typedef_completed_differently_in_two_tus( + chained_alias_tus: list[Path], order: tuple[str, ...] +): + """ + Same ODR-violation scenario as + test_typedef_struct_forward_decl_completed_differently_in_two_tus, but the type is + reached through a chain of typedefs: + + common.h – typedef struct token token; + typedef token token_ref; + a.c – #include "common.h", completes struct token { int type; int line; }; + b.c – #include "common.h", completes struct token { int type; char *text; }; + + Neither typedef is qualified, so this isolates chains from any question of + cv-qualifiers. + + `token_ref` names `token`, not the tag, so splitting the tag apart must rebuild it on + whatever `token` became. Re-expressing it against the tag instead would flatten the + chain into a second alias of the tag — which still compiles, so the indirection is + asserted on the emitted text rather than left for the compiler to catch. + """ + a_c, b_c = chained_alias_tus + base = a_c.parent + by_name = {tu.name: tu for tu in chained_alias_tus} + + compile_commands = _write_compile_commands( + base, [by_name[n] for n in order], extra_flags=f"-I{base}" + ) + consolidated = consolidate_init( + compile_commands, + source_priority=[a_c.resolve(), b_c.resolve()], + ) + + text = str(consolidated) + # Desired behavior: the whole chain is split, each link still naming the one before it. + assert "struct a_token {\n int type;\n int line;\n};\n" in text + assert "struct b_token {\n int type;\n char *text;\n};\n" in text + assert "typedef a_token a_token_ref;" in text + assert "typedef b_token b_token_ref;" in text + + # Each use keeps the link it was written with rather than the tag it resolves to. + assert "int token_line(a_token_ref *t) {\n return t->line;\n}\n" in text + assert "const char *token_text(b_token_ref *t) {\n return t->text;\n}\n" in text + + success, error = compile_c(consolidated, flags=["-Wall", "-Werror"]) + assert success, ( + f"Consolidated code does not compile:\n{error}\n\nConsolidated output:\n{consolidated}" + ) + + +@pytest.fixture(scope="module") +def header_gnu_source_tus(tmp_path_factory: pytest.TempPathFactory) -> list[Path]: + return _build_tus( + tmp_path_factory.mktemp("header_gnu_source"), + { + "config.h": """\ + #define _GNU_SOURCE + """, + "a.c": """\ + #include "config.h" + #include + + const char *not_renamed(int errnum) { + static char buf[256]; + const char *errstr = strerror_r(errnum, buf, sizeof(buf)); + return errstr; + } + """, + "b.c": """\ + #include "config.h" + #include + + const char *not_renamed2(int errnum) { + static char buf[128]; + const char *errstr = strerror_r(errnum, buf, sizeof(buf)); + return errstr; + } + """, + }, + cflags=["-D_DEFAULT_SOURCE"], + ) + + +@pytest.mark.parametrize("order", permutations(["a.c", "b.c"]), ids="-".join) +def test_header_defined_gnu_source_with_default_source_flag( + header_gnu_source_tus: list[Path], order: tuple[str, ...] +): + """ + Compile commands have -D_DEFAULT_SOURCE, but _GNU_SOURCE is defined inside + a project header (config.h/first.h). The consolidator preserves + _DEFAULT_SOURCE (from -D flags) but loses _GNU_SOURCE (from the header). + Code using the GNU strerror_r (returns char*) then fails with: + error: incompatible integer to pointer conversion + + Uses two TUs to exercise the union: both include config.h which defines + _GNU_SOURCE, so the union should emit it exactly once. + """ + base = header_gnu_source_tus[0].parent + by_name = {tu.name: tu for tu in header_gnu_source_tus} + + compile_commands = _write_compile_commands( + base, [by_name[n] for n in order], extra_flags=f"-D_DEFAULT_SOURCE -I{base}" ) consolidated = consolidate_init(compile_commands, source_priority=[]) - success, error = check_c(consolidated, flags=["-fsyntax-only", "-Werror"]) + text = str(consolidated) + assert "not_renamed" in text and "not_renamed2" in text + assert "a_not_renamed" not in text and "b_not_renamed2" not in text, consolidated + + success, error = compile_c(consolidated, flags=["-Wall", "-Werror"]) assert success, ( f"Consolidated code does not compile (header-defined _GNU_SOURCE lost):\n{error}\n\n" f"Consolidated output:\n{consolidated}" diff --git a/test/test_convert_json_to_rust.py b/test/test_convert_json_to_rust.py deleted file mode 100644 index b2bc136..0000000 --- a/test/test_convert_json_to_rust.py +++ /dev/null @@ -1,57 +0,0 @@ -# -# Copyright (C) 2025 Intel Corporation -# -# SPDX-License-Identifier: Apache-2.0 -# - -import pytest -import subprocess -from pathlib import Path - -from ideas import convert_tests - - -@pytest.fixture -def fixtures_dir() -> Path: - return Path(__file__).parent / "fixtures" / "text_processor" - - -@pytest.fixture -def cargo_toml(fixtures_dir: Path) -> Path: - return fixtures_dir / "Cargo.toml" - - -@pytest.fixture -def rust_tests_harness(fixtures_dir: Path) -> Path: - return fixtures_dir / "tests" / "test_cases.rs" - - -@pytest.fixture -def json_test_cases(fixtures_dir: Path) -> list[Path]: - test_case_files = sorted((fixtures_dir / "json_test_cases").glob("test*.json")) - return test_case_files - - -def test_convert_to_cargo_test( - json_test_cases: list[Path], cargo_toml: Path, rust_tests_harness: Path -): - # Write tests to tests/test_cases.rs - test_cases = convert_tests.convert_tests_for_exec(json_test_cases) - original_harness = rust_tests_harness.read_text() - with open(rust_tests_harness, "w") as f: - f.write(test_cases) - - # Execute cargo test --test test_cases - result = subprocess.run( - ["cargo", "test", "--manifest-path", cargo_toml, "--test", "test_cases"], - capture_output=True, - text=True, - ) - assert ( - "test result: ok. 5 passed; 0 failed; 0 ignored; 0 measured; 0 filtered out;" - in result.stdout - ) - - # Restore the original tests/test_cases.rs - with open(rust_tests_harness, "w") as f: - f.write(original_harness) diff --git a/test/test_extract_code_from_tu.py b/test/test_extract_code_from_tu.py deleted file mode 100644 index 6baf583..0000000 --- a/test/test_extract_code_from_tu.py +++ /dev/null @@ -1,98 +0,0 @@ -# -# Copyright (C) 2025 Intel Corporation -# -# SPDX-License-Identifier: Apache-2.0 -# - - -import pickle -import pytest -from pathlib import Path -from textwrap import dedent as d - -from ideas import ast - - -@pytest.fixture -def fixtures_dir() -> Path: - return Path(__file__).parent / "fixtures" / "ast" - - -@pytest.fixture -def i_code(fixtures_dir: Path) -> str: - return (fixtures_dir / "formatting.c.i").read_text() - - -def parse_c(code: str): - return ast.create_translation_unit(ast.CodeC(code=code)) - - -def test_all_code_from_tu(i_code: str): - # Parse the code using clang - tu = parse_c(i_code) - result = ast.extract_info_c(tu) - - assert isinstance(result.symbols["c:@F@foo"].code, ast.CodeC) - - # Check for exact formatting - assert ( - result.symbols["c:@F@foo"].code.code - == d( - """ - void foo() { - int x = 10; - int y = 20; - int z = 20; - if (z > 15) { - z += 5; - } else { - z -= 5; - } - } - """ - ).strip() - + "\n" - ) - - -def test_newline(): - code = "int main(int argc, char **argv) { return 0;\r\n}" - tu = parse_c(code) - result = ast.extract_info_c(tu) - - assert ( - result.symbols["c:@F@main"].code.code - == d( - """ - int main(int argc, char **argv) { - return 0; - } - """ - ).strip() - + "\n" - ) - - -def test_symbol_is_picklable(): - tu = parse_c("int main(int argc, char **argv) { return 0; }") - symbol = ast.extract_info_c(tu).symbols["c:@F@main"] - - blob = pickle.dumps(symbol) - restored = pickle.loads(blob) - - assert restored.name == symbol.name - assert restored.spelling == symbol.spelling - assert restored.kind == symbol.kind - assert restored.tu_preorder_index == symbol.tu_preorder_index - assert restored.code == symbol.code - - -def test_tree_result_is_picklable(): - tu = parse_c("int main(int argc, char **argv) { return 0; }") - result = ast.extract_info_c(tu) - - blob = pickle.dumps(result) - restored = pickle.loads(blob) - - assert isinstance(restored, ast.TreeResult) - assert restored == result diff --git a/test/test_model.py b/test/test_model.py new file mode 100644 index 0000000..7881367 --- /dev/null +++ b/test/test_model.py @@ -0,0 +1,78 @@ +# +# Copyright (C) 2026 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +import logging +import importlib +from unittest.mock import patch + +import pytest +import dspy +import dspy.clients.lm as dspy_lm +from dspy.utils.logging_utils import configure_dspy_loggers +from litellm import ModelResponse +from litellm.types.utils import Choices, Message, Usage + +import ideas.model + + +def _truncated_completion(request, num_retries, cache): + return ModelResponse( + model="openai/gpt-4o-mini", + choices=[ + Choices( + index=0, + finish_reason="length", + message=Message(role="assistant", content="partial output"), + ) + ], + usage=Usage(prompt_tokens=5, completion_tokens=10, total_tokens=15), + ) + + +@pytest.fixture +def translate_log(tmp_path): + log_path = tmp_path / "translate.log" + handler = logging.FileHandler(log_path) + handler.setFormatter(logging.Formatter("[%(name)s][%(levelname)s] %(message)s")) + root = logging.getLogger() + prior_level = root.level + root.addHandler(handler) + root.setLevel(logging.INFO) + try: + yield log_path + finally: + handler.close() + root.removeHandler(handler) + root.setLevel(prior_level) + + +def test_truncation_warning_written_to_translate_log(translate_log): + configure_dspy_loggers("dspy") + importlib.reload(ideas.model) + + # Mimic the app's own logging around a DSPy module call + app_logger = logging.getLogger("ideas.translate_snippet") + app_logger.info("Translating snippet `demo` ...") + + lm = dspy.LM(model="openai/gpt-4o-mini", max_tokens=10, temperature=0.0, cache=False) + with patch.object(dspy_lm, "litellm_completion", _truncated_completion): + lm("hello") + + app_logger.info("Translated snippet `demo`") + + warning = ( + "LM response was truncated due to exceeding max_tokens=10. " + "You can inspect the latest LM interactions with `dspy.inspect_history()`. " + "To avoid truncation, consider passing a larger max_tokens when setting up dspy.LM. " + "You may also consider increasing the temperature (currently 0.0) " + " if the reason for truncation is repetition." + ) + expected = ( + "[ideas.translate_snippet][INFO] Translating snippet `demo` ...\n" + f"[dspy.clients.lm][WARNING] {warning}\n" + "[ideas.translate_snippet][INFO] Translated snippet `demo`\n" + ) + assert translate_log.read_text() == expected diff --git a/test/test_tools.py b/test/test_tools.py deleted file mode 100644 index 0e99a07..0000000 --- a/test/test_tools.py +++ /dev/null @@ -1,123 +0,0 @@ -# -# Copyright (C) 2025 Intel Corporation -# -# SPDX-License-Identifier: Apache-2.0 -# - -from pathlib import Path - -import pytest - -from ideas import tools -from ideas.ast import CodeC - - -@pytest.fixture -def fixtures_dir() -> Path: - return Path(__file__).parent / "fixtures" / "compile" - - -@pytest.fixture -def c_paths(fixtures_dir: Path) -> tuple[Path, ...]: - return fixtures_dir / "hello_world_good.c", fixtures_dir / "hello_world_bad.c" - - -@pytest.fixture -def rust_paths(fixtures_dir: Path) -> tuple[Path, Path]: - return fixtures_dir / "hello_world_good.rs", fixtures_dir / "hello_world_bad.rs" - - -def test_check_c(c_paths: tuple[Path, ...]): - # Compilation should succeed - success1, out1 = tools.check_c(CodeC(c_paths[0].read_text())) - assert success1, out1 - assert out1 == "" - - # Compilation should fail - success2, out2 = tools.check_c(CodeC(c_paths[1].read_text())) - assert not success2, out2 - assert out2 != "" - - -def test_check_rust(rust_paths: tuple[Path, Path]): - # Compilation should succeed - success1, out1 = tools.check_rust(rust_paths[0].read_text()) - assert success1 - assert out1 == "" - - # Compilation should fail - success2, out2 = tools.check_rust(rust_paths[1].read_text()) - assert not success2 - assert out2 != "" - - -@pytest.fixture -def echo_123(fixtures_dir: Path) -> Path: - return fixtures_dir / "echo_123" - - -def test_run_and_check_test_args_in(echo_123): - assert tools.run_and_check_test(echo_123, {"args": None, "in": None, "out": "1 2 3"}) - assert tools.run_and_check_test(echo_123, {"args": [], "in": [], "out": "1 2 3"}) - assert tools.run_and_check_test(echo_123, {"args": "", "in": "", "out": "1 2 3"}) - - -def test_run_and_check_test_in_only(echo_123): - assert tools.run_and_check_test(echo_123, {"in": None, "out": "1 2 3"}) - assert tools.run_and_check_test(echo_123, {"in": [], "out": "1 2 3"}) - assert tools.run_and_check_test(echo_123, {"in": "", "out": "1 2 3"}) - - -def test_run_and_check_test_args_only(echo_123): - assert tools.run_and_check_test(echo_123, {"args": None, "out": "1 2 3"}) - assert tools.run_and_check_test(echo_123, {"args": [], "out": "1 2 3"}) - assert tools.run_and_check_test(echo_123, {"args": "", "out": "1 2 3"}) - - -def test_run_and_check_test(echo_123): - assert tools.run_and_check_test(echo_123, {"out": "1 2 3"}) - assert tools.run_and_check_test(echo_123, {"out": ["1 2 3"]}) - - -def test_run_and_check_test_echo_args(): - assert tools.run_and_check_test("echo", {"args": ["1", "2", "3"], "out": "1 2 3"}) - assert tools.run_and_check_test("echo", {"args": "1 2 3", "out": "1 2 3"}) - - -def test_run_and_check_test_echo_args_number(): - assert tools.run_and_check_test("echo", {"args": [1, 2, 3], "out": "1 2 3"}) - assert tools.run_and_check_test("echo", {"args": [1.0, 2.0, 3.0], "out": "1.0 2.0 3.0"}) - - -def test_run_and_check_test_missing_out(): - with pytest.raises(Exception): - tools.run_and_check_test("echo", {"args": "", "in": ""}) - - -@pytest.fixture -def echo_stdin(fixtures_dir: Path) -> Path: - return fixtures_dir / "echo_stdin" - - -def test_run_and_check_test_echo_stdin_str(echo_stdin): - assert tools.run_and_check_test(echo_stdin, {"in": "1 2 3", "out": "1 2 3"}) - assert tools.run_and_check_test(echo_stdin, {"in": "1 2 3\n", "out": "1 2 3"}) - - -def test_run_and_check_test_echo_stdin_str_newlines(echo_stdin): - assert tools.run_and_check_test(echo_stdin, {"in": "1\n2\n3", "out": "1\n2\n3"}) - assert tools.run_and_check_test(echo_stdin, {"in": "1\n2\n3\n", "out": "1\n2\n3"}) - assert tools.run_and_check_test(echo_stdin, {"in": "1\n2\n3\n", "out": [1, 2, 3]}) - assert tools.run_and_check_test(echo_stdin, {"in": "1\n2\n3\n", "out": ["1", "2", "3"]}) - - -def test_run_and_check_test_echo_stdin_list(echo_stdin): - assert tools.run_and_check_test(echo_stdin, {"in": ["1 2 3"], "out": "1 2 3"}) - assert tools.run_and_check_test(echo_stdin, {"in": ["1 2 3\n"], "out": "1 2 3"}) - - -def test_run_and_check_test_echo_stdin_list_newlines(echo_stdin): - assert tools.run_and_check_test(echo_stdin, {"in": ["1", "2", "3"], "out": "1\n2\n3"}) - assert tools.run_and_check_test(echo_stdin, {"in": ["1", "2", "3", "\n"], "out": "1\n2\n3"}) - assert tools.run_and_check_test(echo_stdin, {"in": ["1", "2", "3"], "out": [1, 2, 3]}) - assert tools.run_and_check_test(echo_stdin, {"in": ["1", "2", "3"], "out": ["1", "2", "3"]}) diff --git a/test/test_translate_recurrent.py b/test/test_translate_recurrent.py new file mode 100644 index 0000000..7662432 --- /dev/null +++ b/test/test_translate_recurrent.py @@ -0,0 +1,856 @@ +# +# Copyright (C) 2026 Intel Corporation +# +# SPDX-License-Identifier: Apache-2.0 +# + +import pytest +from pathlib import Path +from unittest.mock import MagicMock, patch + +import dspy +import networkx as nx + +from ideas import ast +from ideas.ast import CodeC +from ideas.ast_rust import CodeRust +from ideas.consolidate import get_symbols_and_dependencies, create_ast_order +from ideas.translate_recurrent import RecurrentTranslator +from ideas.translate_recurrent import SymbolGroup, SymbolName, TranslationContext +from ideas.translate_snippet import SnippetTranslator +from ideas.wrapper import WrapperGenerator + + +def _make_graph( + c_code: ast.CodeC, +) -> tuple[nx.DiGraph, list[SymbolGroup], dict[SymbolName, ast.Symbol]]: + tu = ast.create_translation_unit(c_code) + tree = ast.extract_info_c(tu) + symbols, dependencies = get_symbols_and_dependencies([tree]) + ast_order = create_ast_order([], [tree]) + G = nx.from_dict_of_lists(dependencies, create_using=nx.DiGraph) # type: ignore[arg-type] + assert isinstance(G, nx.DiGraph) + groups = list( + nx.lexicographical_topological_sort( + G.reverse(copy=False), key=ast.create_symbol_lexical_key_fn(symbols, ast_order) + ) + ) + return G, groups, symbols + + +def test_context_isolated_node(): + G, groups, symbols = _make_graph(ast.CodeC("int standalone(void) { return 42; }")) + + (func_group,) = groups # only one symbol in this translation unit + + ctx = TranslationContext.build(G, func_group, groups, symbols) + + # An isolated node has no dependents, no dependencies, and nothing translated yet. + assert str(ctx.dependent_code) == "" + assert str(ctx.support_code) == "" + assert str(ctx.crate_code) == "" + assert str(ctx.reference_code) == "" + + +_TYPEDEF_CHAIN = ast.CodeC(""" +struct MyStruct { int x; }; +typedef struct MyStruct my_struct_t; +int use_my(my_struct_t s) { return s.x; } +""") + +_NESTED_STRUCT = ast.CodeC(""" +struct Inner { int x; }; +struct Outer { struct Inner inner; }; +int process_outer(struct Outer* o) { return o->inner.x; } +""") + +_NESTED_TYPEDEF = ast.CodeC(""" +struct Point { int x; int y; }; +typedef struct Point point_t; +typedef point_t* point_ptr_t; +int use_point(point_ptr_t p) { (void)p; return 0; } +""") + +_ENUM_TYPEDEF = ast.CodeC(""" +enum Status { STATUS_OK = 0, STATUS_ERROR = 1 }; +typedef enum Status status_t; +int check_status(status_t s) { return (int)s; } +""") + +_TWO_STRUCTS = ast.CodeC(""" +struct A { int x; }; +struct B { int y; }; +""") + +_VAR_INTERMEDIATE = ast.CodeC(""" +struct Config { int timeout; }; +struct Config global_config; +int get_timeout(void) { return global_config.timeout; } +""") + + +def test_context_typedef_indirection(): + G, groups, symbols = _make_graph(_TYPEDEF_CHAIN) + + struct_group: SymbolGroup = ("c:@S@MyStruct",) + + ctx = TranslationContext.build(G, struct_group, groups, symbols) + + # typedef is the immediate predecessor — should be in dependent_code + assert symbols["c:file.c@T@my_struct_t"].code in ctx.dependent_code + # use_my consumes the struct (via typedef); its call signature reveals whether + # the type is passed by value or pointer, which informs Copy/Clone/borrowing decisions + assert symbols["c:@F@use_my"].code in ctx.dependent_code + + +def test_context_linear_chain_middle(): + G, groups, symbols = _make_graph(_TYPEDEF_CHAIN) + + struct_group: SymbolGroup = ("c:@S@MyStruct",) + typedef_group: SymbolGroup = ("c:file.c@T@my_struct_t",) + + struct_translation = CodeRust("pub struct MyStruct { pub x: i32 }") + translations: dict[SymbolGroup, CodeRust] = {struct_group: struct_translation} + + ctx = TranslationContext.build(G, typedef_group, groups, symbols, translations=translations) + + # function is the immediate predecessor of typedef in the dependency graph + assert symbols["c:@F@use_my"].code in ctx.dependent_code + # struct is the immediate successor of typedef → full code used + assert symbols["c:@S@MyStruct"].code in ctx.support_code + # struct has been translated so its Rust code appears in crate_code + assert struct_translation in ctx.crate_code + + +def test_context_nested_struct_fields(): + G, groups, symbols = _make_graph(_NESTED_STRUCT) + + inner_group: SymbolGroup = ("c:@S@Inner",) + + ctx = TranslationContext.build(G, inner_group, groups, symbols) + + # Outer embeds Inner — should be in dependent_code + assert symbols["c:@S@Outer"].code in ctx.dependent_code + # process_outer shows how Outer (and by extension Inner) is used via pointer, + # informing borrowing semantics for Inner's translation + assert symbols["c:@F@process_outer"].code in ctx.dependent_code + + +def test_context_nested_typedefs(): + G, groups, symbols = _make_graph(_NESTED_TYPEDEF) + + struct_group: SymbolGroup = ("c:@S@Point",) + + ctx = TranslationContext.build(G, struct_group, groups, symbols) + + # point_t is 1 hop — should be in dependent_code + assert symbols["c:file.c@T@point_t"].code in ctx.dependent_code + # point_ptr_t aliases a pointer to point_t — still part of the usage chain + assert symbols["c:file.c@T@point_ptr_t"].code in ctx.dependent_code + # use_point is the actual consumer — reveals how the pointer type is used + assert symbols["c:@F@use_point"].code in ctx.dependent_code + + +def test_context_enum_typedef(): + G, groups, symbols = _make_graph(_ENUM_TYPEDEF) + + enum_group: SymbolGroup = ("c:@E@Status",) + + ctx = TranslationContext.build(G, enum_group, groups, symbols) + + # status_t is the immediate predecessor — should be in dependent_code + assert symbols["c:file.c@T@status_t"].code in ctx.dependent_code + # check_status uses the enum; its usage informs how Status should be + # represented in Rust (e.g., as a plain enum vs. integer newtype) + assert symbols["c:@F@check_status"].code in ctx.dependent_code + + +def test_context_variable_intermediate(): + G, groups, symbols = _make_graph(_VAR_INTERMEDIATE) + + struct_group: SymbolGroup = ("c:@S@Config",) + + ctx = TranslationContext.build(G, struct_group, groups, symbols) + + # global_config is the immediate predecessor — should be in dependent_code + assert symbols["c:@global_config"].code in ctx.dependent_code + # get_timeout reveals that Config backs global state, which informs + # Rust's ownership and synchronization strategy (Mutex, OnceCell, etc.) + assert symbols["c:@F@get_timeout"].code in ctx.dependent_code + + +def test_reference_code_none_hop(): + G, groups, symbols = _make_graph(_TWO_STRUCTS) + + a_group: SymbolGroup = ("c:@S@A",) + b_group: SymbolGroup = ("c:@S@B",) + + a_translation = CodeRust("pub struct A { pub x: i32 }") + translations: dict[SymbolGroup, CodeRust] = {a_group: a_translation} + + ctx = TranslationContext.build(G, b_group, groups, symbols, translations=translations) + + # A and B are unrelated — hops.get(a_group) is None when building context for B. + # The None case falls through to strip_fns(..., delete=True), which deletes top-level + # functions but keeps types as-is, since a type may still be needed even when the + # dependency on it is not visible to static analysis. + assert a_translation in ctx.reference_code + + +def _make_translator(tmp_path: Path, **kwargs) -> tuple[RecurrentTranslator, Path, Path, Path]: + # -sys crate + sys_src_dir = tmp_path / "sys" / "src" + sys_src_dir.mkdir(parents=True) + sys_lib = sys_src_dir / "lib.rs" + sys_lib.write_bytes(b"") + sys_lib.with_suffix(".c").write_bytes(b"") + + sys_crate = MagicMock() + sys_crate.lib_src_path = sys_lib + sys_crate.lib_name = "libfoo_sys" + + # -rs crate + rs_src_dir = tmp_path / "rs" / "src" + rs_src_dir.mkdir(parents=True) + (rs_src_dir / "lib.rs").write_bytes(b"") + + rs_crate = MagicMock() + rs_crate.lib_src_path = rs_src_dir / "lib.rs" + rs_crate.main_src_path = None + rs_crate.lib_name = "foo_rs" + rs_crate.cargo_build.return_value = (True, "") + + # hybrid crate + hybrid_src_dir = tmp_path / "hybrid" / "src" + hybrid_src_dir.mkdir(parents=True) + (hybrid_src_dir / "lib.rs").write_bytes(b"") + + crate = MagicMock() + crate.lib_src_path = hybrid_src_dir / "lib.rs" + crate.main_src_path = None + crate.src_dir = hybrid_src_dir + crate.cargo_toml = tmp_path / "hybrid" / "Cargo.toml" + crate.cargo_build.return_value = (True, "") + + # RecurrentTranslator with mocked inputs + translator = RecurrentTranslator( + sys_crate=sys_crate, + crate=crate, + rs_crate=rs_crate, + **kwargs, + ) + return ( + translator, + hybrid_src_dir / "lib.rs", + rs_src_dir / "lib.rs", + sys_lib.with_suffix(".c"), + ) + + +def test_wrapper_only_restore_on_final_wrapper_failure(tmp_path: Path) -> None: + # Create a SnippetTranslator backed by a mock LM that always succeeds + mock_translator = MagicMock( + return_value=dspy.Prediction(translation=CodeRust("pub fn bar() {}")) + ) + symbol_translator = SnippetTranslator( + translator=MagicMock(return_value=mock_translator), # type: ignore[arg-type] + max_iters=1, + ) + + # Create a WrapperGenerator backed by a mock LM that returns a unique but invalid + # wrapper each attempt because validate_changes produces scope_feedback cause a failure + # because we patch ideas.translate_recurrent.generate_unimplemented_function_wrapper below + attempt = 0 + + def mock_wrapper_side_effect(*args, **kwargs): + nonlocal attempt + attempt += 1 + return dspy.Prediction(wrapper=CodeRust(f"// attempt {attempt} wrapper")) + + mock_wrapper = MagicMock(side_effect=mock_wrapper_side_effect) + symbol_wrapper = WrapperGenerator( + wrapper=MagicMock(return_value=mock_wrapper), # type: ignore[arg-type] + max_iters=1, + ) + + # Create a RecurrentTranslator with max_iters=2 to exercise both restore paths + # NOTE: Supplying tests is required so WrapperGenerator is called since the symbol is not global + translator, hybrid_lib, rs_lib, sys_c = _make_translator( + tmp_path, + symbol_translator=symbol_translator, + symbol_wrapper=symbol_wrapper, + tests="dummy_test", + max_iters=2, + ) + + # Construct symbols and dependencies for a single function `bar` to be wrapped + sym: SymbolName = "c:@F@bar" + sym_group: SymbolGroup = (sym,) + mock_symbol = MagicMock() + mock_symbol.name = sym + mock_symbol.spelling = "bar" + mock_symbol.code = CodeC("int bar(void) { return 0; }") + mock_symbol.llm_context_declaration = "int bar(void);" + mock_symbol.is_function = True + mock_symbol.is_definition = True + mock_symbol.is_global = False + mock_symbol.is_variable = False + mock_symbol.is_type = False + + symbols: dict[SymbolName, MagicMock] = {sym: mock_symbol} + dependencies: dict[SymbolGroup, list[SymbolGroup]] = {sym_group: []} + + # Write the C definition that clang_make_extern_ will extern-ify + sys_c.write_text("int bar(void) { return 0; }") + + # Create wrapper file with known content + wrap_bar = hybrid_lib.parent / "wrap_bar.rs" + original_content = "// previous good wrapper" + wrap_bar.write_text(original_content) + + # Run the translator + with patch( + "ideas.translate_recurrent.generate_unimplemented_function_wrapper", + return_value=CodeRust("pub fn bar() { unimplemented!() }"), + ): + pred = translator(symbols=symbols, dependencies=dependencies) + assert pred.success + + # Translator should have iterated twice + assert mock_translator.call_count == 2 + assert mock_wrapper.call_count == 2 + + # Must have rolled back wrap_bar.rs to its pre-call state + assert wrap_bar.read_text() == original_content + + # Must not have duplicated `pub mod wrap_bar;` across retries + assert hybrid_lib.read_text().count("pub mod wrap_bar;") == 1 + + # Must not have created spurious wrapper files across retries + assert list(hybrid_lib.parent.glob("wrap_*.rs")) == [wrap_bar] + + # Must not have duplicated the translation in the -rs crate across retries + assert rs_lib.read_text().count("pub fn bar") == 1 + + # Final try does not restore the C file because translation succeeded + assert "extern int bar(void);" in sys_c.read_text() + assert "{ return 0; }" not in sys_c.read_text() + + # Translation must not bleed into the hybrid crate + assert "pub fn bar" not in hybrid_lib.read_text() + + # Module declaration must not bleed into the -rs crate + assert "pub mod wrap_bar;" not in rs_lib.read_text() + + +def test_full_restore_after_two_translation_failures(tmp_path: Path) -> None: + # Each attempt produces a unique translation so the translation-loop guard does not + # short-circuit before the second full restore runs. + attempt = 0 + + def mock_translator_side_effect(*args, **kwargs): + nonlocal attempt + attempt += 1 + return dspy.Prediction( + translation=CodeRust(f"pub fn bar() {{ /* attempt {attempt} */ }}") + ) + + mock_translator = MagicMock(side_effect=mock_translator_side_effect) + symbol_translator = SnippetTranslator( + translator=MagicMock(return_value=mock_translator), # type: ignore[arg-type] + max_iters=1, + ) + + # Wrapper generator is never reached because translation always fails; a plain mock is enough. + translator, hybrid_lib, rs_lib, sys_c = _make_translator( + tmp_path, + symbol_translator=symbol_translator, + symbol_wrapper=MagicMock(), + tests="dummy_test", + max_iters=2, + ) + + # Make cargo build always fail for the -rs crate so every translation attempt fails + # so a full restore (not wrappers-only) is performed on *both* iterations. + rs_crate_mock: MagicMock = translator.rust_crate # type: ignore[assignment] + rs_crate_mock.cargo_build.return_value = (False, "compilation error") + + sym: SymbolName = "c:@F@bar" + sym_group: SymbolGroup = (sym,) + mock_symbol = MagicMock() + mock_symbol.name = sym + mock_symbol.spelling = "bar" + mock_symbol.code = CodeC("int bar(void) { return 0; }") + mock_symbol.llm_context_declaration = "int bar(void);" + mock_symbol.is_function = True + mock_symbol.is_definition = True + mock_symbol.is_global = False + mock_symbol.is_variable = False + mock_symbol.is_type = False + + symbols: dict[SymbolName, MagicMock] = {sym: mock_symbol} + dependencies: dict[SymbolGroup, list[SymbolGroup]] = {sym_group: []} + + # Write known content so the restore assertions check for something specific, + # not just whatever RecurrentTranslator.__init__ happened to leave behind. + initial_rs_lib = "#![forbid(unsafe_code)]\n\n// known rs baseline\n" + initial_hybrid_lib = "use libfoo_sys as _;\n// known hybrid baseline\n" + initial_c_src = "int bar(void) { return 0; }" + original_wrapper_content = "// previous good wrapper" + rs_lib.write_text(initial_rs_lib) + hybrid_lib.write_text(initial_hybrid_lib) + sys_c.write_text(initial_c_src) + wrap_bar = hybrid_lib.parent / "wrap_bar.rs" + wrap_bar.write_text(original_wrapper_content) + + # Run the translator + pred = translator(symbols=symbols, dependencies=dependencies) + assert not pred.success + + # Both iterations must have attempted translation + assert mock_translator.call_count == 2 + + # Full restore: -rs crate must be rolled back to its pre-call state + assert rs_lib.read_text() == initial_rs_lib + + # Full restore: hybrid crate must be rolled back to its pre-call state + assert hybrid_lib.read_text() == initial_hybrid_lib + + # Full restore: C source must be rolled back (clang_make_extern_ must not have run) + assert sys_c.read_text() == initial_c_src + + # Full restore: the pre-existing wrapper file must be intact + assert wrap_bar.read_text() == original_wrapper_content + + # No spurious wrapper files must have been created across the two iterations + assert list(hybrid_lib.parent.glob("wrap_*.rs")) == [wrap_bar] + + # Translation must not bleed into the hybrid crate across retries + assert "pub fn bar" not in hybrid_lib.read_text() + + # Module declaration must not bleed into the -rs crate across retries + assert "pub mod wrap_bar;" not in rs_lib.read_text() + + +def test_no_restore_on_test_failure(tmp_path: Path) -> None: + # Translation and wrapping both succeed; only the test run fails. + # On the last (only) iteration with failure == "test", the restore branch is: + # `elif result.failure == "test": pass` — nothing is restored. + mock_translator = MagicMock( + return_value=dspy.Prediction(translation=CodeRust("pub fn bar() {}")) + ) + symbol_translator = SnippetTranslator( + translator=MagicMock(return_value=mock_translator), # type: ignore[arg-type] + max_iters=1, + ) + + # Returning the same CodeRust as the unimplemented wrapper means validate_changes + # finds no diffs → scope_feedback is empty → pred.success = True → wrapping succeeds. + unimplemented_stub = CodeRust("pub fn bar_wrapper() -> i32 { unimplemented!() }") + mock_wrapper_gen = MagicMock(return_value=dspy.Prediction(wrapper=unimplemented_stub)) + symbol_wrapper = WrapperGenerator( + wrapper=MagicMock(return_value=mock_wrapper_gen), # type: ignore[arg-type] + max_iters=1, + ) + + translator, hybrid_lib, rs_lib, sys_c = _make_translator( + tmp_path, + symbol_translator=symbol_translator, + symbol_wrapper=symbol_wrapper, + tests="dummy_test", + max_iters=1, + ) + + # Make cargo_test fail so result.failure == "test" + crate_mock: MagicMock = translator.crate # type: ignore[assignment] + crate_mock.cargo_test.return_value = ( + False, + '{"type":"test","name":"test_bar","event":"failed"}', + "1 test failed", + "", + ) + + sym: SymbolName = "c:@F@bar" + sym_group: SymbolGroup = (sym,) + mock_symbol = MagicMock() + mock_symbol.name = sym + mock_symbol.spelling = "bar" + mock_symbol.code = CodeC("int bar(void) { return 0; }") + mock_symbol.llm_context_declaration = "int bar(void);" + mock_symbol.is_function = True + mock_symbol.is_definition = True + mock_symbol.is_global = False + mock_symbol.is_variable = False + mock_symbol.is_type = False + + symbols: dict[SymbolName, MagicMock] = {sym: mock_symbol} + dependencies: dict[SymbolGroup, list[SymbolGroup]] = {sym_group: []} + + sys_c.write_text("int bar(void) { return 0; }") + + # Run the translator + with patch( + "ideas.translate_recurrent.generate_unimplemented_function_wrapper", + return_value=unimplemented_stub, + ): + pred = translator(symbols=symbols, dependencies=dependencies) + assert pred.success + + # No restore: translation is kept in the -rs crate + assert "pub fn bar" in rs_lib.read_text() + + # No restore: wrapper module declaration is kept in the hybrid crate + assert "pub mod wrap_bar;" in hybrid_lib.read_text() + + # No restore: C source is kept with the extern declaration written by clang_make_extern_ + assert "extern int bar(void);" in sys_c.read_text() + + # No restore: wrapper file written during wrapping is kept + wrap_bar = hybrid_lib.parent / "wrap_bar.rs" + assert wrap_bar.exists() + + +def test_feedback_after_test_failure(tmp_path: Path) -> None: + # Translation and wrapping succeed on both iterations + mock_translator = MagicMock( + return_value=dspy.Prediction(translation=CodeRust("pub fn bar() {}")) + ) + symbol_translator = SnippetTranslator( + translator=MagicMock(return_value=mock_translator), # type: ignore[arg-type] + max_iters=1, + ) + + # Returning the same CodeRust as the unimplemented wrapper means validate_changes passes + unimplemented_stub = CodeRust("pub fn bar_wrapper() -> i32 { unimplemented!() }") + mock_wrapper_gen = MagicMock(return_value=dspy.Prediction(wrapper=unimplemented_stub)) + symbol_wrapper = WrapperGenerator( + wrapper=MagicMock(return_value=mock_wrapper_gen), # type: ignore[arg-type] + max_iters=1, + ) + + translator, hybrid_lib, rs_lib, sys_c = _make_translator( + tmp_path, + symbol_translator=symbol_translator, + symbol_wrapper=symbol_wrapper, + tests="dummy_test", + max_iters=2, + ) + + # First cargo_test call fails, second succeeds. + crate_mock: MagicMock = translator.crate # type: ignore[assignment] + crate_mock.cargo_test.side_effect = [ + (False, '{"type":"test","name":"test_bar","event":"failed"}', "1 test failed", ""), + (True, '{"type":"test","name":"test_bar","event":"ok"}', "", ""), + ] + + sym: SymbolName = "c:@F@bar" + sym_group: SymbolGroup = (sym,) + mock_symbol = MagicMock() + mock_symbol.name = sym + mock_symbol.spelling = "bar" + mock_symbol.code = CodeC("int bar(void) { return 0; }") + mock_symbol.llm_context_declaration = "int bar(void);" + mock_symbol.is_function = True + mock_symbol.is_definition = True + mock_symbol.is_global = False + mock_symbol.is_variable = False + mock_symbol.is_type = False + + symbols: dict[SymbolName, MagicMock] = {sym: mock_symbol} + dependencies: dict[SymbolGroup, list[SymbolGroup]] = {sym_group: []} + + sys_c.write_text("int bar(void) { return 0; }") + + # Run the translator + with patch( + "ideas.translate_recurrent.generate_unimplemented_function_wrapper", + return_value=unimplemented_stub, + ): + pred = translator(symbols=symbols, dependencies=dependencies) + assert pred.success + + # Translator must have been called twice: once per outer iteration. + assert mock_translator.call_count == 2 + + # Extract the feedback forwarded to the translator on the second (retry) call. + second_call_feedback: str = mock_translator.call_args_list[1].kwargs["feedback"] + + # Test failures should tell the translator its output doesn't match the C behavior. + assert "does not match the behavior" in second_call_feedback + + +def test_feedback_after_wrap_failure(tmp_path: Path) -> None: + # Translation succeeds on both iterations + mock_translator = MagicMock( + return_value=dspy.Prediction(translation=CodeRust("pub fn bar() {}")) + ) + symbol_translator = SnippetTranslator( + translator=MagicMock(return_value=mock_translator), # type: ignore[arg-type] + max_iters=1, + ) + + # Returning the same CodeRust as the unimplemented wrapper means validate_changes passes + unimplemented_stub = CodeRust("pub fn bar_wrapper() -> i32 { unimplemented!() }") + mock_wrapper_gen = MagicMock(return_value=dspy.Prediction(wrapper=unimplemented_stub)) + symbol_wrapper = WrapperGenerator( + wrapper=MagicMock(return_value=mock_wrapper_gen), # type: ignore[arg-type] + max_iters=1, + ) + + translator, hybrid_lib, rs_lib, sys_c = _make_translator( + tmp_path, + symbol_translator=symbol_translator, + symbol_wrapper=symbol_wrapper, + tests="dummy_test", + max_iters=2, + ) + + # Force failure on the first wrapper build attempt, then succeed on the second. + crate_mock: MagicMock = translator.crate # type: ignore[assignment] + crate_mock.cargo_build.side_effect = [ + (True, ""), # _wrap_function scaffold check (iter 1) + (False, "build error"), # build(wrapper) inside WrapperGenerator (iter 1) + (True, ""), # _wrap_function scaffold check (iter 2) + (True, ""), # build(wrapper) inside WrapperGenerator (iter 2) + (True, ""), # _test_symbol pre-test cargo_build (iter 2) + ] + crate_mock.cargo_test.return_value = (True, "", "", "") + + sym: SymbolName = "c:@F@bar" + sym_group: SymbolGroup = (sym,) + mock_symbol = MagicMock() + mock_symbol.name = sym + mock_symbol.spelling = "bar" + mock_symbol.code = CodeC("int bar(void) { return 0; }") + mock_symbol.llm_context_declaration = "int bar(void);" + mock_symbol.is_function = True + mock_symbol.is_definition = True + mock_symbol.is_global = False + mock_symbol.is_variable = False + mock_symbol.is_type = False + + symbols: dict[SymbolName, MagicMock] = {sym: mock_symbol} + dependencies: dict[SymbolGroup, list[SymbolGroup]] = {sym_group: []} + + sys_c.write_text("int bar(void) { return 0; }") + + # Run the translator + with patch( + "ideas.translate_recurrent.generate_unimplemented_function_wrapper", + return_value=unimplemented_stub, + ): + pred = translator(symbols=symbols, dependencies=dependencies) + assert pred.success + + # Translator must have been called twice: once per outer iteration. + assert mock_translator.call_count == 2 + + # Extract the feedback forwarded to the translator on the second (retry) call. + second_call_feedback: str = mock_translator.call_args_list[1].kwargs["feedback"] + + # Wrap failures should instruct the translator to produce wrapper-friendly code. + assert "C-compatible FFI wrapper" in second_call_feedback + + +def test_feedback_after_translate_failure(tmp_path: Path) -> None: + # Translation fails on the first iteration (the -rs build rejects it) and succeeds + # on the second. + mock_translator = MagicMock( + return_value=dspy.Prediction(translation=CodeRust("pub fn bar() {}")) + ) + symbol_translator = SnippetTranslator( + translator=MagicMock(return_value=mock_translator), # type: ignore[arg-type] + max_iters=1, + ) + + # tests=None: wrapping is skipped for non-global symbols, so only translation runs. + translator, hybrid_lib, rs_lib, sys_c = _make_translator( + tmp_path, + symbol_translator=symbol_translator, + symbol_wrapper=MagicMock(), + tests=None, + max_iters=2, + ) + + # Make the -rs build fail on iteration 1 and succeed on iteration 2. + rs_crate_mock: MagicMock = translator.rust_crate # type: ignore[assignment] + rs_crate_mock.cargo_build.side_effect = [ + (False, "compile error"), # iter 1: build rejects translation → failure="translate" + (True, ""), # iter 2: build accepts translation → success + ] + + sym: SymbolName = "c:@F@bar" + sym_group: SymbolGroup = (sym,) + mock_symbol = MagicMock() + mock_symbol.name = sym + mock_symbol.spelling = "bar" + mock_symbol.code = CodeC("int bar(void) { return 0; }") + mock_symbol.llm_context_declaration = "int bar(void);" + mock_symbol.is_function = True + mock_symbol.is_definition = True + mock_symbol.is_global = False + mock_symbol.is_variable = False + mock_symbol.is_type = False + + symbols: dict[SymbolName, MagicMock] = {sym: mock_symbol} + dependencies: dict[SymbolGroup, list[SymbolGroup]] = {sym_group: []} + + sys_c.write_text("int bar(void) { return 0; }") + + pred = translator(symbols=symbols, dependencies=dependencies) + + assert pred.success + + # Translator must have been called twice: once per outer iteration. + assert mock_translator.call_count == 2 + + # Extract the feedback forwarded to the translator on the second (retry) call. + second_call_feedback: str = mock_translator.call_args_list[1].kwargs["feedback"] + + # The build error from the first iteration must be surfaced to the retry so the + # translator knows why its previous output was rejected. + assert "compile error" in second_call_feedback + + +_TYPEDEF_STRUCT = ast.CodeC("typedef struct MyStruct { int x; } my_struct_t;") + +# An unnamed struct still gets external linkage: C11 6.7.8p3 makes the typedef name the +# tag's name for linkage purposes, so clang reports spelling `my_struct_t` for the +# STRUCT_DECL and the wrapper is named after the typedef instead of the (absent) tag. +_ANONYMOUS_TYPEDEF_STRUCT = ast.CodeC("typedef struct { int x; } my_struct_t;") + + +@pytest.mark.parametrize( + ("c_source", "wrapper_name"), + [ + (_TYPEDEF_STRUCT, "wrap_MyStruct"), + (_ANONYMOUS_TYPEDEF_STRUCT, "wrap_my_struct_t"), + ], + ids=["named_tag", "anonymous_tag"], +) +def test_type_wrapper_generated_for_typedef_struct( + tmp_path: Path, c_source: CodeC, wrapper_name: str +) -> None: + mock_translator = MagicMock( + return_value=dspy.Prediction(translation=CodeRust("pub struct MyStruct { pub x: i32 }")) + ) + symbol_translator = SnippetTranslator( + translator=MagicMock(return_value=mock_translator), # type: ignore[arg-type] + max_iters=1, + ) + + # Returning the unimplemented template unchanged means validate_changes finds no + # diffs, so wrapping succeeds. Both round-trip tests must be present because + # `_wrap_type` rejects wrappers that drop them. + type_wrapper = CodeRust( + "pub unsafe fn c_to_r(_cs: *const MyStruct) -> () { todo!() }\n" + "pub unsafe fn r_to_c(_rs: &(), _cs: *mut MyStruct) { todo!() }\n" + "#[cfg(test)]\nmod tests {\n" + " #[test]\n fn round_trip_zeroed() { todo!() }\n" + " #[test]\n fn round_trip_nontrivial() { todo!() }\n}\n" + ) + mock_wrapper_gen = MagicMock(return_value=dspy.Prediction(wrapper=type_wrapper)) + symbol_wrapper = WrapperGenerator( + wrapper=MagicMock(return_value=mock_wrapper_gen), # type: ignore[arg-type] + max_iters=1, + ) + + translator, hybrid_lib, rs_lib, sys_c = _make_translator( + tmp_path, + symbol_translator=symbol_translator, + symbol_wrapper=symbol_wrapper, + tests="dummy_test", + max_iters=1, + ) + crate_mock: MagicMock = translator.crate # type: ignore[assignment] + crate_mock.cargo_test.return_value = (True, "", "", "") + + # Parse real C so the struct/typedef symbols carry genuine kind and linkage flags + tu = ast.create_translation_unit(c_source) + tree = ast.extract_info_c(tu) + symbols, dependencies = get_symbols_and_dependencies([tree]) + + sys_c.write_text(str(c_source)) + + with patch( + "ideas.translate_recurrent.generate_unimplemented_type_wrapper", + return_value=type_wrapper, + ): + pred = translator(symbols=symbols, dependencies=dependencies) + assert pred.success + + # The struct definition is wrappable, so its wrapper module must be on disk + wrap_struct = hybrid_lib.parent / f"{wrapper_name}.rs" + assert wrap_struct.read_text() == str(type_wrapper) + + # ... and registered in the hybrid crate root exactly once + assert hybrid_lib.read_text().count(f"pub mod {wrapper_name};") == 1 + + # The typedef alias shares the struct's code, so it must not produce a second wrapper + assert list(hybrid_lib.parent.glob("wrap_*.rs")) == [wrap_struct] + assert mock_wrapper_gen.call_count == 1 + + # Wrapper must not bleed into the -rs crate + assert f"pub mod {wrapper_name};" not in rs_lib.read_text() + + +# Both symbols are global definitions and reach `_wrap_type`, but neither is a +# STRUCT_DECL, so there is no field-by-field `c_to_r`/`r_to_c` pair to generate. +_UNION_TYPEDEF = ast.CodeC("typedef union { int x; } u_t;") + +# The STRUCT_DECL here is only a forward declaration (no fields to convert) and the +# typedef merely aliases it, so neither symbol is wrappable. +_STRUCT_ALIAS_TYPEDEF = ast.CodeC("typedef struct Foo foo_alias_t;") + + +@pytest.mark.parametrize( + "c_source", + [_UNION_TYPEDEF, _STRUCT_ALIAS_TYPEDEF], + ids=["union_typedef", "struct_alias_typedef"], +) +def test_no_type_wrapper_for_non_struct_definition(tmp_path: Path, c_source: CodeC) -> None: + mock_translator = MagicMock( + return_value=dspy.Prediction(translation=CodeRust("pub struct Placeholder;")) + ) + symbol_translator = SnippetTranslator( + translator=MagicMock(return_value=mock_translator), # type: ignore[arg-type] + max_iters=1, + ) + + mock_wrapper_gen = MagicMock() + symbol_wrapper = WrapperGenerator( + wrapper=MagicMock(return_value=mock_wrapper_gen), # type: ignore[arg-type] + max_iters=1, + ) + + translator, hybrid_lib, rs_lib, sys_c = _make_translator( + tmp_path, + symbol_translator=symbol_translator, + symbol_wrapper=symbol_wrapper, + tests="dummy_test", + max_iters=1, + ) + crate_mock: MagicMock = translator.crate # type: ignore[assignment] + crate_mock.cargo_test.return_value = (True, "", "", "") + + tu = ast.create_translation_unit(c_source) + tree = ast.extract_info_c(tu) + symbols, dependencies = get_symbols_and_dependencies([tree]) + + sys_c.write_text(str(c_source)) + + mock_unimplemented = MagicMock() + with patch( + "ideas.translate_recurrent.generate_unimplemented_type_wrapper", mock_unimplemented + ): + pred = translator(symbols=symbols, dependencies=dependencies) + + # Translation still succeeds; only wrapping is skipped + assert pred.success + + # `_wrap_type` must bail out before seeding a template or invoking the generator + assert mock_unimplemented.call_count == 0 + assert mock_wrapper_gen.call_count == 0 + assert list(hybrid_lib.parent.glob("wrap_*.rs")) == [] + assert "pub mod wrap_" not in hybrid_lib.read_text() + assert "pub mod wrap_" not in rs_lib.read_text() diff --git a/test/test_wrapper.py b/test/test_wrapper.py index e777d4a..6afe150 100644 --- a/test/test_wrapper.py +++ b/test/test_wrapper.py @@ -9,6 +9,7 @@ import pytest +from ideas import translate_recurrent as translate_recurrent_mod from ideas import wrapper as wrapper_mod @@ -146,7 +147,7 @@ def test_bindgen_emits_expected_text_for_global_shapes( c_path = tmp_path / "input.c" c_path.write_text(source) - binding = wrapper_mod.bindgen(c_path, symbol) + binding = translate_recurrent_mod.bindgen(c_path, symbol) assert str(binding).strip() == expected assert c_path.read_text() == source @@ -160,7 +161,9 @@ def test_bindgen_restores_source_when_bindgen_fails( c_path.write_text(original_src) monkeypatch.setattr( - wrapper_mod, "run_subprocess", lambda *_args, **_kwargs: (False, "", "boom", 1) + wrapper_mod, + "run_subprocess", + lambda *_args, **_kwargs: (False, "", "boom", 1), ) with pytest.raises(ValueError, match="Bindgen failed"): @@ -174,7 +177,9 @@ def test_bindgen_raises_for_empty_binding(tmp_path: Path, monkeypatch: pytest.Mo c_path.write_text("int foo(void) { return 1; }\n") monkeypatch.setattr( - wrapper_mod, "run_subprocess", lambda *_args, **_kwargs: (True, " \n", "", 0) + wrapper_mod, + "run_subprocess", + lambda *_args, **_kwargs: (True, " \n", "", 0), ) with pytest.raises(ValueError, match="empty binding"): @@ -187,11 +192,11 @@ def test_bindgen_handles_dependent_declarations_for_target_global(tmp_path: Path dependent_decl = "static const int num_arr = sizeof(arr) / sizeof(arr[0]);\n" c_path.write_text(array_decl) - baseline_binding = wrapper_mod.bindgen(c_path, "arr") + baseline_binding = translate_recurrent_mod.bindgen(c_path, "arr") assert c_path.read_text() == array_decl c_path.write_text(array_decl + dependent_decl) - dependent_binding = wrapper_mod.bindgen(c_path, "arr") + dependent_binding = translate_recurrent_mod.bindgen(c_path, "arr") assert c_path.read_text() == array_decl + dependent_decl assert str(dependent_binding).strip() == str(baseline_binding).strip() @@ -203,11 +208,11 @@ def test_bindgen_handles_dependent_declarations_for_target_function(tmp_path: Pa dependent_source = baseline_source + "int (*pf)(int) = f;\n" c_path.write_text(baseline_source) - baseline_binding = wrapper_mod.bindgen(c_path, "f") + baseline_binding = translate_recurrent_mod.bindgen(c_path, "f") assert c_path.read_text() == baseline_source c_path.write_text(dependent_source) - dependent_binding = wrapper_mod.bindgen(c_path, "f") + dependent_binding = translate_recurrent_mod.bindgen(c_path, "f") assert c_path.read_text() == dependent_source assert str(dependent_binding).strip() == str(baseline_binding).strip() diff --git a/uv.lock b/uv.lock index 9a86732..270ba87 100644 --- a/uv.lock +++ b/uv.lock @@ -85,15 +85,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/ba/88/6237e97e3385b57b5f1528647addea5cc03d4d65d5979ab24327d41fb00d/alembic-1.17.2-py3-none-any.whl", hash = "sha256:f483dd1fe93f6c5d49217055e4d15b905b425b6af906746abb35b69c1996c4e6", size = 248554, upload-time = "2025-11-14T20:35:05.699Z" }, ] -[[package]] -name = "annotated-doc" -version = "0.0.4" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/57/ba/046ceea27344560984e26a590f90bc7f4a75b06701f653222458922b558c/annotated_doc-0.0.4.tar.gz", hash = "sha256:fbcda96e87e9c92ad167c2e53839e57503ecfda18804ea28102353485033faa4", size = 7288, upload-time = "2025-11-10T22:07:42.062Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/d3/26bf1008eb3d2daa8ef4cacc7f3bfdc11818d111f7e2d0201bc6e3b49d45/annotated_doc-0.0.4-py3-none-any.whl", hash = "sha256:571ac1dc6991c450b25a9c2d84a3705e2ae7a53467b5d111c24fa8baabbed320", size = 5303, upload-time = "2025-11-10T22:07:40.673Z" }, -] - [[package]] name = "annotated-types" version = "0.7.0" @@ -173,15 +164,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d5/dc/180fe721a2574fb3aad4051adcca196ac2d18adaf75122f5eeb47436cca2/basedpyright-1.29.4-py3-none-any.whl", hash = "sha256:e087513979972f83010639c6c1a1c13dd3b1d24ee45f8ecff747962cc2063d6f", size = 11476859, upload-time = "2025-06-11T22:25:52.01Z" }, ] -[[package]] -name = "blinker" -version = "1.9.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/21/28/9b3f50ce0e048515135495f198351908d99540d69bfdc8c1d15b73dc55ce/blinker-1.9.0.tar.gz", hash = "sha256:b4ce2265a7abece45e7cc896e98dbebe6cead56bcf805a3d23136d145f5445bf", size = 22460, upload-time = "2024-11-08T17:25:47.436Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/10/cb/f2ad4230dc2eb1a74edf38f1a38b9b52277f75bef262d8908e60d957e13c/blinker-1.9.0-py3-none-any.whl", hash = "sha256:ba0efaa9080b619ff2f3459d1d500c57bddea4a6b424b60a91141db6fd2f08bc", size = 8458, upload-time = "2024-11-08T17:25:46.184Z" }, -] - [[package]] name = "cachetools" version = "6.2.4" @@ -266,24 +248,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/50/a4/608c542925949b300a295baa422b568f835044c5e3ad20820676b840228a/clang-21.1.7-py3-none-any.whl", hash = "sha256:23ee8f7b62af648009aee5139516b2a2a9320680dbce6e42a53e48bd5e8983ea", size = 40240, upload-time = "2025-12-18T22:04:50.636Z" }, ] -[[package]] -name = "claude-agent-sdk" -version = "0.1.80" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "mcp" }, - { name = "sniffio" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/d1/06/0984d8bc2f0f7b05aca2005461d587f9d04d8009fc4a2d333dec1c2f3164/claude_agent_sdk-0.1.80.tar.gz", hash = "sha256:1938d376cd6db273583266b184fc9caf53779841f131bf3fe308014707536019", size = 250299, upload-time = "2026-05-09T06:44:58.202Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/84/4d/fc78dae356a43126d0142921a73254f371b359b0508d5af046c43bc680bf/claude_agent_sdk-0.1.80-py3-none-macosx_11_0_arm64.whl", hash = "sha256:0a26cfea92029f1e3bcc468657e9bbb464a7bf04519b528ec0182ede3415a311", size = 60909658, upload-time = "2026-05-09T06:45:01.651Z" }, - { url = "https://files.pythonhosted.org/packages/aa/08/586c98a59d30bea43d83a9db7f8468a24affd2f7d3721a0dd010bf4784c8/claude_agent_sdk-0.1.80-py3-none-macosx_11_0_x86_64.whl", hash = "sha256:09f025305524e909c8ee190e73ad319b0d14f2c5d2d1ec567995c1eb833f4de7", size = 62949213, upload-time = "2026-05-09T06:45:04.848Z" }, - { url = "https://files.pythonhosted.org/packages/2f/a8/e7825005610e711fdebcc5c82c5de2214bb967f1cf5a14edd50ef16e0bc0/claude_agent_sdk-0.1.80-py3-none-manylinux_2_17_aarch64.whl", hash = "sha256:be269e118cce52b638b17232f2a52ce4d0877218672261d987cc20b4e5d9c83a", size = 70625763, upload-time = "2026-05-09T06:45:07.899Z" }, - { url = "https://files.pythonhosted.org/packages/fb/dd/a754eed2ab4f8437aac52d4d321e28c4d8bfd6ca126b5179b441aa7aeadf/claude_agent_sdk-0.1.80-py3-none-manylinux_2_17_x86_64.whl", hash = "sha256:653fb53600777c253885f9536c17da19d12f9d7fedd5e419c522854f1089449a", size = 70806172, upload-time = "2026-05-09T06:45:11.51Z" }, - { url = "https://files.pythonhosted.org/packages/6e/a8/3b27d7aa434b471a3100d158c7e709d09e7be0179a6c34be27def3ffaa1f/claude_agent_sdk-0.1.80-py3-none-win_amd64.whl", hash = "sha256:51ecfc32257201fc2cb6c061ba4d78e27b789a736fd5ed1e6ec0af60fd5d81aa", size = 71422151, upload-time = "2026-05-09T06:45:15.043Z" }, -] - [[package]] name = "click" version = "8.1.8" @@ -365,64 +329,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/ff/371ea7d252656ee1eb6d83eeeef3d1d0c6baf1d6497687d081ea03814670/cryptography-48.0.1-cp39-abi3-win_amd64.whl", hash = "sha256:9a49ca6c81417f6a5edb50375a60cccdd70fa0a91a5211829dbea74eba94d2ac", size = 3793408, upload-time = "2026-06-09T22:32:15.191Z" }, ] -[[package]] -name = "datasets" -version = "4.8.5" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "dill" }, - { name = "filelock" }, - { name = "fsspec", extra = ["http"] }, - { name = "httpx" }, - { name = "huggingface-hub" }, - { name = "multiprocess" }, - { name = "numpy" }, - { name = "packaging" }, - { name = "pandas" }, - { name = "pyarrow" }, - { name = "pyyaml" }, - { name = "requests" }, - { name = "tqdm" }, - { name = "xxhash" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/66/34/14cd8e76f907f7d4dca2334cfeec9f81d30fd15c25a015f99aaea694eaed/datasets-4.8.5.tar.gz", hash = "sha256:0f0c1c3d56ffff2c93b2f4c63c95bac94f3d7e8621aea2a2a576275233bba772", size = 605649, upload-time = "2026-04-27T15:43:57.384Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/65/99/00f3196036501b53032c4b1ab8337a0b978dee832ed276dae3815df4e8b5/datasets-4.8.5-py3-none-any.whl", hash = "sha256:5079900781719c0e063a8efdd2cd95a31ad0c63209178669cd23cf1b926149ff", size = 528973, upload-time = "2026-04-27T15:43:53.702Z" }, -] - -[[package]] -name = "deprecation" -version = "2.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "packaging" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5a/d3/8ae2869247df154b64c1884d7346d412fed0c49df84db635aab2d1c40e62/deprecation-2.1.0.tar.gz", hash = "sha256:72b3bde64e5d778694b0cf68178aed03d15e15477116add3fb773e581f9518ff", size = 173788, upload-time = "2020-04-20T14:23:38.738Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/02/c3/253a89ee03fc9b9682f1541728eb66db7db22148cd94f89ab22528cd1e1b/deprecation-2.1.0-py2.py3-none-any.whl", hash = "sha256:a10811591210e1fb0e768a8c25517cabeabcba6f0bf96564f8ff45189f90b14a", size = 11178, upload-time = "2020-04-20T14:23:36.581Z" }, -] - -[[package]] -name = "dill" -version = "0.4.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/81/e1/56027a71e31b02ddc53c7d65b01e68edf64dea2932122fe7746a516f75d5/dill-0.4.1.tar.gz", hash = "sha256:423092df4182177d4d8ba8290c8a5b640c66ab35ec7da59ccfa00f6fa3eea5fa", size = 187315, upload-time = "2026-01-19T02:36:56.85Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/1e/77/dc8c558f7593132cf8fefec57c4f60c83b16941c574ac5f619abb3ae7933/dill-0.4.1-py3-none-any.whl", hash = "sha256:1e1ce33e978ae97fcfcff5638477032b801c46c7c65cf717f95fbc2248f79a9d", size = 120019, upload-time = "2026-01-19T02:36:55.663Z" }, -] - -[[package]] -name = "dirhash" -version = "0.5.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "scantree" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1d/70/49f93897f3a4f7ab5f20a854ebc91aad47854e9fb2cd169e3a4452fa3f5e/dirhash-0.5.0.tar.gz", hash = "sha256:e60760f0ab2e935d8cb088923ea2c6492398dca42cec785df778985fd4cd5386", size = 21377, upload-time = "2024-08-03T22:14:13.322Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5e/1f/c8bf92552b7f0a13b9f12b85e3de8df6d9814240e0f8ce8f37433df028b3/dirhash-0.5.0-py3-none-any.whl", hash = "sha256:523dfd6b058c64f45b31604376926c6e2bd2ea301d0df23095d4055674e38b09", size = 13119, upload-time = "2024-08-03T22:14:11.688Z" }, -] - [[package]] name = "diskcache" version = "5.6.3" @@ -450,20 +356,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/12/b3/231ffd4ab1fc9d679809f356cebee130ac7daa00d6d6f3206dd4fd137e9e/distro-1.9.0-py3-none-any.whl", hash = "sha256:7bffd925d65168f85027d8da9af6bddab658135b840670a223589bc0c8ef02b2", size = 20277, upload-time = "2023-12-24T09:54:30.421Z" }, ] -[[package]] -name = "docker" -version = "7.1.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pywin32", marker = "sys_platform == 'win32'" }, - { name = "requests" }, - { name = "urllib3" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/91/9b/4a2ea29aeba62471211598dac5d96825bb49348fa07e906ea930394a83ce/docker-7.1.0.tar.gz", hash = "sha256:ad8c70e6e3f8926cb8a92619b832b4ea5299e2831c14284663184e200546fa6c", size = 117834, upload-time = "2024-05-23T11:13:57.216Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e3/26/57c6fb270950d476074c087527a558ccb6f4436657314bfb6cdf484114c4/docker-7.1.0-py3-none-any.whl", hash = "sha256:c96b93b7f0a746f9e77d325bcfb87422a3d8bd4f03136ae8a85b37f1898d5fc0", size = 147774, upload-time = "2024-05-23T11:13:55.01Z" }, -] - [[package]] name = "docstring-parser" version = "0.17.0" @@ -502,22 +394,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/91/18/1c93641f25f4e76772b4ffb52a4e289f1706d6bda4f3b59bb6f7c339df46/dspy-3.1.2-py3-none-any.whl", hash = "sha256:23b98bf5abeda260722c445d397d07ea27488c204b8c0ccd6d3e607c4b41bc6b", size = 312290, upload-time = "2026-01-19T14:21:45.776Z" }, ] -[[package]] -name = "fastapi" -version = "0.136.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "annotated-doc" }, - { name = "pydantic" }, - { name = "starlette" }, - { name = "typing-extensions" }, - { name = "typing-inspection" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5d/45/c130091c2dfa061bbfe3150f2a5091ef1adf149f2a8d2ae769ecaf6e99a2/fastapi-0.136.1.tar.gz", hash = "sha256:7af665ad7acfa0a3baf8983d393b6b471b9da10ede59c60045f49fbc89a0fa7f", size = 397448, upload-time = "2026-04-23T16:49:44.046Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5a/ff/2e4eca3ade2c22fe1dea7043b8ee9dabe47753349eb1b56a202de8af6349/fastapi-0.136.1-py3-none-any.whl", hash = "sha256:a6e9d7eeada96c93a4d69cb03836b44fa34e2854accb7244a1ece36cd4781c3f", size = 117683, upload-time = "2026-04-23T16:49:42.437Z" }, -] - [[package]] name = "fastuuid" version = "0.14.0" @@ -546,23 +422,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b5/36/7fb70f04bf00bc646cd5bb45aa9eddb15e19437a28b8fb2b4a5249fac770/filelock-3.20.3-py3-none-any.whl", hash = "sha256:4b0dda527ee31078689fc205ec4f1c1bf7d56cf88b6dc9426c4f230e46c2dce1", size = 16701, upload-time = "2026-01-09T17:55:04.334Z" }, ] -[[package]] -name = "flask" -version = "3.1.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "blinker" }, - { name = "click" }, - { name = "itsdangerous" }, - { name = "jinja2" }, - { name = "markupsafe" }, - { name = "werkzeug" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/26/00/35d85dcce6c57fdc871f3867d465d780f302a175ea360f62533f12b27e2b/flask-3.1.3.tar.gz", hash = "sha256:0ef0e52b8a9cd932855379197dd8f94047b359ca0a78695144304cb45f87c9eb", size = 759004, upload-time = "2026-02-19T05:00:57.678Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7f/9c/34f6962f9b9e9c71f6e5ed806e0d0ff03c9d1b0b2340088a0cf4bce09b18/flask-3.1.3-py3-none-any.whl", hash = "sha256:f4bcbefc124291925f1a26446da31a5178f9483862233b23c0c96a20701f670c", size = 103424, upload-time = "2026-02-19T05:00:56.027Z" }, -] - [[package]] name = "frozenlist" version = "1.8.0" @@ -613,11 +472,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/51/c7/b64cae5dba3a1b138d7123ec36bb5ccd39d39939f18454407e5468f4763f/fsspec-2025.12.0-py3-none-any.whl", hash = "sha256:8bf1fe301b7d8acfa6e8571e3b1c3d158f909666642431cc78a1b7b4dbc5ec5b", size = 201422, upload-time = "2025-12-03T15:23:41.434Z" }, ] -[package.optional-dependencies] -http = [ - { name = "aiohttp" }, -] - [[package]] name = "gepa" version = "0.0.24" @@ -627,38 +481,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/b1/33b035ff1aaf22d4e104c5b15ba48fe5050639764457048e967c20d6317a/gepa-0.0.24-py3-none-any.whl", hash = "sha256:6d8b16699e7b24ed01435dea7bbbc89156a88cbb4b877b14d90e7455db2b0032", size = 137539, upload-time = "2026-01-05T16:45:29.244Z" }, ] -[[package]] -name = "google-api-core" -version = "2.30.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-auth" }, - { name = "googleapis-common-protos" }, - { name = "proto-plus" }, - { name = "protobuf" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/16/ce/502a57fb0ec752026d24df1280b162294b22a0afb98a326084f9a979138b/google_api_core-2.30.3.tar.gz", hash = "sha256:e601a37f148585319b26db36e219df68c5d07b6382cff2d580e83404e44d641b", size = 177001, upload-time = "2026-04-10T00:41:28.035Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/03/15/e56f351cf6ef1cfea58e6ac226a7318ed1deb2218c4b3cc9bd9e4b786c5a/google_api_core-2.30.3-py3-none-any.whl", hash = "sha256:a85761ba72c444dad5d611c2220633480b2b6be2521eca69cca2dbb3ffd6bfe8", size = 173274, upload-time = "2026-04-09T22:57:16.198Z" }, -] - -[[package]] -name = "google-api-python-client" -version = "2.196.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-api-core" }, - { name = "google-auth" }, - { name = "google-auth-httplib2" }, - { name = "httplib2" }, - { name = "uritemplate" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/6d/f3/34ef8aca7909675fe327f96c1ed927f0520e7acf68af19157e96acc05e76/google_api_python_client-2.196.0.tar.gz", hash = "sha256:9f335d38f6caaa2747bcf64335ed1a9a19047d53e86538eda6a1b17d37f1743d", size = 14628129, upload-time = "2026-05-06T23:47:35.655Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/99/c7/1817b4edf966d5afcac1c0781ca36d621bc0cb58104c4e7c2a475ab185f7/google_api_python_client-2.196.0-py3-none-any.whl", hash = "sha256:2591e9b47dcb17e4e62a09370aaee3bcf323af8f28ccecdabcd0a42a23ca4db5", size = 15206663, upload-time = "2026-05-06T23:47:32.886Z" }, -] - [[package]] name = "google-auth" version = "2.49.1" @@ -677,32 +499,6 @@ requests = [ { name = "requests" }, ] -[[package]] -name = "google-auth-httplib2" -version = "0.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-auth" }, - { name = "httplib2" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1c/b3/f192c8bc7e41e0ebdbd95afcae4783417a34b6a6af62d22daf22c3fd38fc/google_auth_httplib2-0.4.0.tar.gz", hash = "sha256:d5b030a204b7a4b4d553ba9ca701b62481ee2b74419325580be70f7d85ffed35", size = 11161, upload-time = "2026-05-07T08:03:46.878Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/97/be/954c35a62b9e31de66b0a43c225c9b6bb9e0f98d6b1dc110a2308e3644f5/google_auth_httplib2-0.4.0-py3-none-any.whl", hash = "sha256:8e55cfafa3358cba85f6cad4a886138e88e158d71e7e5c9ee5936a5c1507fb91", size = 9529, upload-time = "2026-05-07T08:02:12.375Z" }, -] - -[[package]] -name = "google-auth-oauthlib" -version = "1.4.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "google-auth" }, - { name = "requests-oauthlib" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/70/18/90c7fac516e63cf2058166fce0c88c353647c677b51cc036c09c49bb5cbb/google_auth_oauthlib-1.4.0.tar.gz", hash = "sha256:18b5e28880eb8eba9065c436becdc0ee8e4b59117a73a510679c82f70cd363d2", size = 21675, upload-time = "2026-05-07T08:03:47.816Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/37/d3/d7dff0d58a9e9244b48044bfb6a898bfcc8ecc42e0031d1bebc695344725/google_auth_oauthlib-1.4.0-py3-none-any.whl", hash = "sha256:251314f213a9ee46a5ae73988e84fd7cca8bb68e7ecf4bfd45940f9e7f51d070", size = 19261, upload-time = "2026-05-07T08:02:13.798Z" }, -] - [[package]] name = "google-genai" version = "1.70.0" @@ -724,18 +520,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/36/a3/d4564c8a9beaf6a3cef8d70fa6354318572cebfee65db4f01af0d41f45ba/google_genai-1.70.0-py3-none-any.whl", hash = "sha256:b74c24549d8b4208f4c736fd11857374788e1ffffc725de45d706e35c97fceee", size = 760584, upload-time = "2026-04-01T10:52:44.349Z" }, ] -[[package]] -name = "googleapis-common-protos" -version = "1.75.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "protobuf" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b5/c8/f439cffde755cffa462bfbb156278fa6f9d09119719af9814b858fd4f81f/googleapis_common_protos-1.75.0.tar.gz", hash = "sha256:53a062ff3c32552fbd62c11fe23768b78e4ddf0494d5e5fd97d3f4689c75fbbd", size = 151035, upload-time = "2026-05-07T08:04:49.423Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e7/c8/e2645aa8ed02fd4c7a2f59d68783b65b1f3cbdfe39a6308e156509d1fee8/googleapis_common_protos-1.75.0-py3-none-any.whl", hash = "sha256:961ed60399c457ceb0ee8f285a84c870aabc9c6a832b9d37bb281b5bebde43ed", size = 300631, upload-time = "2026-05-07T08:03:30.345Z" }, -] - [[package]] name = "greenlet" version = "3.3.0" @@ -745,7 +529,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/02/2f/28592176381b9ab2cafa12829ba7b472d177f3acc35d8fbcf3673d966fff/greenlet-3.3.0-cp313-cp313-macosx_11_0_universal2.whl", hash = "sha256:a1e41a81c7e2825822f4e068c48cb2196002362619e2d70b148f20a831c00739", size = 275140, upload-time = "2025-12-04T14:23:01.282Z" }, { url = "https://files.pythonhosted.org/packages/2c/80/fbe937bf81e9fca98c981fe499e59a3f45df2a04da0baa5c2be0dca0d329/greenlet-3.3.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:9f515a47d02da4d30caaa85b69474cec77b7929b2e936ff7fb853d42f4bf8808", size = 599219, upload-time = "2025-12-04T14:50:08.309Z" }, { url = "https://files.pythonhosted.org/packages/c2/ff/7c985128f0514271b8268476af89aee6866df5eec04ac17dcfbc676213df/greenlet-3.3.0-cp313-cp313-manylinux_2_24_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:7d2d9fd66bfadf230b385fdc90426fcd6eb64db54b40c495b72ac0feb5766c54", size = 610211, upload-time = "2025-12-04T14:57:43.968Z" }, - { url = "https://files.pythonhosted.org/packages/79/07/c47a82d881319ec18a4510bb30463ed6891f2ad2c1901ed5ec23d3de351f/greenlet-3.3.0-cp313-cp313-manylinux_2_24_s390x.manylinux_2_28_s390x.whl", hash = "sha256:30a6e28487a790417d036088b3bcb3f3ac7d8babaa7d0139edbaddebf3af9492", size = 624311, upload-time = "2025-12-04T15:07:14.697Z" }, { url = "https://files.pythonhosted.org/packages/fd/8e/424b8c6e78bd9837d14ff7df01a9829fc883ba2ab4ea787d4f848435f23f/greenlet-3.3.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:087ea5e004437321508a8d6f20efc4cfec5e3c30118e1417ea96ed1d93950527", size = 612833, upload-time = "2025-12-04T14:26:03.669Z" }, { url = "https://files.pythonhosted.org/packages/b5/ba/56699ff9b7c76ca12f1cdc27a886d0f81f2189c3455ff9f65246780f713d/greenlet-3.3.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ab97cf74045343f6c60a39913fa59710e4bd26a536ce7ab2397adf8b27e67c39", size = 1567256, upload-time = "2025-12-04T15:04:25.276Z" }, { url = "https://files.pythonhosted.org/packages/1e/37/f31136132967982d698c71a281a8901daf1a8fbab935dce7c0cf15f942cc/greenlet-3.3.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:5375d2e23184629112ca1ea89a53389dddbffcf417dad40125713d88eb5f96e8", size = 1636483, upload-time = "2025-12-04T14:27:30.804Z" }, @@ -761,51 +544,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/04/4b/29cac41a4d98d144bf5f6d33995617b185d14b22401f75ca86f384e87ff1/h11-0.16.0-py3-none-any.whl", hash = "sha256:63cf8bbe7522de3bf65932fda1d9c2772064ffb3dae62d55932da54b31cb6c86", size = 37515, upload-time = "2025-04-24T03:35:24.344Z" }, ] -[[package]] -name = "h2" -version = "4.3.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "hpack" }, - { name = "hyperframe" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/1d/17/afa56379f94ad0fe8defd37d6eb3f89a25404ffc71d4d848893d270325fc/h2-4.3.0.tar.gz", hash = "sha256:6c59efe4323fa18b47a632221a1888bd7fde6249819beda254aeca909f221bf1", size = 2152026, upload-time = "2025-08-23T18:12:19.778Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/69/b2/119f6e6dcbd96f9069ce9a2665e0146588dc9f88f29549711853645e736a/h2-4.3.0-py3-none-any.whl", hash = "sha256:c438f029a25f7945c69e0ccf0fb951dc3f73a5f6412981daee861431b70e2bdd", size = 61779, upload-time = "2025-08-23T18:12:17.779Z" }, -] - -[[package]] -name = "harbor" -version = "0.6.6" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "claude-agent-sdk" }, - { name = "datasets" }, - { name = "dirhash" }, - { name = "fastapi" }, - { name = "httpx" }, - { name = "jinja2" }, - { name = "litellm" }, - { name = "packaging" }, - { name = "pathspec" }, - { name = "pydantic" }, - { name = "python-dotenv" }, - { name = "pyyaml" }, - { name = "requests" }, - { name = "rich" }, - { name = "ruff" }, - { name = "shortuuid" }, - { name = "supabase" }, - { name = "tenacity" }, - { name = "toml" }, - { name = "typer" }, - { name = "uvicorn" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ed/05/3290ec39da674c3512a289b0f374dda7d016825f204e4815d6f6b7482e33/harbor-0.6.6.tar.gz", hash = "sha256:5653feb22ff4364fd87447d062cc4f9ab99c9b91d02c0c518cb330fafc2abf03", size = 986277, upload-time = "2026-05-07T19:23:39.099Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/0a/69/4ea696ff5cd29de03d784ed4348fefb843b19d5a2d67ba269a2c90cbedea/harbor-0.6.6-py3-none-any.whl", hash = "sha256:30477bf698d6853d6c4bb76d85aec85e1e95190ad3a6e4bf4730435926007652", size = 1120578, upload-time = "2026-05-07T19:23:40.757Z" }, -] - [[package]] name = "hf-xet" version = "1.2.0" @@ -828,15 +566,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/44/870d44b30e1dcfb6a65932e3e1506c103a8a5aea9103c337e7a53180322c/hf_xet-1.2.0-cp37-abi3-win_amd64.whl", hash = "sha256:e6584a52253f72c9f52f9e549d5895ca7a471608495c4ecaa6cc73dba2b24d69", size = 2905735, upload-time = "2025-10-24T19:04:35.928Z" }, ] -[[package]] -name = "hpack" -version = "4.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/2c/48/71de9ed269fdae9c8057e5a4c0aa7402e8bb16f2c6e90b3aa53327b113f8/hpack-4.1.0.tar.gz", hash = "sha256:ec5eca154f7056aa06f196a557655c5b009b382873ac8d1e66e79e87535f1dca", size = 51276, upload-time = "2025-01-22T21:44:58.347Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/07/c6/80c95b1b2b94682a72cbdbfb85b81ae2daffa4291fbfa1b1464502ede10d/hpack-4.1.0-py3-none-any.whl", hash = "sha256:157ac792668d995c657d93111f46b4535ed114f0c9c8d672271bbec7eae1b496", size = 34357, upload-time = "2025-01-22T21:44:56.92Z" }, -] - [[package]] name = "httpcore" version = "1.0.9" @@ -850,18 +579,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7e/f5/f66802a942d491edb555dd61e3a9961140fd64c90bce1eafd741609d334d/httpcore-1.0.9-py3-none-any.whl", hash = "sha256:2d400746a40668fc9dec9810239072b40b4484b640a8c38fd654a024c7a1bf55", size = 78784, upload-time = "2025-04-24T22:06:20.566Z" }, ] -[[package]] -name = "httplib2" -version = "0.31.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pyparsing" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c1/1f/e86365613582c027dda5ddb64e1010e57a3d53e99ab8a72093fa13d565ec/httplib2-0.31.2.tar.gz", hash = "sha256:385e0869d7397484f4eab426197a4c020b606edd43372492337c0b4010ae5d24", size = 250800, upload-time = "2026-01-23T11:04:44.165Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/2f/90/fd509079dfcab01102c0fdd87f3a9506894bc70afcf9e9785ef6b2b3aff6/httplib2-0.31.2-py3-none-any.whl", hash = "sha256:dbf0c2fa3862acf3c55c078ea9c0bc4481d7dc5117cae71be9514912cf9f8349", size = 91099, upload-time = "2026-01-23T11:04:42.78Z" }, -] - [[package]] name = "httpx" version = "0.28.1" @@ -877,20 +594,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, ] -[package.optional-dependencies] -http2 = [ - { name = "h2" }, -] - -[[package]] -name = "httpx-sse" -version = "0.4.3" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0f/4c/751061ffa58615a32c31b2d82e8482be8dd4a89154f003147acee90f2be9/httpx_sse-0.4.3.tar.gz", hash = "sha256:9b1ed0127459a66014aec3c56bebd93da3c1bc8bb6618c8082039a44889a755d", size = 15943, upload-time = "2025-10-10T21:48:22.271Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d2/fd/6668e5aec43ab844de6fc74927e155a3b37bf40d7c3790e49fc0406b6578/httpx_sse-0.4.3-py3-none-any.whl", hash = "sha256:0ac1c9fe3c0afad2e0ebb25a934a59f4c7823b60792691f779fad2c5568830fc", size = 8960, upload-time = "2025-10-10T21:48:21.158Z" }, -] - [[package]] name = "huggingface-hub" version = "1.2.4" @@ -926,15 +629,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c6/50/e0edd38dcd63fb26a8547f13d28f7a008bc4a3fd4eb4ff030673f22ad41a/hydra_core-1.3.2-py3-none-any.whl", hash = "sha256:fa0238a9e31df3373b35b0bfb672c34cc92718d21f81311d8996a16de1141d8b", size = 154547, upload-time = "2023-02-23T18:33:40.801Z" }, ] -[[package]] -name = "hyperframe" -version = "6.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/02/e7/94f8232d4a74cc99514c13a9f995811485a6903d48e5d952771ef6322e30/hyperframe-6.1.0.tar.gz", hash = "sha256:f630908a00854a7adeabd6382b43923a4c4cd4b821fcb527e6ab9e15382a3b08", size = 26566, upload-time = "2025-01-22T21:41:49.302Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/48/30/47d0bf6072f7252e6521f3447ccfa40b421b6824517f82854703d0f5a98b/hyperframe-6.1.0-py3-none-any.whl", hash = "sha256:b03380493a519fce58ea5af42e4a42317bf9bd425596f7a0835ffce80f1a42e5", size = 13007, upload-time = "2025-01-22T21:41:47.295Z" }, -] - [[package]] name = "ideas" version = "2026.3" @@ -943,7 +637,7 @@ dependencies = [ { name = "clang" }, { name = "dspy" }, { name = "hydra-core" }, - { name = "kiss-agent-framework" }, + { name = "kiss-agent-framework", extra = ["core"] }, { name = "networkx" }, { name = "tomlkit" }, { name = "tree-sitter" }, @@ -956,6 +650,7 @@ dev = [ { name = "pre-commit" }, { name = "pytest" }, { name = "ruff" }, + { name = "vulture" }, ] [package.metadata] @@ -963,7 +658,7 @@ requires-dist = [ { name = "clang", specifier = "==21.1.7" }, { name = "dspy", specifier = "==3.1.2" }, { name = "hydra-core", specifier = "==1.3.2" }, - { name = "kiss-agent-framework", specifier = "==2026.5.22" }, + { name = "kiss-agent-framework", extras = ["core"], git = "https://github.com/mariusarvinte/kiss_ai?rev=f4f6cc1ef1fc5e2962aec6da71677be6e057f194" }, { name = "networkx", specifier = "==3.6.1" }, { name = "tomlkit", specifier = ">=0.14.0" }, { name = "tree-sitter", specifier = "==0.25.2" }, @@ -976,6 +671,7 @@ dev = [ { name = "pre-commit", specifier = "==4.2.0" }, { name = "pytest", specifier = "==9.0.3" }, { name = "ruff", specifier = "==0.13.0" }, + { name = "vulture", specifier = "==2.16" }, ] [[package]] @@ -1017,15 +713,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/b1/3846dd7f199d53cb17f49cba7e651e9ce294d8497c8c150530ed11865bb8/iniconfig-2.3.0-py3-none-any.whl", hash = "sha256:f631c04d2c48c52b84d0d0549c99ff3859c98df65b3101406327ecc7d53fbf12", size = 7484, upload-time = "2025-10-18T21:55:41.639Z" }, ] -[[package]] -name = "itsdangerous" -version = "2.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9c/cb/8ac0172223afbccb63986cc25049b154ecfb5e85932587206f42317be31d/itsdangerous-2.2.0.tar.gz", hash = "sha256:e0050c0b7da1eea53ffaf149c0cfbb5c6e2e2b69c4bef22c81fa6eb73e5f6173", size = 54410, upload-time = "2024-04-16T21:28:15.614Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/04/96/92447566d16df59b2a776c0fb82dbc4d9e07cd95062562af01e408583fc4/itsdangerous-2.2.0-py3-none-any.whl", hash = "sha256:c6242fc49e35958c8b15141343aa660db5fc54d4f13a1db01a3f5891b98700ef", size = 16234, upload-time = "2024-04-16T21:28:14.499Z" }, -] - [[package]] name = "jinja2" version = "3.1.6" @@ -1102,30 +789,18 @@ wheels = [ [[package]] name = "kiss-agent-framework" -version = "2026.5.22" -source = { registry = "https://pypi.org/simple" } -dependencies = [ +version = "2026.7.23" +source = { git = "https://github.com/mariusarvinte/kiss_ai?rev=f4f6cc1ef1fc5e2962aec6da71677be6e057f194#f4f6cc1ef1fc5e2962aec6da71677be6e057f194" } + +[package.optional-dependencies] +core = [ { name = "anthropic" }, - { name = "cryptography" }, - { name = "docker" }, - { name = "flask" }, - { name = "google-api-python-client" }, - { name = "google-auth-oauthlib" }, { name = "google-genai" }, - { name = "harbor" }, { name = "openai" }, - { name = "playwright" }, { name = "pydantic" }, { name = "pydantic-settings" }, { name = "pyyaml" }, - { name = "requests" }, { name = "rich" }, - { name = "slack-sdk" }, - { name = "websockets" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/eb/f2/7d61c2d1f849f2638b7cb14b0cf5cc8e9c134d598f925fae62a7c1e6616d/kiss_agent_framework-2026.5.22.tar.gz", hash = "sha256:ac179c21fb1bbe8c020ddfe95ffcb32684ffd86396605d0b4435aa066584a5c7", size = 97834964, upload-time = "2026-05-07T22:42:02.092Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f1/d4/756db46fc33ff2c2dab1b58c939e8e5a835297d6447018f36f7ca8edf82d/kiss_agent_framework-2026.5.22-py3-none-any.whl", hash = "sha256:3a86b1738a8ad7f86fd86057cf6da61cceccaba21d1363ccf23609c7a96caba0", size = 3231386, upload-time = "2026-05-07T22:41:52.836Z" }, ] [[package]] @@ -1205,31 +880,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0e/72/e3cc540f351f316e9ed0f092757459afbc595824ca724cbc5a5d4263713f/markupsafe-3.0.3-cp313-cp313t-win_arm64.whl", hash = "sha256:ad2cf8aa28b8c020ab2fc8287b0f823d0a7d8630784c31e9ee5edea20f406287", size = 13973, upload-time = "2025-09-27T18:37:04.929Z" }, ] -[[package]] -name = "mcp" -version = "1.27.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "httpx" }, - { name = "httpx-sse" }, - { name = "jsonschema" }, - { name = "pydantic" }, - { name = "pydantic-settings" }, - { name = "pyjwt", extra = ["crypto"] }, - { name = "python-multipart" }, - { name = "pywin32", marker = "sys_platform == 'win32'" }, - { name = "sse-starlette" }, - { name = "starlette" }, - { name = "typing-extensions" }, - { name = "typing-inspection" }, - { name = "uvicorn", marker = "sys_platform != 'emscripten'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/38/83/d1efe7c2980d8a3afa476f4e3d42d53dd54c0ab94c27bee5d755b45c8b73/mcp-1.27.1.tar.gz", hash = "sha256:0f47e1820f8f8f941466b39749eb1d1839a04caddca2bc60e9d46e8a99914924", size = 608458, upload-time = "2026-05-08T16:50:12.601Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/fd/73/42d9596facebdb533b7f0b86c1b0364ef350d1f8ba78b1052e8a58b48b65/mcp-1.27.1-py3-none-any.whl", hash = "sha256:1af3c4203b329430fde7a87b4fcb6392a041f5cb851fd68fc674016ab4e7c06f", size = 216260, upload-time = "2026-05-08T16:50:10.547Z" }, -] - [[package]] name = "mdurl" version = "0.1.2" @@ -1239,35 +889,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b3/38/89ba8ad64ae25be8de66a6d463314cf1eb366222074cfda9ee839c56a4b4/mdurl-0.1.2-py3-none-any.whl", hash = "sha256:84008a41e51615a49fc9966191ff91509e3c40b939176e643fd50a5c2196b8f8", size = 9979, upload-time = "2022-08-14T12:40:09.779Z" }, ] -[[package]] -name = "mmh3" -version = "5.2.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/91/1a/edb23803a168f070ded7a3014c6d706f63b90c84ccc024f89d794a3b7a6d/mmh3-5.2.1.tar.gz", hash = "sha256:bbea5b775f0ac84945191fb83f845a6fd9a21a03ea7f2e187defac7e401616ad", size = 33775, upload-time = "2026-03-05T15:55:57.716Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/25/a5/9daa0508a1569a54130f6198d5462a92deda870043624aa3ea72721aa765/mmh3-5.2.1-cp313-cp313-android_21_arm64_v8a.whl", hash = "sha256:723b2681ed4cc07d3401bbea9c201ad4f2a4ca6ba8cddaff6789f715dd2b391e", size = 40832, upload-time = "2026-03-05T15:54:43.212Z" }, - { url = "https://files.pythonhosted.org/packages/0a/6b/3230c6d80c1f4b766dedf280a92c2241e99f87c1504ff74205ec8cebe451/mmh3-5.2.1-cp313-cp313-android_21_x86_64.whl", hash = "sha256:3619473a0e0d329fd4aec8075628f8f616be2da41605300696206d6f36920c3d", size = 41964, upload-time = "2026-03-05T15:54:44.204Z" }, - { url = "https://files.pythonhosted.org/packages/62/fb/648bfddb74a872004b6ee751551bfdda783fe6d70d2e9723bad84dbe5311/mmh3-5.2.1-cp313-cp313-ios_13_0_arm64_iphoneos.whl", hash = "sha256:e48d4dbe0f88e53081da605ae68644e5182752803bbc2beb228cca7f1c4454d6", size = 39114, upload-time = "2026-03-05T15:54:45.205Z" }, - { url = "https://files.pythonhosted.org/packages/95/c2/ab7901f87af438468b496728d11264cb397b3574d41506e71b92128e0373/mmh3-5.2.1-cp313-cp313-ios_13_0_arm64_iphonesimulator.whl", hash = "sha256:a482ac121de6973897c92c2f31defc6bafb11c83825109275cffce54bb64933f", size = 39819, upload-time = "2026-03-05T15:54:46.509Z" }, - { url = "https://files.pythonhosted.org/packages/2f/ed/6f88dda0df67de1612f2e130ffea34cf84aaee5bff5b0aff4dbff2babe34/mmh3-5.2.1-cp313-cp313-ios_13_0_x86_64_iphonesimulator.whl", hash = "sha256:17fbb47f0885ace8327ce1235d0416dc86a211dcd8cc1e703f41523be32cfec8", size = 40330, upload-time = "2026-03-05T15:54:47.864Z" }, - { url = "https://files.pythonhosted.org/packages/3d/66/7516d23f53cdf90f43fce24ab80c28f45e6851d78b46bef8c02084edf583/mmh3-5.2.1-cp313-cp313-macosx_10_13_universal2.whl", hash = "sha256:d51fde50a77f81330523562e3c2734ffdca9c4c9e9d355478117905e1cfe16c6", size = 56078, upload-time = "2026-03-05T15:54:48.9Z" }, - { url = "https://files.pythonhosted.org/packages/bc/34/4d152fdf4a91a132cb226b671f11c6b796eada9ab78080fb5ce1e95adaab/mmh3-5.2.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:19bbd3b841174ae6ed588536ab5e1b1fe83d046e668602c20266547298d939a9", size = 40498, upload-time = "2026-03-05T15:54:49.942Z" }, - { url = "https://files.pythonhosted.org/packages/d4/4c/8e3af1b6d85a299767ec97bd923f12b06267089c1472c27c1696870d1175/mmh3-5.2.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:be77c402d5e882b6fbacfd90823f13da8e0a69658405a39a569c6b58fdb17b03", size = 40033, upload-time = "2026-03-05T15:54:50.994Z" }, - { url = "https://files.pythonhosted.org/packages/8b/f2/966ea560e32578d453c9e9db53d602cbb1d0da27317e232afa7c38ceba11/mmh3-5.2.1-cp313-cp313-manylinux1_i686.manylinux_2_28_i686.manylinux_2_5_i686.whl", hash = "sha256:fd96476f04db5ceba1cfa0f21228f67c1f7402296f0e73fee3513aa680ad237b", size = 97320, upload-time = "2026-03-05T15:54:52.072Z" }, - { url = "https://files.pythonhosted.org/packages/bb/0d/2c5f9893b38aeb6b034d1a44ecd55a010148054f6a516abe53b5e4057297/mmh3-5.2.1-cp313-cp313-manylinux1_x86_64.manylinux_2_28_x86_64.manylinux_2_5_x86_64.whl", hash = "sha256:707151644085dd0f20fe4f4b573d28e5130c4aaa5f587e95b60989c5926653b5", size = 103299, upload-time = "2026-03-05T15:54:53.569Z" }, - { url = "https://files.pythonhosted.org/packages/1c/fc/2ebaef4a4d4376f89761274dc274035ffd96006ab496b4ee5af9b08f21a9/mmh3-5.2.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:3737303ca9ea0f7cb83028781148fcda4f1dac7821db0c47672971dabcf63593", size = 106222, upload-time = "2026-03-05T15:54:55.092Z" }, - { url = "https://files.pythonhosted.org/packages/57/09/ea7ffe126d0ba0406622602a2d05e1e1a6841cc92fc322eb576c95b27fad/mmh3-5.2.1-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.manylinux_2_28_ppc64le.whl", hash = "sha256:2778fed822d7db23ac5008b181441af0c869455b2e7d001f4019636ac31b6fe4", size = 113048, upload-time = "2026-03-05T15:54:56.305Z" }, - { url = "https://files.pythonhosted.org/packages/85/57/9447032edf93a64aa9bef4d9aa596400b1756f40411890f77a284f6293ca/mmh3-5.2.1-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.manylinux_2_28_s390x.whl", hash = "sha256:d57dea657357230cc780e13920d7fa7db059d58fe721c80020f94476da4ca0a1", size = 120742, upload-time = "2026-03-05T15:54:57.453Z" }, - { url = "https://files.pythonhosted.org/packages/53/82/a86cc87cc88c92e9e1a598fee509f0409435b57879a6129bf3b3e40513c7/mmh3-5.2.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:169e0d178cb59314456ab30772429a802b25d13227088085b0d49b9fe1533104", size = 99132, upload-time = "2026-03-05T15:54:58.583Z" }, - { url = "https://files.pythonhosted.org/packages/54/f7/6b16eb1b40ee89bb740698735574536bc20d6cdafc65ae702ea235578e05/mmh3-5.2.1-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:7e4e1f580033335c6f76d1e0d6b56baf009d1a64d6a4816347e4271ba951f46d", size = 98686, upload-time = "2026-03-05T15:55:00.078Z" }, - { url = "https://files.pythonhosted.org/packages/e8/88/a601e9f32ad1410f438a6d0544298ea621f989bd34a0731a7190f7dec799/mmh3-5.2.1-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:2bd9f19f7f1fcebd74e830f4af0f28adad4975d40d80620be19ffb2b2af56c9f", size = 106479, upload-time = "2026-03-05T15:55:01.532Z" }, - { url = "https://files.pythonhosted.org/packages/d6/5c/ce29ae3dfc4feec4007a437a1b7435fb9507532a25147602cd5b52be86db/mmh3-5.2.1-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:c88653877aeb514c089d1b3d473451677b8b9a6d1497dbddf1ae7934518b06d2", size = 110030, upload-time = "2026-03-05T15:55:02.934Z" }, - { url = "https://files.pythonhosted.org/packages/13/30/ae444ef2ff87c805d525da4fa63d27cda4fe8a48e77003a036b8461cfd5c/mmh3-5.2.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:fceef7fe67c81e1585198215e42ad3fdba3a25644beda8fbdaf85f4d7b93175a", size = 97536, upload-time = "2026-03-05T15:55:04.135Z" }, - { url = "https://files.pythonhosted.org/packages/4b/f9/dc3787ee5c813cc27fe79f45ad4500d9b5437f23a7402435cc34e07c7718/mmh3-5.2.1-cp313-cp313-win32.whl", hash = "sha256:54b64fb2433bc71488e7a449603bf8bd31fbcf9cb56fbe1eb6d459e90b86c37b", size = 40769, upload-time = "2026-03-05T15:55:05.277Z" }, - { url = "https://files.pythonhosted.org/packages/43/67/850e0b5a1e97799822ebfc4ca0e8c6ece3ed8baf7dcdf64de817dfdda2ca/mmh3-5.2.1-cp313-cp313-win_amd64.whl", hash = "sha256:cae6383181f1e345317742d2ddd88f9e7d2682fa4c9432e3a74e47d92dce0229", size = 41563, upload-time = "2026-03-05T15:55:06.283Z" }, - { url = "https://files.pythonhosted.org/packages/c0/cc/98c90b28e1da5458e19fbfaf4adb5289208d3bfccd45dd14eab216a2f0bb/mmh3-5.2.1-cp313-cp313-win_arm64.whl", hash = "sha256:022aa1a528604e6c83d0a7705fdef0b5355d897a9e0fa3a8d26709ceaa06965d", size = 39310, upload-time = "2026-03-05T15:55:07.323Z" }, -] - [[package]] name = "multidict" version = "6.7.0" @@ -1313,22 +934,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/b7/da/7d22601b625e241d4f23ef1ebff8acfc60da633c9e7e7922e24d10f592b3/multidict-6.7.0-py3-none-any.whl", hash = "sha256:394fc5c42a333c9ffc3e421a4c85e08580d990e08b99f6bf35b4132114c5dcb3", size = 12317, upload-time = "2025-10-06T14:52:29.272Z" }, ] -[[package]] -name = "multiprocess" -version = "0.70.19" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "dill" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/a2/f2/e783ac7f2aeeed14e9e12801f22529cc7e6b7ab80928d6dcce4e9f00922d/multiprocess-0.70.19.tar.gz", hash = "sha256:952021e0e6c55a4a9fe4cd787895b86e239a40e76802a789d6305398d3975897", size = 2079989, upload-time = "2026-01-19T06:47:39.744Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/e3/45/8004d1e6b9185c1a444d6b55ac5682acf9d98035e54386d967366035a03a/multiprocess-0.70.19-py310-none-any.whl", hash = "sha256:97404393419dcb2a8385910864eedf47a3cadf82c66345b44f036420eb0b5d87", size = 134948, upload-time = "2026-01-19T06:47:32.325Z" }, - { url = "https://files.pythonhosted.org/packages/86/c2/dec9722dc3474c164a0b6bcd9a7ed7da542c98af8cabce05374abab35edd/multiprocess-0.70.19-py311-none-any.whl", hash = "sha256:928851ae7973aea4ce0eaf330bbdafb2e01398a91518d5c8818802845564f45c", size = 144457, upload-time = "2026-01-19T06:47:33.711Z" }, - { url = "https://files.pythonhosted.org/packages/71/70/38998b950a97ea279e6bd657575d22d1a2047256caf707d9a10fbce4f065/multiprocess-0.70.19-py312-none-any.whl", hash = "sha256:3a56c0e85dd5025161bac5ce138dcac1e49174c7d8e74596537e729fd5c53c28", size = 150281, upload-time = "2026-01-19T06:47:35.037Z" }, - { url = "https://files.pythonhosted.org/packages/7f/74/d2c27e03cb84251dfe7249b8e82923643c6d48fa4883b9476b025e7dc7eb/multiprocess-0.70.19-py313-none-any.whl", hash = "sha256:8d5eb4ec5017ba2fab4e34a747c6d2c2b6fecfe9e7236e77988db91580ada952", size = 156414, upload-time = "2026-01-19T06:47:35.915Z" }, - { url = "https://files.pythonhosted.org/packages/7e/82/69e539c4c2027f1e1697e09aaa2449243085a0edf81ae2c6341e84d769b6/multiprocess-0.70.19-py39-none-any.whl", hash = "sha256:0d4b4397ed669d371c81dcd1ef33fd384a44d6c3de1bd0ca7ac06d837720d3c5", size = 133477, upload-time = "2026-01-19T06:47:38.619Z" }, -] - [[package]] name = "networkx" version = "3.6.1" @@ -1392,15 +997,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6c/f8/fa85b2eac68ec631d0b631abc448552cb17d39afd17ec53dcbcc3537681a/numpy-2.4.1-cp313-cp313t-win_arm64.whl", hash = "sha256:a7870e8c5fc11aef57d6fea4b4085e537a3a60ad2cdd14322ed531fdca68d261", size = 10382981, upload-time = "2026-01-10T06:43:52.575Z" }, ] -[[package]] -name = "oauthlib" -version = "3.3.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/0b/5f/19930f824ffeb0ad4372da4812c50edbd1434f678c90c2733e1188edfc63/oauthlib-3.3.1.tar.gz", hash = "sha256:0f0f8aa759826a193cf66c12ea1af1637f87b9b4622d46e866952bb022e538c9", size = 185918, upload-time = "2025-06-19T22:48:08.269Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/be/9c/92789c596b8df838baa98fa71844d84283302f7604ed565dafe5a6b5041a/oauthlib-3.3.1-py3-none-any.whl", hash = "sha256:88119c938d2b8fb88561af5f6ee0eec8cc8d552b7bb1f712743136eb7523b7a1", size = 160065, upload-time = "2025-06-19T22:48:06.508Z" }, -] - [[package]] name = "omegaconf" version = "2.3.0" @@ -1483,43 +1079,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/20/12/38679034af332785aac8774540895e234f4d07f7545804097de4b666afd8/packaging-25.0-py3-none-any.whl", hash = "sha256:29572ef2b1f17581046b3a2227d5c611fb25ec70ca1ba8554b24b0e69331a484", size = 66469, upload-time = "2025-04-19T11:48:57.875Z" }, ] -[[package]] -name = "pandas" -version = "3.0.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "numpy" }, - { name = "python-dateutil" }, - { name = "tzdata", marker = "sys_platform == 'emscripten' or sys_platform == 'win32'" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/da/99/b342345300f13440fe9fe385c3c481e2d9a595ee3bab4d3219247ac94e9a/pandas-3.0.2.tar.gz", hash = "sha256:f4753e73e34c8d83221ba58f232433fca2748be8b18dbca02d242ed153945043", size = 4645855, upload-time = "2026-03-31T06:48:30.816Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/bf/ca/3e639a1ea6fcd0617ca4e8ca45f62a74de33a56ae6cd552735470b22c8d3/pandas-3.0.2-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b5918ba197c951dec132b0c5929a00c0bf05d5942f590d3c10a807f6e15a57d3", size = 10321105, upload-time = "2026-03-31T06:46:57.327Z" }, - { url = "https://files.pythonhosted.org/packages/0b/77/dbc82ff2fb0e63c6564356682bf201edff0ba16c98630d21a1fb312a8182/pandas-3.0.2-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:d606a041c89c0a474a4702d532ab7e73a14fe35c8d427b972a625c8e46373668", size = 9864088, upload-time = "2026-03-31T06:46:59.935Z" }, - { url = "https://files.pythonhosted.org/packages/5c/2b/341f1b04bbca2e17e13cd3f08c215b70ef2c60c5356ef1e8c6857449edc7/pandas-3.0.2-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:710246ba0616e86891b58ab95f2495143bb2bc83ab6b06747c74216f583a6ac9", size = 10369066, upload-time = "2026-03-31T06:47:02.792Z" }, - { url = "https://files.pythonhosted.org/packages/12/c5/cbb1ffefb20a93d3f0e1fdcda699fb84976210d411b008f97f48bf6ce27e/pandas-3.0.2-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5d3cfe227c725b1f3dff4278b43d8c784656a42a9325b63af6b1492a8232209e", size = 10876780, upload-time = "2026-03-31T06:47:06.205Z" }, - { url = "https://files.pythonhosted.org/packages/98/fe/2249ae5e0a69bd0ddf17353d0a5d26611d70970111f5b3600cdc8be883e7/pandas-3.0.2-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:c3b723df9087a9a9a840e263ebd9f88b64a12075d1bf2ea401a5a42f254f084d", size = 11375181, upload-time = "2026-03-31T06:47:09.383Z" }, - { url = "https://files.pythonhosted.org/packages/de/64/77a38b09e70b6464883b8d7584ab543e748e42c1b5d337a2ee088e0df741/pandas-3.0.2-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:a3096110bf9eac0070b7208465f2740e2d8a670d5cb6530b5bb884eca495fd39", size = 11928899, upload-time = "2026-03-31T06:47:12.686Z" }, - { url = "https://files.pythonhosted.org/packages/5e/52/42855bf626868413f761addd574acc6195880ae247a5346477a4361c3acb/pandas-3.0.2-cp313-cp313-win_amd64.whl", hash = "sha256:07a10f5c36512eead51bc578eb3354ad17578b22c013d89a796ab5eee90cd991", size = 9746574, upload-time = "2026-03-31T06:47:15.64Z" }, - { url = "https://files.pythonhosted.org/packages/88/39/21304ae06a25e8bf9fc820d69b29b2c495b2ae580d1e143146c309941760/pandas-3.0.2-cp313-cp313-win_arm64.whl", hash = "sha256:5fdbfa05931071aba28b408e59226186b01eb5e92bea2ab78b65863ca3228d84", size = 9047156, upload-time = "2026-03-31T06:47:18.595Z" }, - { url = "https://files.pythonhosted.org/packages/72/20/7defa8b27d4f330a903bb68eea33be07d839c5ea6bdda54174efcec0e1d2/pandas-3.0.2-cp313-cp313t-macosx_10_13_x86_64.whl", hash = "sha256:dbc20dea3b9e27d0e66d74c42b2d0c1bed9c2ffe92adea33633e3bedeb5ac235", size = 10756238, upload-time = "2026-03-31T06:47:22.012Z" }, - { url = "https://files.pythonhosted.org/packages/e9/95/49433c14862c636afc0e9b2db83ff16b3ad92959364e52b2955e44c8e94c/pandas-3.0.2-cp313-cp313t-macosx_11_0_arm64.whl", hash = "sha256:b75c347eff42497452116ce05ef461822d97ce5b9ff8df6edacb8076092c855d", size = 10408520, upload-time = "2026-03-31T06:47:25.197Z" }, - { url = "https://files.pythonhosted.org/packages/3b/f8/462ad2b5881d6b8ec8e5f7ed2ea1893faa02290d13870a1600fe72ad8efc/pandas-3.0.2-cp313-cp313t-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:d1478075142e83a5571782ad007fb201ed074bdeac7ebcc8890c71442e96adf7", size = 10324154, upload-time = "2026-03-31T06:47:28.097Z" }, - { url = "https://files.pythonhosted.org/packages/0a/65/d1e69b649cbcddda23ad6e4c40ef935340f6f652a006e5cbc3555ac8adb3/pandas-3.0.2-cp313-cp313t-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:5880314e69e763d4c8b27937090de570f1fb8d027059a7ada3f7f8e98bdcb677", size = 10714449, upload-time = "2026-03-31T06:47:30.85Z" }, - { url = "https://files.pythonhosted.org/packages/47/a4/85b59bc65b8190ea3689882db6cdf32a5003c0ccd5a586c30fdcc3ffc4fc/pandas-3.0.2-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:b5329e26898896f06035241a626d7c335daa479b9bbc82be7c2742d048e41172", size = 11338475, upload-time = "2026-03-31T06:47:34.026Z" }, - { url = "https://files.pythonhosted.org/packages/1e/c4/bc6966c6e38e5d9478b935272d124d80a589511ed1612a5d21d36f664c68/pandas-3.0.2-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:81526c4afd31971f8b62671442a4b2b51e0aa9acc3819c9f0f12a28b6fcf85f1", size = 11786568, upload-time = "2026-03-31T06:47:36.941Z" }, - { url = "https://files.pythonhosted.org/packages/e8/74/09298ca9740beed1d3504e073d67e128aa07e5ca5ca2824b0c674c0b8676/pandas-3.0.2-cp313-cp313t-win_amd64.whl", hash = "sha256:7cadd7e9a44ec13b621aec60f9150e744cfc7a3dd32924a7e2f45edff31823b0", size = 10488652, upload-time = "2026-03-31T06:47:40.612Z" }, -] - -[[package]] -name = "pathspec" -version = "1.1.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5a/82/42f767fc1c1143d6fd36efb827202a2d997a375e160a71eb2888a925aac1/pathspec-1.1.1.tar.gz", hash = "sha256:17db5ecd524104a120e173814c90367a96a98d07c45b2e10c2f3919fff91bf5a", size = 135180, upload-time = "2026-04-27T01:46:08.907Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f1/d9/7fb5aa316bc299258e68c73ba3bddbc499654a07f151cba08f6153988714/pathspec-1.1.1-py3-none-any.whl", hash = "sha256:a00ce642f577bf7f473932318056212bc4f8bfdf53128c78bbd5af0b9b20b189", size = 57328, upload-time = "2026-04-27T01:46:07.06Z" }, -] - [[package]] name = "platformdirs" version = "4.5.1" @@ -1529,25 +1088,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/cb/28/3bfe2fa5a7b9c46fe7e13c97bda14c895fb10fa2ebf1d0abb90e0cea7ee1/platformdirs-4.5.1-py3-none-any.whl", hash = "sha256:d03afa3963c806a9bed9d5125c8f4cb2fdaf74a55ab60e5d59b3fde758104d31", size = 18731, upload-time = "2025-12-05T13:52:56.823Z" }, ] -[[package]] -name = "playwright" -version = "1.58.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "greenlet" }, - { name = "pyee" }, -] -wheels = [ - { url = "https://files.pythonhosted.org/packages/f8/c9/9c6061d5703267f1baae6a4647bfd1862e386fbfdb97d889f6f6ae9e3f64/playwright-1.58.0-py3-none-macosx_10_13_x86_64.whl", hash = "sha256:96e3204aac292ee639edbfdef6298b4be2ea0a55a16b7068df91adac077cc606", size = 42251098, upload-time = "2026-01-30T15:09:24.028Z" }, - { url = "https://files.pythonhosted.org/packages/e0/40/59d34a756e02f8c670f0fee987d46f7ee53d05447d43cd114ca015cb168c/playwright-1.58.0-py3-none-macosx_11_0_arm64.whl", hash = "sha256:70c763694739d28df71ed578b9c8202bb83e8fe8fb9268c04dd13afe36301f71", size = 41039625, upload-time = "2026-01-30T15:09:27.558Z" }, - { url = "https://files.pythonhosted.org/packages/e1/ee/3ce6209c9c74a650aac9028c621f357a34ea5cd4d950700f8e2c4b7fe2c4/playwright-1.58.0-py3-none-macosx_11_0_universal2.whl", hash = "sha256:185e0132578733d02802dfddfbbc35f42be23a45ff49ccae5081f25952238117", size = 42251098, upload-time = "2026-01-30T15:09:30.461Z" }, - { url = "https://files.pythonhosted.org/packages/f1/af/009958cbf23fac551a940d34e3206e6c7eed2b8c940d0c3afd1feb0b0589/playwright-1.58.0-py3-none-manylinux1_x86_64.whl", hash = "sha256:c95568ba1eda83812598c1dc9be60b4406dffd60b149bc1536180ad108723d6b", size = 46235268, upload-time = "2026-01-30T15:09:33.787Z" }, - { url = "https://files.pythonhosted.org/packages/d9/a6/0e66ad04b6d3440dae73efb39540c5685c5fc95b17c8b29340b62abbd952/playwright-1.58.0-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f9999948f1ab541d98812de25e3a8c410776aa516d948807140aff797b4bffa", size = 45964214, upload-time = "2026-01-30T15:09:36.751Z" }, - { url = "https://files.pythonhosted.org/packages/0e/4b/236e60ab9f6d62ed0fd32150d61f1f494cefbf02304c0061e78ed80c1c32/playwright-1.58.0-py3-none-win32.whl", hash = "sha256:1e03be090e75a0fabbdaeab65ce17c308c425d879fa48bb1d7986f96bfad0b99", size = 36815998, upload-time = "2026-01-30T15:09:39.627Z" }, - { url = "https://files.pythonhosted.org/packages/41/f8/5ec599c5e59d2f2f336a05b4f318e733077cd5044f24adb6f86900c3e6a7/playwright-1.58.0-py3-none-win_amd64.whl", hash = "sha256:a2bf639d0ce33b3ba38de777e08697b0d8f3dc07ab6802e4ac53fb65e3907af8", size = 36816005, upload-time = "2026-01-30T15:09:42.449Z" }, - { url = "https://files.pythonhosted.org/packages/c8/c4/cc0229fea55c87d6c9c67fe44a21e2cd28d1d558a5478ed4d617e9fb0c93/playwright-1.58.0-py3-none-win_arm64.whl", hash = "sha256:32ffe5c303901a13a0ecab91d1c3f74baf73b84f4bedbb6b935f5bc11cc98e1b", size = 33085919, upload-time = "2026-01-30T15:09:45.71Z" }, -] - [[package]] name = "pluggy" version = "1.6.0" @@ -1557,21 +1097,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/54/20/4d324d65cc6d9205fabedc306948156824eb9f0ee1633355a8f7ec5c66bf/pluggy-1.6.0-py3-none-any.whl", hash = "sha256:e920276dd6813095e9377c0bc5566d94c932c33b27a3e3945d8389c374dd4746", size = 20538, upload-time = "2025-05-15T12:30:06.134Z" }, ] -[[package]] -name = "postgrest" -version = "2.30.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "deprecation" }, - { name = "httpx", extra = ["http2"] }, - { name = "pydantic" }, - { name = "yarl" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/56/7c/54e7be05adc9fd6fd98dc572ddfc8982d45bec314a55711e37277d440698/postgrest-2.30.0.tar.gz", hash = "sha256:4f89eec56ce605ab6fbddd9b96d526a9bb44962796d44a5d85cb77640eb766c3", size = 14430, upload-time = "2026-05-06T17:35:21.559Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/22/aa/ff2e09f99f95ea96fddeb373646bf907dd89a24fc00b5d38e5674ca7c9ca/postgrest-2.30.0-py3-none-any.whl", hash = "sha256:30631e7993da542419f4217cf3b60aa641084731ea15e66a18526a3a52e40a7d", size = 23108, upload-time = "2026-05-06T17:35:20.531Z" }, -] - [[package]] name = "pre-commit" version = "4.2.0" @@ -1627,62 +1152,13 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/5b/5a/bc7b4a4ef808fa59a816c17b20c4bef6884daebbdf627ff2a161da67da19/propcache-0.4.1-py3-none-any.whl", hash = "sha256:af2a6052aeb6cf17d3e46ee169099044fd8224cbaf75c76a2ef596e8163e2237", size = 13305, upload-time = "2025-10-08T19:49:00.792Z" }, ] -[[package]] -name = "proto-plus" -version = "1.28.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "protobuf" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/c9/56/e647b0c675392d2da368da7b6f158f7368b18542fd6f7d7400a2f39de000/proto_plus-1.28.0.tar.gz", hash = "sha256:38e5696342835b08fc116f30a25665b29531cda9d5d5643e9b81fc312385abd9", size = 57221, upload-time = "2026-05-07T08:04:50.811Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7c/20/b122d4626976acb81132036d2ad1bb35a1a8775fceb837ec30964622516a/proto_plus-1.28.0-py3-none-any.whl", hash = "sha256:a630604310899e73c59ec302e5765c058d412b2f090b9c79c8822589f14955b8", size = 50410, upload-time = "2026-05-07T08:03:31.962Z" }, -] - -[[package]] -name = "protobuf" -version = "7.34.1" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/6b/6b/a0e95cad1ad7cc3f2c6821fcab91671bd5b78bd42afb357bb4765f29bc41/protobuf-7.34.1.tar.gz", hash = "sha256:9ce42245e704cc5027be797c1db1eb93184d44d1cdd71811fb2d9b25ad541280", size = 454708, upload-time = "2026-03-20T17:34:47.036Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/11/3325d41e6ee15bf1125654301211247b042563bcc898784351252549a8ad/protobuf-7.34.1-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:d8b2cc79c4d8f62b293ad9b11ec3aebce9af481fa73e64556969f7345ebf9fc7", size = 429247, upload-time = "2026-03-20T17:34:37.024Z" }, - { url = "https://files.pythonhosted.org/packages/eb/9d/aa69df2724ff63efa6f72307b483ce0827f4347cc6d6df24b59e26659fef/protobuf-7.34.1-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:5185e0e948d07abe94bb76ec9b8416b604cfe5da6f871d67aad30cbf24c3110b", size = 325753, upload-time = "2026-03-20T17:34:38.751Z" }, - { url = "https://files.pythonhosted.org/packages/92/e8/d174c91fd48e50101943f042b09af9029064810b734e4160bbe282fa1caa/protobuf-7.34.1-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:403b093a6e28a960372b44e5eb081775c9b056e816a8029c61231743d63f881a", size = 340198, upload-time = "2026-03-20T17:34:39.871Z" }, - { url = "https://files.pythonhosted.org/packages/53/1b/3b431694a4dc6d37b9f653f0c64b0a0d9ec074ee810710c0c3da21d67ba7/protobuf-7.34.1-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:8ff40ce8cd688f7265326b38d5a1bed9bfdf5e6723d49961432f83e21d5713e4", size = 324267, upload-time = "2026-03-20T17:34:41.1Z" }, - { url = "https://files.pythonhosted.org/packages/85/29/64de04a0ac142fb685fd09999bc3d337943fb386f3a0ec57f92fd8203f97/protobuf-7.34.1-cp310-abi3-win32.whl", hash = "sha256:34b84ce27680df7cca9f231043ada0daa55d0c44a2ddfaa58ec1d0d89d8bf60a", size = 426628, upload-time = "2026-03-20T17:34:42.536Z" }, - { url = "https://files.pythonhosted.org/packages/4d/87/cb5e585192a22b8bd457df5a2c16a75ea0db9674c3a0a39fc9347d84e075/protobuf-7.34.1-cp310-abi3-win_amd64.whl", hash = "sha256:e97b55646e6ce5cbb0954a8c28cd39a5869b59090dfaa7df4598a7fba869468c", size = 437901, upload-time = "2026-03-20T17:34:44.112Z" }, - { url = "https://files.pythonhosted.org/packages/88/95/608f665226bca68b736b79e457fded9a2a38c4f4379a4a7614303d9db3bc/protobuf-7.34.1-py3-none-any.whl", hash = "sha256:bb3812cd53aefea2b028ef42bd780f5b96407247f20c6ef7c679807e9d188f11", size = 170715, upload-time = "2026-03-20T17:34:45.384Z" }, -] - -[[package]] -name = "pyarrow" -version = "24.0.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/91/13/13e1069b351bdc3881266e11147ffccf687505dbb0ea74036237f5d454a5/pyarrow-24.0.0.tar.gz", hash = "sha256:85fe721a14dd823aca09127acbb06c3ca723efbd436c004f16bca601b04dcc83", size = 1180261, upload-time = "2026-04-21T10:51:25.837Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/6f/d3/a1abf004482026ddc17f4503db227787fa3cfe41ec5091ff20e4fea55e57/pyarrow-24.0.0-cp313-cp313-macosx_12_0_arm64.whl", hash = "sha256:02b001b3ed4723caa44f6cd1af2d5c86aa2cf9971dacc2ffa55b21237713dfba", size = 34976759, upload-time = "2026-04-21T10:48:07.258Z" }, - { url = "https://files.pythonhosted.org/packages/4f/4a/34f0a36d28a2dd32225301b79daad44e243dc1a2bb77d43b60749be255c4/pyarrow-24.0.0-cp313-cp313-macosx_12_0_x86_64.whl", hash = "sha256:04920d6a71aabd08a0417709efce97d45ea8e6fb733d9ca9ecffb13c67839f68", size = 36658471, upload-time = "2026-04-21T10:48:13.347Z" }, - { url = "https://files.pythonhosted.org/packages/1f/78/543b94712ae8bb1a6023bcc1acf1a740fbff8286747c289cd9468fced2a5/pyarrow-24.0.0-cp313-cp313-manylinux_2_28_aarch64.whl", hash = "sha256:a964266397740257f16f7bb2e4f08a0c81454004beab8ff59dd531b73610e9f2", size = 45675981, upload-time = "2026-04-21T10:48:20.201Z" }, - { url = "https://files.pythonhosted.org/packages/84/9f/8fb7c222b100d314137fa40ec050de56cd8c6d957d1cfff685ce72f15b17/pyarrow-24.0.0-cp313-cp313-manylinux_2_28_x86_64.whl", hash = "sha256:6f066b179d68c413374294bc1735f68475457c933258df594443bb9d88ddc2a0", size = 48859172, upload-time = "2026-04-21T10:48:27.541Z" }, - { url = "https://files.pythonhosted.org/packages/a7/d3/1ea72538e6c8b3b475ed78d1049a2c518e655761ea50fe1171fc855fcab7/pyarrow-24.0.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:1183baeb14c5f587b1ec52831e665718ce632caab84b7cd6b85fd44f96114495", size = 49385733, upload-time = "2026-04-21T10:48:34.7Z" }, - { url = "https://files.pythonhosted.org/packages/c3/be/c3d8b06a1ba35f2260f8e1f771abbee7d5e345c0937aab90675706b1690a/pyarrow-24.0.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:806f24b4085453c197a5078218d1ee08783ebbba271badd153d1ae22a3ee804f", size = 51934335, upload-time = "2026-04-21T10:48:42.099Z" }, - { url = "https://files.pythonhosted.org/packages/9c/62/89e07a1e7329d2cde3e3c6994ba0839a24977a2beda8be6005ea3d860b99/pyarrow-24.0.0-cp313-cp313-win_amd64.whl", hash = "sha256:e4505fc6583f7b05ab854934896bcac8253b04ac1171a77dfb73efef92076d91", size = 27271748, upload-time = "2026-04-21T10:49:42.532Z" }, - { url = "https://files.pythonhosted.org/packages/17/1a/cff3a59f80b5b1658549d46611b67163f65e0664431c076ad728bf9d5af4/pyarrow-24.0.0-cp313-cp313t-macosx_12_0_arm64.whl", hash = "sha256:1a4e45017efbf115032e4475ee876d525e0e36c742214fbe405332480ecd6275", size = 35238554, upload-time = "2026-04-21T10:48:48.526Z" }, - { url = "https://files.pythonhosted.org/packages/a8/99/cce0f42a327bfef2c420fb6078a3eb834826e5d6697bf3009fe11d2ad051/pyarrow-24.0.0-cp313-cp313t-macosx_12_0_x86_64.whl", hash = "sha256:7986f1fa71cee060ad00758bcc79d3a93bab8559bf978fab9e53472a2e25a17b", size = 36782301, upload-time = "2026-04-21T10:48:55.181Z" }, - { url = "https://files.pythonhosted.org/packages/2a/66/8e560d5ff6793ca29aca213c53eec0dd482dd46cb93b2819e5aab52e4252/pyarrow-24.0.0-cp313-cp313t-manylinux_2_28_aarch64.whl", hash = "sha256:d3e0b61e8efb24ed38898e5cdc5fffa9124be480008d401a1f8071500494ae42", size = 45721929, upload-time = "2026-04-21T10:49:03.676Z" }, - { url = "https://files.pythonhosted.org/packages/27/0c/a26e25505d030716e078d9f16eb74973cbf0b33b672884e9f9da1c83b871/pyarrow-24.0.0-cp313-cp313t-manylinux_2_28_x86_64.whl", hash = "sha256:55a3bc1e3df3b5567b7d27ef551b2283f0c68a5e86f1cd56abc569da4f31335b", size = 48825365, upload-time = "2026-04-21T10:49:11.714Z" }, - { url = "https://files.pythonhosted.org/packages/5f/eb/771f9ecb0c65e73fe9dccdd1717901b9594f08c4515d000c7c62df573811/pyarrow-24.0.0-cp313-cp313t-musllinux_1_2_aarch64.whl", hash = "sha256:641f795b361874ac9da5294f8f443dfdbee355cf2bd9e3b8d97aaac2306b9b37", size = 49451819, upload-time = "2026-04-21T10:49:21.474Z" }, - { url = "https://files.pythonhosted.org/packages/48/da/61ae89a88732f5a785646f3ec6125dbb640fa98a540eb2b9889caa561403/pyarrow-24.0.0-cp313-cp313t-musllinux_1_2_x86_64.whl", hash = "sha256:8adc8e6ce5fccf5dc707046ae4914fd537def529709cc0d285d37a7f9cd442ca", size = 51909252, upload-time = "2026-04-21T10:49:31.164Z" }, - { url = "https://files.pythonhosted.org/packages/cb/1a/8dd5cafab7b66573fa91c03d06d213356ad4edd71813aa75e08ce2b3a844/pyarrow-24.0.0-cp313-cp313t-win_amd64.whl", hash = "sha256:9b18371ad2f44044b81a8d23bc2d8a9b6a6226dca775e8e16cfee640473d6c5d", size = 27388127, upload-time = "2026-04-21T10:49:37.334Z" }, -] - [[package]] name = "pyasn1" -version = "0.6.3" +version = "0.6.4" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/5c/5f/6583902b6f79b399c9c40674ac384fd9cd77805f9e6205075f828ef11fb2/pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf", size = 148685, upload-time = "2026-03-17T01:06:53.382Z" } +sdist = { url = "https://files.pythonhosted.org/packages/a4/9a/23310166d960def5897e91fe20e5b724601b02a22e84ba1f94232c0b7f67/pyasn1-0.6.4.tar.gz", hash = "sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81", size = 151262, upload-time = "2026-07-09T01:12:33.988Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/5d/a0/7d793dce3fa811fe047d6ae2431c672364b462850c6235ae306c0efd025f/pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde", size = 83997, upload-time = "2026-03-17T01:06:52.036Z" }, + { url = "https://files.pythonhosted.org/packages/9a/3b/6163796d69c3977d1e4287bea4a6979161cbbdd170ebb430511e8e1999ce/pyasn1-0.6.4-py3-none-any.whl", hash = "sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b", size = 84410, upload-time = "2026-07-09T01:12:32.92Z" }, ] [[package]] @@ -1760,18 +1236,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/77/c1/6e422f34e569cf8e18df68d1939c81c099d2b61e4f7d9621c8a77560799c/pydantic_settings-2.14.2-py3-none-any.whl", hash = "sha256:a20c97b37910b6550d5ea50fbcc2d4187defe58cd57070b73863d069419c9440", size = 61715, upload-time = "2026-06-19T13:44:55.02Z" }, ] -[[package]] -name = "pyee" -version = "13.0.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "typing-extensions" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/8b/04/e7c1fe4dc78a6fdbfd6c337b1c3732ff543b8a397683ab38378447baa331/pyee-13.0.1.tar.gz", hash = "sha256:0b931f7c14535667ed4c7e0d531716368715e860b988770fc7eb8578d1f67fc8", size = 31655, upload-time = "2026-02-14T21:12:28.044Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a0/c4/b4d4827c93ef43c01f599ef31453ccc1c132b353284fc6c87d535c233129/pyee-13.0.1-py3-none-any.whl", hash = "sha256:af2f8fede4171ef667dfded53f96e2ed0d6e6bd7ee3bb46437f77e3b57689228", size = 15659, upload-time = "2026-02-14T21:12:26.263Z" }, -] - [[package]] name = "pygments" version = "2.20.0" @@ -1781,78 +1245,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f4/7e/a72dd26f3b0f4f2bf1dd8923c85f7ceb43172af56d63c7383eb62b332364/pygments-2.20.0-py3-none-any.whl", hash = "sha256:81a9e26dd42fd28a23a2d169d86d7ac03b46e2f8b59ed4698fb4785f946d0176", size = 1231151, upload-time = "2026-03-29T13:29:30.038Z" }, ] -[[package]] -name = "pyiceberg" -version = "0.11.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "cachetools" }, - { name = "click" }, - { name = "fsspec" }, - { name = "mmh3" }, - { name = "pydantic" }, - { name = "pyparsing" }, - { name = "pyroaring" }, - { name = "requests" }, - { name = "rich" }, - { name = "strictyaml" }, - { name = "tenacity" }, - { name = "zstandard" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/ce/f0/7616676603fdbd05ab97816337a9b31be08a5f9e1ffd636260812b217e0f/pyiceberg-0.11.1.tar.gz", hash = "sha256:366fe0d5a74e3cf1d4e7cbf3c49e308da60e7835ea268667be9185388f05d7a5", size = 1076075, upload-time = "2026-03-03T00:10:27.61Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/35/4c/a122d80d98cb6125d87024681263406433f0c25c699d503f5633521e6809/pyiceberg-0.11.1-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:b7ec5db19feab98a31fcd5caccf4a9a4e83f96933d1ca393ba7aea665710c2bb", size = 532644, upload-time = "2026-03-03T00:10:18.574Z" }, - { url = "https://files.pythonhosted.org/packages/10/94/9a8fa5fc580e6dccd34bbbf51e7658cd7b49540e2458783addeff5e22a91/pyiceberg-0.11.1-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:cec0616d2ba6e7dda6327089a2f34ec723aa9ac2c389857ef0b83f65fb135dd6", size = 532787, upload-time = "2026-03-03T00:10:19.656Z" }, - { url = "https://files.pythonhosted.org/packages/b3/ab/ab7c88828bc17d77dbbc5a765419dfec2135629e1d74cdd0762cd38ad867/pyiceberg-0.11.1-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:ddb360da76c62c7c23ec3da40e1af48e6712a563905fea2d1a8911ff7a3b6c4d", size = 722202, upload-time = "2026-03-03T00:10:21.012Z" }, - { url = "https://files.pythonhosted.org/packages/df/38/079cf1c0bf86da315472a926eec0dba10135f43374a2e267336eb98d8c76/pyiceberg-0.11.1-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:4d8790f420ebc484236017edba59182cf2a21bd3e4224a0bd0760a9c7268e96a", size = 724037, upload-time = "2026-03-03T00:10:22.176Z" }, - { url = "https://files.pythonhosted.org/packages/08/6b/08eaef477debb110438d943ef3f5985096f660ccb735d6344701cbd075a9/pyiceberg-0.11.1-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:ae27ba4d37925d5b2cff192acaa70c8bb114d632bbc527cc91fea0370702b866", size = 716035, upload-time = "2026-03-03T00:10:23.789Z" }, - { url = "https://files.pythonhosted.org/packages/0b/59/7671d6a630ab1d85c6e7ca8ddf438dc63a0b0dd183bc4be69bf25c0fa5f6/pyiceberg-0.11.1-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:db66a4e0fdfbf4090631d59c3f65e960d9a5561e9259f6f3993cbe91e396837e", size = 720887, upload-time = "2026-03-03T00:10:24.824Z" }, - { url = "https://files.pythonhosted.org/packages/f0/2b/5c8ad37807efaedb14b20f01f36462684468c80da5b74f4018fb4c1804b5/pyiceberg-0.11.1-cp313-cp313-win_amd64.whl", hash = "sha256:eb3a0a3e630ee89758eb96b39b456f4697732351fb0c080e9498ea578f9b71f9", size = 530923, upload-time = "2026-03-03T00:10:26.196Z" }, -] - -[[package]] -name = "pyjwt" -version = "2.13.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/3b/81/58d0ac84e1ef3a3843791d6954d94c0b33d526c75eeb1efbce9d0a4c4077/pyjwt-2.13.0.tar.gz", hash = "sha256:41571c89ca91598c79e8ef18a2d07367d4810fbbd6f637794879baf1b7703423", size = 107515, upload-time = "2026-05-21T19:54:36.618Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a3/5e/ecf12fdb62546d64385c158514e9b2b671f7832108ef2ecd2020ce0af2d1/pyjwt-2.13.0-py3-none-any.whl", hash = "sha256:66adcc2aff09b3f1bbd95fc1e1577df8ac8723c978552fd43304c8a290ac5728", size = 31274, upload-time = "2026-05-21T19:54:35.362Z" }, -] - -[package.optional-dependencies] -crypto = [ - { name = "cryptography" }, -] - -[[package]] -name = "pyparsing" -version = "3.3.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/f3/91/9c6ee907786a473bf81c5f53cf703ba0957b23ab84c264080fb5a450416f/pyparsing-3.3.2.tar.gz", hash = "sha256:c777f4d763f140633dcb6d8a3eda953bf7a214dc4eff598413c070bcdc117cbc", size = 6851574, upload-time = "2026-01-21T03:57:59.36Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/10/bd/c038d7cc38edc1aa5bf91ab8068b63d4308c66c4c8bb3cbba7dfbc049f9c/pyparsing-3.3.2-py3-none-any.whl", hash = "sha256:850ba148bd908d7e2411587e247a1e4f0327839c40e2e5e6d05a007ecc69911d", size = 122781, upload-time = "2026-01-21T03:57:55.912Z" }, -] - -[[package]] -name = "pyroaring" -version = "1.1.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/d7/46/a50510d080f8cb089303ec0f7cd80736b2949ca3d148f48f1cc90c49e345/pyroaring-1.1.0.tar.gz", hash = "sha256:f02e4021397ae02a139defdc6813b9942ab163de90affddd4ce4efbac299f619", size = 200298, upload-time = "2026-04-24T21:29:25.212Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/7d/75/1d39ecb04e6cd96d191eb8884864355051df80928dd5096a9dea43fbf63b/pyroaring-1.1.0-cp313-cp313-macosx_14_0_arm64.whl", hash = "sha256:72f68a16b00b35481d9b3bfe897ecd8a1f7da69efd92ba5b17347ca11c21cb0d", size = 333363, upload-time = "2026-04-24T21:28:23.838Z" }, - { url = "https://files.pythonhosted.org/packages/20/3e/65cd0871e86d11c5c5cfd0f5abb0ca80eb2b6b5dbe5a2433f315a9ebd90c/pyroaring-1.1.0-cp313-cp313-macosx_14_0_universal2.whl", hash = "sha256:4c443e9f942b6089efe8c9b264576e9d116f90be28a315679375bba2d8a915d6", size = 710573, upload-time = "2026-04-24T21:28:24.884Z" }, - { url = "https://files.pythonhosted.org/packages/f9/a2/f8f23515f41414332e60cd86e4957e2a6838070b2ad5fe25e80f136de635/pyroaring-1.1.0-cp313-cp313-macosx_14_0_x86_64.whl", hash = "sha256:3beb40eb1220d1ce4fb3661bb019e9a21857e5bb294fe8c1c5016aeb6e82318c", size = 384880, upload-time = "2026-04-24T21:28:25.864Z" }, - { url = "https://files.pythonhosted.org/packages/b0/5b/82dc44b5074a1ff62e702d12611272d1711a60d5518dab23f94e1f7a9b3d/pyroaring-1.1.0-cp313-cp313-manylinux_2_24_aarch64.manylinux_2_28_aarch64.whl", hash = "sha256:8f1f56004e8f1c1489bf279c25f1fa4764252cd9af5fb35675774268a4a615ba", size = 1999529, upload-time = "2026-04-24T21:28:26.859Z" }, - { url = "https://files.pythonhosted.org/packages/11/40/b07bac8cdc4b709a05f5c55bb52d4f684e5ea1fadfa0b6d9decf477a9d2a/pyroaring-1.1.0-cp313-cp313-manylinux_2_24_armv7l.manylinux_2_31_armv7l.whl", hash = "sha256:13660386ea8905ee4d42c21a6275463e2dc7d31e0b5d65eec210aa7043ad96f4", size = 1842927, upload-time = "2026-04-24T21:28:28.056Z" }, - { url = "https://files.pythonhosted.org/packages/0d/60/c4b511965802dfc77978a9e16f2813f47fb3083db1822019ba1bb169c685/pyroaring-1.1.0-cp313-cp313-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:0dfb6cf50fd8898179e460e699a6b8326ca508c627d083f7bf62f769fe1717d5", size = 2199538, upload-time = "2026-04-24T21:28:29.425Z" }, - { url = "https://files.pythonhosted.org/packages/e8/12/38f6b50b3f3f41a8b752d3e9efcf105b18eb2c66811831059f25613734ac/pyroaring-1.1.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:81ebbc0c880c8a10f13118632e5c0d59159ceada8b651bba18f2e6dc70efdeda", size = 2896904, upload-time = "2026-04-24T21:28:30.67Z" }, - { url = "https://files.pythonhosted.org/packages/5a/b6/b5436e4b93c6bf2bd3dd6ccb88cbdc64b12084151a43e2f5c94be50eb710/pyroaring-1.1.0-cp313-cp313-musllinux_1_2_armv7l.whl", hash = "sha256:370d191b0d1b32bbd99452ef5f0485f22fcc4bf7404d33b821d0ce2459951152", size = 2733819, upload-time = "2026-04-24T21:28:31.882Z" }, - { url = "https://files.pythonhosted.org/packages/ab/8f/f392f268de9607a5c7a95aaed6b9c8a81f00c14d85c33855e9f492095478/pyroaring-1.1.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:e8b3bfad0ae3ef0e67b40c193863dce8b7d79de545dadbe53c19acc3ace38f66", size = 3161730, upload-time = "2026-04-24T21:28:33.244Z" }, - { url = "https://files.pythonhosted.org/packages/9e/a1/03250fd4834b6a5c13e6600bca47ea20fda579f80bce3551d4985185d164/pyroaring-1.1.0-cp313-cp313-win32.whl", hash = "sha256:eead129046822cb0fd47c78740b81bdaffd0515c0bb0306a2318acf0f0540b58", size = 211194, upload-time = "2026-04-24T21:28:35.001Z" }, - { url = "https://files.pythonhosted.org/packages/70/63/d9b307462cddc82fe94a67d6810e5c802818690e131ba690c1de674d8558/pyroaring-1.1.0-cp313-cp313-win_amd64.whl", hash = "sha256:90ab2f00c09eed5bd986a80c8641e2dc10e7aca1a2d892d89a44b396e39c08ea", size = 263110, upload-time = "2026-04-24T21:28:35.976Z" }, - { url = "https://files.pythonhosted.org/packages/d9/4a/aa6e9833a6ba9a630efdbec8783b63da6602f763b37a5b5fbc01d73a1af1/pyroaring-1.1.0-cp313-cp313-win_arm64.whl", hash = "sha256:51dd2490a64ad4ed53c4fb58ef1ee3f84f6cbd97cdb47abd9065c9f714ab72ef", size = 216546, upload-time = "2026-04-24T21:28:37.065Z" }, -] - [[package]] name = "pytest" version = "9.0.3" @@ -1869,18 +1261,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d4/24/a372aaf5c9b7208e7112038812994107bc65a84cd00e0354a88c2c77a617/pytest-9.0.3-py3-none-any.whl", hash = "sha256:2c5efc453d45394fdd706ade797c0a81091eccd1d6e4bccfcd476e2b8e0ab5d9", size = 375249, upload-time = "2026-04-07T17:16:16.13Z" }, ] -[[package]] -name = "python-dateutil" -version = "2.9.0.post0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "six" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" }, -] - [[package]] name = "python-dotenv" version = "1.2.2" @@ -1890,25 +1270,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/0b/d7/1959b9648791274998a9c3526f6d0ec8fd2233e4d4acce81bbae76b44b2a/python_dotenv-1.2.2-py3-none-any.whl", hash = "sha256:1d8214789a24de455a8b8bd8ae6fe3c6b69a5e3d64aa8a8e5d68e694bbcb285a", size = 22101, upload-time = "2026-03-01T16:00:25.09Z" }, ] -[[package]] -name = "python-multipart" -version = "0.0.31" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/64/7e/9b35ad8f3d9ca680f7c87a88f19612fdd8da9796c4d3b46e560ac79dcc4a/python_multipart-0.0.31.tar.gz", hash = "sha256:fc631183bb13e56db3158a4909908dfb2e23565286744e798241e63750e5d680", size = 46689, upload-time = "2026-06-04T08:27:49.014Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/5e/1e/7f7f299527a5a8ad90acd5f2f78dfa6c8495c6301a3205106ea68a84de96/python_multipart-0.0.31-py3-none-any.whl", hash = "sha256:8408153d68a9773291fc1da39a8b85a50044bddbabd2dd72e9229776b7b15e28", size = 29996, upload-time = "2026-06-04T08:27:47.804Z" }, -] - -[[package]] -name = "pywin32" -version = "311" -source = { registry = "https://pypi.org/simple" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a5/be/3fd5de0979fcb3994bfee0d65ed8ca9506a8a1260651b86174f6a86f52b3/pywin32-311-cp313-cp313-win32.whl", hash = "sha256:f95ba5a847cba10dd8c4d8fefa9f2a6cf283b8b88ed6178fa8a6c1ab16054d0d", size = 8705700, upload-time = "2025-07-14T20:13:26.471Z" }, - { url = "https://files.pythonhosted.org/packages/e3/28/e0a1909523c6890208295a29e05c2adb2126364e289826c0a8bc7297bd5c/pywin32-311-cp313-cp313-win_amd64.whl", hash = "sha256:718a38f7e5b058e76aee1c56ddd06908116d35147e133427e59a3983f703a20d", size = 9494700, upload-time = "2025-07-14T20:13:28.243Z" }, - { url = "https://files.pythonhosted.org/packages/04/bf/90339ac0f55726dce7d794e6d79a18a91265bdf3aa70b6b9ca52f35e022a/pywin32-311-cp313-cp313-win_arm64.whl", hash = "sha256:7b4075d959648406202d92a2310cb990fea19b535c7f4a78d3f5e10b926eeb8a", size = 8709318, upload-time = "2025-07-14T20:13:30.348Z" }, -] - [[package]] name = "pyyaml" version = "6.0.3" @@ -1927,20 +1288,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/73/e8/2bdf3ca2090f68bb3d75b44da7bbc71843b19c9f2b9cb9b0f4ab7a5a4329/pyyaml-6.0.3-cp313-cp313-win_arm64.whl", hash = "sha256:5498cd1645aa724a7c71c8f378eb29ebe23da2fc0d7a08071d89469bf1d2defb", size = 140246, upload-time = "2025-09-25T21:32:34.663Z" }, ] -[[package]] -name = "realtime" -version = "2.30.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "pydantic" }, - { name = "typing-extensions" }, - { name = "websockets" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b2/a2/0328d49d3b5fb427068e9200e7de5b0d708d021a1ad98d004bc685d2529e/realtime-2.30.0.tar.gz", hash = "sha256:7aa593da52ed5f92c34ec4e50e32043afa62f219c94f717ad64a66ab0ef9f1ba", size = 18718, upload-time = "2026-05-06T17:35:23.925Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b4/75/1b2cfc949595e22d8c05a2aa2cfc222921f7f94177d7e8a90542f3f73b33/realtime-2.30.0-py3-none-any.whl", hash = "sha256:7c93b63d2cf99aa1da4fa8826b03b00cd32f7b38abb27ff47b19eb5dcb5707c6", size = 22376, upload-time = "2026-05-06T17:35:22.568Z" }, -] - [[package]] name = "referencing" version = "0.37.0" @@ -2005,19 +1352,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/56/5d/c814546c2333ceea4ba42262d8c4d55763003e767fa169adc693bd524478/requests-2.33.0-py3-none-any.whl", hash = "sha256:3324635456fa185245e24865e810cecec7b4caf933d7eb133dcde67d48cee69b", size = 65017, upload-time = "2026-03-25T15:10:40.382Z" }, ] -[[package]] -name = "requests-oauthlib" -version = "2.0.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "oauthlib" }, - { name = "requests" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/42/f2/05f29bc3913aea15eb670be136045bf5c5bbf4b99ecb839da9b422bb2c85/requests-oauthlib-2.0.0.tar.gz", hash = "sha256:b3dffaebd884d8cd778494369603a9e7b58d29111bf6b41bdc2dcd87203af4e9", size = 55650, upload-time = "2024-03-22T20:32:29.939Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/3b/5d/63d4ae3b9daea098d5d6f5da83984853c1bbacd5dc826764b249fe119d24/requests_oauthlib-2.0.0-py2.py3-none-any.whl", hash = "sha256:7dd8a5c40426b779b0868c404bdef9768deccf22749cde15852df527e6269b36", size = 24179, upload-time = "2024-03-22T20:32:28.055Z" }, -] - [[package]] name = "rich" version = "14.3.3" @@ -2094,19 +1428,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e1/a3/03216a6a86c706df54422612981fb0f9041dbb452c3401501d4a22b942c9/ruff-0.13.0-py3-none-win_arm64.whl", hash = "sha256:ab80525317b1e1d38614addec8ac954f1b3e662de9d59114ecbf771d00cf613e", size = 12312357, upload-time = "2025-09-10T16:25:35.595Z" }, ] -[[package]] -name = "scantree" -version = "0.0.4" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "attrs" }, - { name = "pathspec" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b3/e4/40998faefc72ba1ddeb640a44fba92935353525dba110488806da8339c0b/scantree-0.0.4.tar.gz", hash = "sha256:15bd5cb24483b04db2c70653604e8ea3522e98087db7e38ab8482f053984c0ac", size = 24643, upload-time = "2024-08-03T20:08:59.413Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/93/ce/828467ddfa0d2fe473673026442d2032d552a168e42cfbf25fd0e5264e0c/scantree-0.0.4-py3-none-any.whl", hash = "sha256:7616ab65aa6b7f16fcf8e6fa1d9afaa99a27ab72bba05c61b691853b96763174", size = 20690, upload-time = "2024-08-03T20:08:58.137Z" }, -] - [[package]] name = "shellingham" version = "1.5.4" @@ -2116,33 +1437,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/e0/f9/0595336914c5619e5f28a1fb793285925a8cd4b432c9da0a987836c7f822/shellingham-1.5.4-py2.py3-none-any.whl", hash = "sha256:7ecfff8f2fd72616f7481040475a65b2bf8af90a56c89140852d1120324e8686", size = 9755, upload-time = "2023-10-24T04:13:38.866Z" }, ] -[[package]] -name = "shortuuid" -version = "1.0.13" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/8c/e2/bcf761f3bff95856203f9559baf3741c416071dd200c0fc19fad7f078f86/shortuuid-1.0.13.tar.gz", hash = "sha256:3bb9cf07f606260584b1df46399c0b87dd84773e7b25912b7e391e30797c5e72", size = 9662, upload-time = "2024-03-11T20:11:06.879Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c0/44/21d6bf170bf40b41396480d8d49ad640bca3f2b02139cd52aa1e272830a5/shortuuid-1.0.13-py3-none-any.whl", hash = "sha256:a482a497300b49b4953e15108a7913244e1bb0d41f9d332f5e9925dba33a3c5a", size = 10529, upload-time = "2024-03-11T20:11:04.807Z" }, -] - -[[package]] -name = "six" -version = "1.17.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" }, -] - -[[package]] -name = "slack-sdk" -version = "3.41.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/22/35/fc009118a13187dd9731657c60138e5a7c2dea88681a7f04dc406af5da7d/slack_sdk-3.41.0.tar.gz", hash = "sha256:eb61eb12a65bebeca9cb5d36b3f799e836ed2be21b456d15df2627cfe34076ca", size = 250568, upload-time = "2026-03-12T16:10:11.381Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a1/df/2e4be347ff98281b505cc0ccf141408cdd25eb5ca9f3830deb361b2472d3/slack_sdk-3.41.0-py2.py3-none-any.whl", hash = "sha256:bb18dcdfff1413ec448e759cf807ec3324090993d8ab9111c74081623b692a89", size = 313885, upload-time = "2026-03-12T16:10:09.811Z" }, -] - [[package]] name = "sniffio" version = "1.3.1" @@ -2173,114 +1467,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/bf/e1/3ccb13c643399d22289c6a9786c1a91e3dcbb68bce4beb44926ac2c557bf/sqlalchemy-2.0.45-py3-none-any.whl", hash = "sha256:5225a288e4c8cc2308dbdd874edad6e7d0fd38eac1e9e5f23503425c8eee20d0", size = 1936672, upload-time = "2025-12-09T21:54:52.608Z" }, ] -[[package]] -name = "sse-starlette" -version = "3.4.2" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, - { name = "starlette" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/38/82/10cdfab4ab663a6b6bd624d33f55b2cfa41af5105be033a6d5d135a92c5f/sse_starlette-3.4.2.tar.gz", hash = "sha256:2f9a7f51ed84395a0427fb9f66cb1ec11f7899d977a72cbc9070b962a2e14489", size = 35236, upload-time = "2026-05-06T19:42:13.727Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/c1/27/351c71e803c56090d8d3bf9520422debeb8ed938871fd4f7ef519805a6c5/sse_starlette-3.4.2-py3-none-any.whl", hash = "sha256:6ea5d35b7ce979a3de5a0db5f77fe886b1616e4b3e1ad93fba502bd9b5fb662f", size = 16516, upload-time = "2026-05-06T19:42:12.201Z" }, -] - -[[package]] -name = "starlette" -version = "1.3.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "anyio" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/eb/e3/7c1dc7381d9f8ab7d854328ebfa884e62cb3f3d8549ddfd37c7814f42afa/starlette-1.3.1.tar.gz", hash = "sha256:05d0213193f2fbaae60e2ecb593b4add4262ad4e46536b54abe36f11a71724e0", size = 2703240, upload-time = "2026-06-12T09:23:11.602Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ec/bb/2799cc2ede3ed41131f8975621e7213dfc7ef4acbbaadfa440f32500c370/starlette-1.3.1-py3-none-any.whl", hash = "sha256:c7372aae11c3c3f26a42df7bd626cec2f47d03483d261d369516a615a53714c6", size = 73632, upload-time = "2026-06-12T09:23:10.017Z" }, -] - -[[package]] -name = "storage3" -version = "2.30.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "deprecation" }, - { name = "httpx", extra = ["http2"] }, - { name = "pydantic" }, - { name = "pyiceberg" }, - { name = "yarl" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/9b/b2/6df208d64630744704d00f2c07197170390d6b4d0098617740f6a7a4fa98/storage3-2.30.0.tar.gz", hash = "sha256:b74e3cac149f2c0553dcb5f4d55d8c35d420d88183a1a2df77727d482665972b", size = 20162, upload-time = "2026-05-06T17:35:25.71Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/91/5c/bb8c8cc448cfae671c4ffee67f3651892ea59b341f27bed54666190eb8ef/storage3-2.30.0-py3-none-any.whl", hash = "sha256:2bd23a34011c018bd9c130d8a70a09ebd060ae80d946c6204a6fc08161ad728d", size = 28284, upload-time = "2026-05-06T17:35:24.659Z" }, -] - -[[package]] -name = "strenum" -version = "0.4.15" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/85/ad/430fb60d90e1d112a62ff57bdd1f286ec73a2a0331272febfddd21f330e1/StrEnum-0.4.15.tar.gz", hash = "sha256:878fb5ab705442070e4dd1929bb5e2249511c0bcf2b0eeacf3bcd80875c82eff", size = 23384, upload-time = "2023-06-29T22:02:58.399Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/81/69/297302c5f5f59c862faa31e6cb9a4cd74721cd1e052b38e464c5b402df8b/StrEnum-0.4.15-py3-none-any.whl", hash = "sha256:a30cda4af7cc6b5bf52c8055bc4bf4b2b6b14a93b574626da33df53cf7740659", size = 8851, upload-time = "2023-06-29T22:02:56.947Z" }, -] - -[[package]] -name = "strictyaml" -version = "1.7.3" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "python-dateutil" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/b3/08/efd28d49162ce89c2ad61a88bd80e11fb77bc9f6c145402589112d38f8af/strictyaml-1.7.3.tar.gz", hash = "sha256:22f854a5fcab42b5ddba8030a0e4be51ca89af0267961c8d6cfa86395586c407", size = 115206, upload-time = "2023-03-10T12:50:27.062Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/96/7c/a81ef5ef10978dd073a854e0fa93b5d8021d0594b639cc8f6453c3c78a1d/strictyaml-1.7.3-py3-none-any.whl", hash = "sha256:fb5c8a4edb43bebb765959e420f9b3978d7f1af88c80606c03fb420888f5d1c7", size = 123917, upload-time = "2023-03-10T12:50:17.242Z" }, -] - -[[package]] -name = "supabase" -version = "2.30.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "httpx" }, - { name = "postgrest" }, - { name = "realtime" }, - { name = "storage3" }, - { name = "supabase-auth" }, - { name = "supabase-functions" }, - { name = "yarl" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5c/a6/d2b17021c2db1a9d219c383e0762ac03a62b25468e61ab126b6b561c2f21/supabase-2.30.0.tar.gz", hash = "sha256:efdba41d474038ed220736ba4e64946df56043057ad785c4c3499d27e459975c", size = 9689, upload-time = "2026-05-06T17:35:27.781Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/f0/82/d213be7d0ce0bb18018744c0ee38ba0d6648d41dbc46ac8558cffe80541f/supabase-2.30.0-py3-none-any.whl", hash = "sha256:f9b259194554f7bfd2dca6c23261f2df588016ca18b18e774f4d85bc941edb03", size = 16634, upload-time = "2026-05-06T17:35:26.696Z" }, -] - -[[package]] -name = "supabase-auth" -version = "2.30.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "httpx", extra = ["http2"] }, - { name = "pydantic" }, - { name = "pyjwt", extra = ["crypto"] }, -] -sdist = { url = "https://files.pythonhosted.org/packages/8e/8a/48bbbe0b6703d0670b67e45b90d6a791fd01aace67443d286f760bf48895/supabase_auth-2.30.0.tar.gz", hash = "sha256:6138a53a306a95ed59c03d4e4975469dfc3343a0ade33cc4b37e4ef967ad83f8", size = 39135, upload-time = "2026-05-06T17:35:30.371Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/db/40/a99cb4373353bcbf302d962e51da9eac78b3b0f257eb0362c0852b1667f4/supabase_auth-2.30.0-py3-none-any.whl", hash = "sha256:e85e1f51ec0de2172c3a2a8514205f71731a9914f9a770ed199ac0cf054bc82c", size = 48352, upload-time = "2026-05-06T17:35:28.936Z" }, -] - -[[package]] -name = "supabase-functions" -version = "2.30.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "httpx", extra = ["http2"] }, - { name = "strenum" }, - { name = "yarl" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/f0/e6/5cd8559ec2bb332e6027840c1be292f9989c2fc7b47bf40800aec5586791/supabase_functions-2.30.0.tar.gz", hash = "sha256:025acfd25f1c000ba43d0f7b8e366b0d2e9dfc784b842528e21973eb33006113", size = 4683, upload-time = "2026-05-06T17:35:32.246Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/53/da/9dedab32775df04cc22ca72f194b78e895d940f195bed3e02882a65daa9b/supabase_functions-2.30.0-py3-none-any.whl", hash = "sha256:92419459f102767b954cd034856e4ded8e34c78660b32442d66c8b2899c68011", size = 8803, upload-time = "2026-05-06T17:35:31.342Z" }, -] - [[package]] name = "tenacity" version = "9.1.2" @@ -2342,15 +1528,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/72/f4/0de46cfa12cdcbcd464cc59fde36912af405696f687e53a091fb432f694c/tokenizers-0.22.2-cp39-abi3-win_arm64.whl", hash = "sha256:9ce725d22864a1e965217204946f830c37876eee3b2ba6fc6255e8e903d5fcbc", size = 2612133, upload-time = "2026-01-05T10:45:17.232Z" }, ] -[[package]] -name = "toml" -version = "0.10.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/be/ba/1f744cdc819428fc6b5084ec34d9b30660f6f9daaf70eead706e3203ec3c/toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f", size = 22253, upload-time = "2020-11-01T01:40:22.204Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/44/6f/7120676b6d73228c96e17f1f794d8ab046fc910d781c8d151120c3f1569e/toml-0.10.2-py2.py3-none-any.whl", hash = "sha256:806143ae5bfb6a3c6e736a764057db0e6a0e05e338b5630894a5f779cabb4f9b", size = 16588, upload-time = "2020-11-01T01:40:20.672Z" }, -] - [[package]] name = "tomlkit" version = "0.14.0" @@ -2402,21 +1579,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/c8/33/70b320d24cd127d6ca427d2bef1279830f0786a1f2cde160f59b4fb80728/tree_sitter_rust-0.24.0-cp39-abi3-win_arm64.whl", hash = "sha256:7a0538eaf4063b443c6cd80a47df19249f65e27dbdf129396a9193749912d0c0", size = 128583, upload-time = "2025-04-01T21:06:02.58Z" }, ] -[[package]] -name = "typer" -version = "0.23.1" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "annotated-doc" }, - { name = "click" }, - { name = "rich" }, - { name = "shellingham" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/fd/07/b822e1b307d40e263e8253d2384cf98c51aa2368cc7ba9a07e523a1d964b/typer-0.23.1.tar.gz", hash = "sha256:2070374e4d31c83e7b61362fd859aa683576432fd5b026b060ad6b4cd3b86134", size = 120047, upload-time = "2026-02-13T10:04:30.984Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/d5/91/9b286ab899c008c2cb05e8be99814807e7fbbd33f0c0c960470826e5ac82/typer-0.23.1-py3-none-any.whl", hash = "sha256:3291ad0d3c701cbf522012faccfbb29352ff16ad262db2139e6b01f15781f14e", size = 56813, upload-time = "2026-02-13T10:04:32.008Z" }, -] - [[package]] name = "typer-slim" version = "0.21.1" @@ -2451,24 +1613,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/dc/9b/47798a6c91d8bdb567fe2698fe81e0c6b7cb7ef4d13da4114b41d239f65d/typing_inspection-0.4.2-py3-none-any.whl", hash = "sha256:4ed1cacbdc298c220f1bd249ed5287caa16f34d44ef4e9c3d0cbad5b521545e7", size = 14611, upload-time = "2025-10-01T02:14:40.154Z" }, ] -[[package]] -name = "tzdata" -version = "2026.2" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/ba/19/1b9b0e29f30c6d35cb345486df41110984ea67ae69dddbc0e8a100999493/tzdata-2026.2.tar.gz", hash = "sha256:9173fde7d80d9018e02a662e168e5a2d04f87c41ea174b139fbef642eda62d10", size = 198254, upload-time = "2026-04-24T15:22:08.651Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/ce/e4/dccd7f47c4b64213ac01ef921a1337ee6e30e8c6466046018326977efd95/tzdata-2026.2-py2.py3-none-any.whl", hash = "sha256:bbe9af844f658da81a5f95019480da3a89415801f6cc966806612cc7169bffe7", size = 349321, upload-time = "2026-04-24T15:22:05.876Z" }, -] - -[[package]] -name = "uritemplate" -version = "4.2.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/98/60/f174043244c5306c9988380d2cb10009f91563fc4b31293d27e17201af56/uritemplate-4.2.0.tar.gz", hash = "sha256:480c2ed180878955863323eea31b0ede668795de182617fef9c6ca09e6ec9d0e", size = 33267, upload-time = "2025-06-02T15:12:06.318Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/a9/99/3ae339466c9183ea5b8ae87b34c0b897eda475d2aec2307cae60e5cd4f29/uritemplate-4.2.0-py3-none-any.whl", hash = "sha256:962201ba1c4edcab02e60f9a0d3821e82dfc5d2d6662a21abd533879bdb8a686", size = 11488, upload-time = "2025-06-02T15:12:03.405Z" }, -] - [[package]] name = "urllib3" version = "2.7.0" @@ -2478,19 +1622,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/7f/3e/5db95bcf282c52709639744ca2a8b149baccf648e39c8cc87553df9eae0c/urllib3-2.7.0-py3-none-any.whl", hash = "sha256:9fb4c81ebbb1ce9531cce37674bbc6f1360472bc18ca9a553ede278ef7276897", size = 131087, upload-time = "2026-05-07T16:13:17.151Z" }, ] -[[package]] -name = "uvicorn" -version = "0.44.0" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "click" }, - { name = "h11" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/5e/da/6eee1ff8b6cbeed47eeb5229749168e81eb4b7b999a1a15a7176e51410c9/uvicorn-0.44.0.tar.gz", hash = "sha256:6c942071b68f07e178264b9152f1f16dfac5da85880c4ce06366a96d70d4f31e", size = 86947, upload-time = "2026-04-06T09:23:22.826Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/b7/23/a5bbd9600dd607411fa644c06ff4951bec3a4d82c4b852374024359c19c0/uvicorn-0.44.0-py3-none-any.whl", hash = "sha256:ce937c99a2cc70279556967274414c087888e8cec9f9c94644dfca11bd3ced89", size = 69425, upload-time = "2026-04-06T09:23:21.524Z" }, -] - [[package]] name = "virtualenv" version = "20.36.1" @@ -2505,6 +1636,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/6a/2a/dc2228b2888f51192c7dc766106cd475f1b768c10caaf9727659726f7391/virtualenv-20.36.1-py3-none-any.whl", hash = "sha256:575a8d6b124ef88f6f51d56d656132389f961062a9177016a50e4f507bbcc19f", size = 6008258, upload-time = "2026-01-09T18:20:59.425Z" }, ] +[[package]] +name = "vulture" +version = "2.16" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/66/3e/4d08c5903b2c0c70cad583c170cc4a663fc6a61e2ad00b711fcda61358cd/vulture-2.16.tar.gz", hash = "sha256:f8d9f6e2af03011664a3c6c240c9765b3f392917d3135fddca6d6a68d359f717", size = 52680, upload-time = "2026-03-25T14:41:27.141Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/f5/be/f935130312330614811dae2ea9df3f395f6d63889eb6c2e68c14507152ee/vulture-2.16-py3-none-any.whl", hash = "sha256:6e0f1c312cef1c87856957e5c2ca9608834a7c794c2180477f30bf0e4cc58eee", size = 26993, upload-time = "2026-03-25T14:41:26.21Z" }, +] + [[package]] name = "websockets" version = "15.0.1" @@ -2525,18 +1665,6 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/fa/a8/5b41e0da817d64113292ab1f8247140aac61cbf6cfd085d6a0fa77f4984f/websockets-15.0.1-py3-none-any.whl", hash = "sha256:f7a866fbc1e97b5c617ee4116daaa09b722101d4a3c170c787450ba409f9736f", size = 169743, upload-time = "2025-03-05T20:03:39.41Z" }, ] -[[package]] -name = "werkzeug" -version = "3.1.8" -source = { registry = "https://pypi.org/simple" } -dependencies = [ - { name = "markupsafe" }, -] -sdist = { url = "https://files.pythonhosted.org/packages/dd/b2/381be8cfdee792dd117872481b6e378f85c957dd7c5bca38897b08f765fd/werkzeug-3.1.8.tar.gz", hash = "sha256:9bad61a4268dac112f1c5cd4630a56ede601b6ed420300677a869083d70a4c44", size = 875852, upload-time = "2026-04-02T18:49:14.268Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/93/8c/2e650f2afeb7ee576912636c23ddb621c91ac6a98e66dc8d29c3c69446e1/werkzeug-3.1.8-py3-none-any.whl", hash = "sha256:63a77fb8892bf28ebc3178683445222aa500e48ebad5ec77b0ad80f8726b1f50", size = 226459, upload-time = "2026-04-02T18:49:12.72Z" }, -] - [[package]] name = "xxhash" version = "3.6.0" @@ -2629,28 +1757,3 @@ sdist = { url = "https://files.pythonhosted.org/packages/e3/02/0f2892c661036d50e wheels = [ { url = "https://files.pythonhosted.org/packages/2e/54/647ade08bf0db230bfea292f893923872fd20be6ac6f53b2b936ba839d75/zipp-3.23.0-py3-none-any.whl", hash = "sha256:071652d6115ed432f5ce1d34c336c0adfd6a884660d1e9712a256d3d3bd4b14e", size = 10276, upload-time = "2025-06-08T17:06:38.034Z" }, ] - -[[package]] -name = "zstandard" -version = "0.25.0" -source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/fd/aa/3e0508d5a5dd96529cdc5a97011299056e14c6505b678fd58938792794b1/zstandard-0.25.0.tar.gz", hash = "sha256:7713e1179d162cf5c7906da876ec2ccb9c3a9dcbdffef0cc7f70c3667a205f0b", size = 711513, upload-time = "2025-09-14T22:15:54.002Z" } -wheels = [ - { url = "https://files.pythonhosted.org/packages/35/0b/8df9c4ad06af91d39e94fa96cc010a24ac4ef1378d3efab9223cc8593d40/zstandard-0.25.0-cp313-cp313-macosx_10_13_x86_64.whl", hash = "sha256:ec996f12524f88e151c339688c3897194821d7f03081ab35d31d1e12ec975e94", size = 795735, upload-time = "2025-09-14T22:17:26.042Z" }, - { url = "https://files.pythonhosted.org/packages/3f/06/9ae96a3e5dcfd119377ba33d4c42a7d89da1efabd5cb3e366b156c45ff4d/zstandard-0.25.0-cp313-cp313-macosx_11_0_arm64.whl", hash = "sha256:a1a4ae2dec3993a32247995bdfe367fc3266da832d82f8438c8570f989753de1", size = 640440, upload-time = "2025-09-14T22:17:27.366Z" }, - { url = "https://files.pythonhosted.org/packages/d9/14/933d27204c2bd404229c69f445862454dcc101cd69ef8c6068f15aaec12c/zstandard-0.25.0-cp313-cp313-manylinux2010_i686.manylinux2014_i686.manylinux_2_12_i686.manylinux_2_17_i686.whl", hash = "sha256:e96594a5537722fdfb79951672a2a63aec5ebfb823e7560586f7484819f2a08f", size = 5343070, upload-time = "2025-09-14T22:17:28.896Z" }, - { url = "https://files.pythonhosted.org/packages/6d/db/ddb11011826ed7db9d0e485d13df79b58586bfdec56e5c84a928a9a78c1c/zstandard-0.25.0-cp313-cp313-manylinux2014_aarch64.manylinux_2_17_aarch64.whl", hash = "sha256:bfc4e20784722098822e3eee42b8e576b379ed72cca4a7cb856ae733e62192ea", size = 5063001, upload-time = "2025-09-14T22:17:31.044Z" }, - { url = "https://files.pythonhosted.org/packages/db/00/87466ea3f99599d02a5238498b87bf84a6348290c19571051839ca943777/zstandard-0.25.0-cp313-cp313-manylinux2014_ppc64le.manylinux_2_17_ppc64le.whl", hash = "sha256:457ed498fc58cdc12fc48f7950e02740d4f7ae9493dd4ab2168a47c93c31298e", size = 5394120, upload-time = "2025-09-14T22:17:32.711Z" }, - { url = "https://files.pythonhosted.org/packages/2b/95/fc5531d9c618a679a20ff6c29e2b3ef1d1f4ad66c5e161ae6ff847d102a9/zstandard-0.25.0-cp313-cp313-manylinux2014_s390x.manylinux_2_17_s390x.whl", hash = "sha256:fd7a5004eb1980d3cefe26b2685bcb0b17989901a70a1040d1ac86f1d898c551", size = 5451230, upload-time = "2025-09-14T22:17:34.41Z" }, - { url = "https://files.pythonhosted.org/packages/63/4b/e3678b4e776db00f9f7b2fe58e547e8928ef32727d7a1ff01dea010f3f13/zstandard-0.25.0-cp313-cp313-manylinux2014_x86_64.manylinux_2_17_x86_64.whl", hash = "sha256:8e735494da3db08694d26480f1493ad2cf86e99bdd53e8e9771b2752a5c0246a", size = 5547173, upload-time = "2025-09-14T22:17:36.084Z" }, - { url = "https://files.pythonhosted.org/packages/4e/d5/ba05ed95c6b8ec30bd468dfeab20589f2cf709b5c940483e31d991f2ca58/zstandard-0.25.0-cp313-cp313-musllinux_1_1_aarch64.whl", hash = "sha256:3a39c94ad7866160a4a46d772e43311a743c316942037671beb264e395bdd611", size = 5046736, upload-time = "2025-09-14T22:17:37.891Z" }, - { url = "https://files.pythonhosted.org/packages/50/d5/870aa06b3a76c73eced65c044b92286a3c4e00554005ff51962deef28e28/zstandard-0.25.0-cp313-cp313-musllinux_1_1_x86_64.whl", hash = "sha256:172de1f06947577d3a3005416977cce6168f2261284c02080e7ad0185faeced3", size = 5576368, upload-time = "2025-09-14T22:17:40.206Z" }, - { url = "https://files.pythonhosted.org/packages/5d/35/398dc2ffc89d304d59bc12f0fdd931b4ce455bddf7038a0a67733a25f550/zstandard-0.25.0-cp313-cp313-musllinux_1_2_aarch64.whl", hash = "sha256:3c83b0188c852a47cd13ef3bf9209fb0a77fa5374958b8c53aaa699398c6bd7b", size = 4954022, upload-time = "2025-09-14T22:17:41.879Z" }, - { url = "https://files.pythonhosted.org/packages/9a/5c/36ba1e5507d56d2213202ec2b05e8541734af5f2ce378c5d1ceaf4d88dc4/zstandard-0.25.0-cp313-cp313-musllinux_1_2_i686.whl", hash = "sha256:1673b7199bbe763365b81a4f3252b8e80f44c9e323fc42940dc8843bfeaf9851", size = 5267889, upload-time = "2025-09-14T22:17:43.577Z" }, - { url = "https://files.pythonhosted.org/packages/70/e8/2ec6b6fb7358b2ec0113ae202647ca7c0e9d15b61c005ae5225ad0995df5/zstandard-0.25.0-cp313-cp313-musllinux_1_2_ppc64le.whl", hash = "sha256:0be7622c37c183406f3dbf0cba104118eb16a4ea7359eeb5752f0794882fc250", size = 5433952, upload-time = "2025-09-14T22:17:45.271Z" }, - { url = "https://files.pythonhosted.org/packages/7b/01/b5f4d4dbc59ef193e870495c6f1275f5b2928e01ff5a81fecb22a06e22fb/zstandard-0.25.0-cp313-cp313-musllinux_1_2_s390x.whl", hash = "sha256:5f5e4c2a23ca271c218ac025bd7d635597048b366d6f31f420aaeb715239fc98", size = 5814054, upload-time = "2025-09-14T22:17:47.08Z" }, - { url = "https://files.pythonhosted.org/packages/b2/e5/fbd822d5c6f427cf158316d012c5a12f233473c2f9c5fe5ab1ae5d21f3d8/zstandard-0.25.0-cp313-cp313-musllinux_1_2_x86_64.whl", hash = "sha256:4f187a0bb61b35119d1926aee039524d1f93aaf38a9916b8c4b78ac8514a0aaf", size = 5360113, upload-time = "2025-09-14T22:17:48.893Z" }, - { url = "https://files.pythonhosted.org/packages/8e/e0/69a553d2047f9a2c7347caa225bb3a63b6d7704ad74610cb7823baa08ed7/zstandard-0.25.0-cp313-cp313-win32.whl", hash = "sha256:7030defa83eef3e51ff26f0b7bfb229f0204b66fe18e04359ce3474ac33cbc09", size = 436936, upload-time = "2025-09-14T22:17:52.658Z" }, - { url = "https://files.pythonhosted.org/packages/d9/82/b9c06c870f3bd8767c201f1edbdf9e8dc34be5b0fbc5682c4f80fe948475/zstandard-0.25.0-cp313-cp313-win_amd64.whl", hash = "sha256:1f830a0dac88719af0ae43b8b2d6aef487d437036468ef3c2ea59c51f9d55fd5", size = 506232, upload-time = "2025-09-14T22:17:50.402Z" }, - { url = "https://files.pythonhosted.org/packages/d4/57/60c3c01243bb81d381c9916e2a6d9e149ab8627c0c7d7abb2d73384b3c0c/zstandard-0.25.0-cp313-cp313-win_arm64.whl", hash = "sha256:85304a43f4d513f5464ceb938aa02c1e78c2943b29f44a750b48b25ac999a049", size = 462671, upload-time = "2025-09-14T22:17:51.533Z" }, -]