diff --git a/toolkit/Makefile b/toolkit/Makefile index f65636fe78d..cf129ae8209 100644 --- a/toolkit/Makefile +++ b/toolkit/Makefile @@ -33,6 +33,8 @@ TEST_RUN_LIST ?= TEST_RERUN_LIST ?= ##help:var:TEST_IGNORE_LIST:=List of space-separated spec folders to ignore for package tests. Must not overlap with "TEST_RERUN_LIST", may overlap with "TEST_RUN_LIST". Example: TEST_IGNORE_LIST="acl". TEST_IGNORE_LIST ?= +##help:var:EXTRA_MACROS_FILES: =Space separated list of additional files whose contents will be appended to the versions macro file used during package builds. Example: EXTRA_MACROS_FILES="file1 file2". +EXTRA_MACROS_FILES ?= ######## SET INCREMENTAL BUILD FLAGS ######## @@ -168,7 +170,7 @@ endif VALIDATE_IMAGE_GPG ?= n # Default GPG keys for package GPG validation, used with VALIDATE_TOOLCHAIN_GPG and VALIDATE_IMAGE_GPG -default_gpg_keys := $(wildcard $(PROJECT_ROOT)/SPECS/azurelinux-repos/MICROSOFT-*-GPG-KEY) $(wildcard $(toolkit_root)/repos/MICROSOFT-*-GPG-KEY) +default_gpg_keys := $(strip $(wildcard $(PROJECT_ROOT)/SPECS/azurelinux-repos/MICROSOFT-*-GPG-KEY) $(wildcard $(toolkit_root)/repos/MICROSOFT-*-GPG-KEY)) TOOLCHAIN_GPG_VALIDATION_KEYS ?= $(default_gpg_keys) IMAGE_GPG_VALIDATION_KEYS ?= $(default_gpg_keys) diff --git a/toolkit/docs/building/building.md b/toolkit/docs/building/building.md index 951dd2f4e6b..f878584ade8 100644 --- a/toolkit/docs/building/building.md +++ b/toolkit/docs/building/building.md @@ -814,6 +814,7 @@ To reproduce an ISO build, run the same make invocation as before, but set: | TEST_RUN_LIST | | Explicit list of packages to test. The package test will be skipped if the build system thinks it is already up-to-date. The argument accepts both spec and package names. Example: for `python-werkzeug.spec`, which builds the `python3-werkzeug` package both `python-werkzeug` and `python3-werkzeug` are correct. | TEST_RERUN_LIST | | Always test these package, even if it its corresponding package is up-to-date. The argument accepts both spec and package names. Example: for `python-werkzeug.spec`, which builds the `python3-werkzeug` package both `python-werkzeug` and `python3-werkzeug` are correct. | TEST_IGNORE_LIST | | Ignore testing these packages. Ignoring and forcing the same test re-run is invalid and will fail the build. The argument accepts both spec and package names. Example: for `python-werkzeug.spec`, which builds the `python3-werkzeug` package both `python-werkzeug` and `python3-werkzeug` are correct. +| EXTRA_MACROS_FILES | | Space separated list of additional `` containing additional RPM macros, which will be available to the build. Used to resolve versions of kernel Out of Tree Module packages. --- diff --git a/toolkit/docs/how_it_works/3_package_building.md b/toolkit/docs/how_it_works/3_package_building.md index 387e2fbc63d..9153b04083c 100644 --- a/toolkit/docs/how_it_works/3_package_building.md +++ b/toolkit/docs/how_it_works/3_package_building.md @@ -117,6 +117,10 @@ The files can be ingested into the `graphviz` tools to visualize them, although dot -Tpng -o visualized.png < graph.dot ``` +### Dynamic versioning +We have a versionsprocessor tool that iterates over all Specs and writes their release and versions into a macro file in a format of +`azl__release`, `azl__version`, note that the `` needs any `-` replaced with `_` due to macros not allowing `-`. + ### Stage 1: Grapher The `grapher` tool reads the `specs.json` file and converts it into an acyclic directed graph. Inter-package dependencies are represented by directed edges in the graph. diff --git a/toolkit/scripts/pkggen.mk b/toolkit/scripts/pkggen.mk index aaed4f211f5..e907675f411 100644 --- a/toolkit/scripts/pkggen.mk +++ b/toolkit/scripts/pkggen.mk @@ -33,6 +33,7 @@ validate-pkggen-config = $(STATUS_FLAGS_DIR)/validate-image-config-pkggen.flag # Outputs specs_file = $(PKGBUILD_DIR)/specs.json +rel_versions_macro_file = $(PKGBUILD_DIR)/macros.releaseversions graph_file = $(PKGBUILD_DIR)/graph.dot cached_file = $(PKGBUILD_DIR)/cached_graph.dot preprocessed_file = $(PKGBUILD_DIR)/preprocessed_graph.dot @@ -50,6 +51,8 @@ $(call create_folder,$(rpmbuilding_logs_dir)) parse-specs: $(specs_file) ##help:target:graph-cache=Resolve package dependencies and cache the results. graph-cache: $(cached_file) +##help:target:generate-versions-macros-file=Generate a macros file containing version and release macros for all specs. +generate-versions-macros-file: $(rel_versions_macro_file) ##help:target:graph=Create the initial package build graph. workplan graph: $(graph_file) graph-preprocessed: $(preprocessed_file) @@ -89,10 +92,29 @@ analyze-built-graph: $(go-graphanalytics) exit 1; \ fi +# Parse all SPECS in $(SPECS_DIR) and generate a release versions macros file containing macros of spec file versions and release. +$(rel_versions_macro_file): $(chroot_worker) $(SPECS_DIR) $(build_specs) $(build_spec_dirs) $(go-versionsprocessor) + $(go-versionsprocessor) \ + --dir $(SPECS_DIR) \ + --dist-tag $(DIST_TAG) \ + $(logging_command) \ + --build-dir $(parse_working_dir) \ + --worker-tar $(chroot_worker) \ + --cpu-prof-file=$(PROFILE_DIR)/versionsprocessor.cpu.pprof \ + --mem-prof-file=$(PROFILE_DIR)/versionsprocessor.mem.pprof \ + --trace-file=$(PROFILE_DIR)/versionsprocessor.trace \ + $(if $(EXTRA_MACROS_FILES),$(foreach file,$(EXTRA_MACROS_FILES),--extra-macros-file=$(file))) \ + $(if $(filter y,$(ENABLE_CPU_PROFILE)),--enable-cpu-prof) \ + $(if $(filter y,$(ENABLE_MEM_PROFILE)),--enable-mem-prof) \ + $(if $(filter y,$(ENABLE_TRACE)),--enable-trace) \ + --timestamp-file=$(TIMESTAMP_DIR)/versionsprocessor.jsonl \ + $(if $(TARGET_ARCH),--target-arch="$(TARGET_ARCH)") \ + --output $@ + # Parse specs in $(SPECS_DIR) and generate a specs.json file encoding all dependency information # We look at the same pack list as the srpmpacker tool via the target $(SRPM_PACK_LIST) if it is set. # We only parse the spec files we will actually pack. -$(specs_file): $(chroot_worker) $(SPECS_DIR) $(build_specs) $(build_spec_dirs) $(go-specreader) $(depend_SPECS_DIR) $(depend_SRPM_PACK_LIST) $(depend_RUN_CHECK) +$(specs_file): $(rel_versions_macro_file) $(chroot_worker) $(SPECS_DIR) $(build_specs) $(build_spec_dirs) $(go-specreader) $(depend_SPECS_DIR) $(depend_SRPM_PACK_LIST) $(depend_RUN_CHECK) $(go-specreader) \ --dir $(SPECS_DIR) \ $(if $(SRPM_PACK_LIST),--spec-list="$(SRPM_PACK_LIST)") \ @@ -103,6 +125,7 @@ $(specs_file): $(chroot_worker) $(SPECS_DIR) $(build_specs) $(build_spec_dirs) $ --toolchain-rpms-dir="$(TOOLCHAIN_RPMS_DIR)" \ --dist-tag $(DIST_TAG) \ --worker-tar $(chroot_worker) \ + --versions-macro-file $(rel_versions_macro_file) \ $(if $(filter y,$(RUN_CHECK)),--run-check) \ $(logging_command) \ --cpu-prof-file=$(PROFILE_DIR)/specreader.cpu.pprof \ @@ -297,7 +320,7 @@ $(RPMS_DIR): @touch $@ endif -$(STATUS_FLAGS_DIR)/build-rpms.flag: $(no_repo_acl) $(preprocessed_file) $(chroot_worker) $(go-scheduler) $(go-pkgworker) $(depend_STOP_ON_PKG_FAIL) $(CONFIG_FILE) $(depend_CONFIG_FILE) $(depend_PACKAGE_BUILD_LIST) $(depend_PACKAGE_REBUILD_LIST) $(depend_PACKAGE_IGNORE_LIST) $(depend_MAX_CASCADING_REBUILDS) $(depend_TEST_RUN_LIST) $(depend_TEST_RERUN_LIST) $(depend_TEST_IGNORE_LIST) $(pkggen_rpms) $(srpms) $(BUILD_SRPMS_DIR) $(depend_EXTRA_BUILD_LAYERS) $(depend_LICENSE_CHECK_MODE) +$(STATUS_FLAGS_DIR)/build-rpms.flag: $(rel_versions_macro_file) $(no_repo_acl) $(preprocessed_file) $(chroot_worker) $(go-scheduler) $(go-pkgworker) $(depend_STOP_ON_PKG_FAIL) $(CONFIG_FILE) $(depend_CONFIG_FILE) $(depend_PACKAGE_BUILD_LIST) $(depend_PACKAGE_REBUILD_LIST) $(depend_PACKAGE_IGNORE_LIST) $(depend_MAX_CASCADING_REBUILDS) $(depend_TEST_RUN_LIST) $(depend_TEST_RERUN_LIST) $(depend_TEST_IGNORE_LIST) $(pkggen_rpms) $(srpms) $(BUILD_SRPMS_DIR) $(depend_EXTRA_BUILD_LAYERS) $(depend_LICENSE_CHECK_MODE) $(go-scheduler) \ --input="$(preprocessed_file)" \ --output="$(built_file)" \ @@ -315,6 +338,7 @@ $(STATUS_FLAGS_DIR)/build-rpms.flag: $(no_repo_acl) $(preprocessed_file) $(chroo --distro-release-version="$(RELEASE_VERSION)" \ --distro-build-number="$(BUILD_NUMBER)" \ --rpmmacros-file="$(TOOLCHAIN_MANIFESTS_DIR)/macros.override" \ + --versions-macro-file="$(rel_versions_macro_file)" \ --build-attempts="$$(($(PACKAGE_BUILD_RETRIES)+1))" \ --check-attempts="$$(($(CHECK_BUILD_RETRIES)+1))" \ $(if $(MAX_CASCADING_REBUILDS),--max-cascading-rebuilds="$(MAX_CASCADING_REBUILDS)") \ diff --git a/toolkit/scripts/srpm_pack.mk b/toolkit/scripts/srpm_pack.mk index dca22888c7f..34eea4246f4 100644 --- a/toolkit/scripts/srpm_pack.mk +++ b/toolkit/scripts/srpm_pack.mk @@ -13,9 +13,9 @@ # update - Check signatures and updating any mismatches in the signatures file SRPM_FILE_SIGNATURE_HANDLING ?= enforce -SRPM_BUILD_CHROOT_DIR = $(BUILD_DIR)/SRPM_packaging -SRPM_BUILD_LOGS_DIR = $(LOGS_DIR)/pkggen/srpms - +SRPM_BUILD_CHROOT_DIR = $(BUILD_DIR)/SRPM_packaging +SRPM_BUILD_LOGS_DIR = $(LOGS_DIR)/pkggen/srpms +rel_versions_macro_file = $(PKGBUILD_DIR)/macros.releaseversions # Configure the list of packages we want to process into SRPMs # Strip any whitespace from user input and reasign using override so we can compare it with the empty string override SRPM_PACK_LIST := $(strip $(SRPM_PACK_LIST)) @@ -77,7 +77,7 @@ $(STATUS_FLAGS_DIR)/build_srpms.flag: $(local_specs) $(local_spec_dirs) $(local_ $(STATUS_FLAGS_DIR)/build_toolchain_srpms.flag: $(STATUS_FLAGS_DIR)/build_srpms.flag @touch $@ else -$(STATUS_FLAGS_DIR)/build_srpms.flag: $(chroot_worker) $(local_specs) $(local_spec_dirs) $(SPECS_DIR) $(go-srpmpacker) $(depend_SRPM_PACK_LIST) $(local_spec_sources) +$(STATUS_FLAGS_DIR)/build_srpms.flag: $(rel_versions_macro_file) $(chroot_worker) $(local_specs) $(local_spec_dirs) $(SPECS_DIR) $(go-srpmpacker) $(depend_SRPM_PACK_LIST) $(local_spec_sources) GODEBUG=netdns=go $(go-srpmpacker) \ --dir=$(SPECS_DIR) \ --output-dir=$(BUILD_SRPMS_DIR) \ @@ -88,6 +88,7 @@ $(STATUS_FLAGS_DIR)/build_srpms.flag: $(chroot_worker) $(local_specs) $(local_sp --tls-cert=$(TLS_CERT) \ --tls-key=$(TLS_KEY) \ --build-dir=$(SRPM_BUILD_CHROOT_DIR) \ + --versions-macro-file=$(rel_versions_macro_file) \ --signature-handling=$(SRPM_FILE_SIGNATURE_HANDLING) \ --worker-tar=$(chroot_worker) \ $(if $(filter y,$(RUN_CHECK)),--run-check) \ diff --git a/toolkit/scripts/toolkit.mk b/toolkit/scripts/toolkit.mk index aa8fe439708..7392072490c 100644 --- a/toolkit/scripts/toolkit.mk +++ b/toolkit/scripts/toolkit.mk @@ -25,6 +25,7 @@ rpms_snapshot_dir_name = rpms_snapshots rpms_snapshot_build_dir = $(BUILD_DIR)/$(rpms_snapshot_dir_name) rpms_snapshot_logs_path = $(LOGS_DIR)/$(rpms_snapshot_dir_name)/rpms_snapshot.log rpms_snapshot_per_specs = $(rpms_snapshot_build_dir)/$(specs_dir_name)_$(rpms_snapshot_name) +rel_versions_macro_file = $(PKGBUILD_DIR)/macros.releaseversions valid_arch_spec_names_build_dir = $(BUILD_DIR)/valid_arch_spec_names valid_arch_spec_names = $(valid_arch_spec_names_build_dir)/valid_arch_spec_names.txt @@ -75,7 +76,7 @@ $(toolkit_archive_versioned_compressed): $(toolkit_archive) $(rpms_snapshot) $(d tar --update -f $(toolkit_archive_versioned) -C $(toolkit_build_dir) $(toolkit_release_file_relative_path) $(toolkit_rpms_snapshot_file_relative_path) && \ $(ARCHIVE_TOOL) --best -c $(toolkit_archive_versioned) > $(toolkit_archive_versioned_compressed) -$(toolkit_archive): $(go_tool_targets) $(mariner_repos_files) $(toolkit_component_extra_files) $(toolkit_root_files) +$(toolkit_archive): $(go_tool_targets) $(mariner_repos_files) $(toolkit_component_extra_files) $(toolkit_root_files) $(rel_versions_macro_file) rm -rf $(toolkit_prep_dir) && \ mkdir -p $(toolkit_prep_dir) && \ mkdir -p $(toolkit_repos_dir) && \ @@ -83,6 +84,7 @@ $(toolkit_archive): $(go_tool_targets) $(mariner_repos_files) $(toolkit_componen cp -r $(toolkit_root_files) $(toolkit_prep_dir) && \ cp $(mariner_repos_files) $(toolkit_repos_dir) && \ cp $(toolkit_component_extra_files) $(toolkit_prep_dir) && \ + cp $(rel_versions_macro_file) $(toolkit_prep_dir) && \ cp $(go_tool_targets) $(toolkit_tools_dir) && \ rm -rf $(toolkit_prep_dir)/out && \ tar -cvp -f $(toolkit_archive) -C $(dir $(toolkit_prep_dir)) $(notdir $(toolkit_prep_dir)) @@ -93,12 +95,13 @@ rpms-snapshot: $(rpms_snapshot) $(rpms_snapshot): $(rpms_snapshot_per_specs) $(depend_SPECS_DIR) cp $(rpms_snapshot_per_specs) $(rpms_snapshot) -$(rpms_snapshot_per_specs): $(go-rpmssnapshot) $(chroot_worker) $(local_specs) $(local_spec_dirs) $(SPECS_DIR) +$(rpms_snapshot_per_specs): $(go-rpmssnapshot) $(chroot_worker) $(local_specs) $(local_spec_dirs) $(SPECS_DIR) $(rel_versions_macro_file) @mkdir -p "$(rpms_snapshot_build_dir)" $(go-rpmssnapshot) \ --input="$(SPECS_DIR)" \ --output="$(rpms_snapshot_per_specs)" \ --build-dir="$(rpms_snapshot_build_dir)" \ + --versions-macro-file $(rel_versions_macro_file) \ --dist-tag=$(DIST_TAG) \ --worker-tar="$(chroot_worker)" \ --log-level=$(LOG_LEVEL) \ @@ -114,13 +117,14 @@ run-specarchchecker: $(valid_arch_spec_names) @cat $(valid_arch_spec_names) && echo "" # File doesn't have a newline at the end, so add one via echo. @echo "Valid arch spec names generated under '$(valid_arch_spec_names)'." -$(valid_arch_spec_names): $(go-specarchchecker) $(chroot_worker) $(local_specs) $(local_spec_dirs) $(SPECS_DIR) $(depend_PACKAGE_BUILD_LIST) $(depend_PACKAGE_REBUILD_LIST) +$(valid_arch_spec_names): $(go-specarchchecker) $(chroot_worker) $(local_specs) $(local_spec_dirs) $(SPECS_DIR) $(depend_PACKAGE_BUILD_LIST) $(depend_PACKAGE_REBUILD_LIST) $(rel_versions_macro_file) $(go-specarchchecker) \ --input="$(SPECS_DIR)" \ --output="$@" \ --packages="$(PACKAGE_BUILD_LIST)" \ --rebuild-packages="$(PACKAGE_REBUILD_LIST)" \ --build-dir="$(valid_arch_spec_names_build_dir)" \ + --versions-macro-file $(rel_versions_macro_file) \ $(if $(filter y,$(RUN_CHECK)),--test-only) \ --dist-tag=$(DIST_TAG) \ --worker-tar="$(chroot_worker)" \ diff --git a/toolkit/scripts/tools.mk b/toolkit/scripts/tools.mk index 3292d0c218e..feecf1c369e 100644 --- a/toolkit/scripts/tools.mk +++ b/toolkit/scripts/tools.mk @@ -62,6 +62,7 @@ go_tool_list = \ scheduler \ specarchchecker \ specreader \ + versionsprocessor \ srpmpacker \ validatechroot \ diff --git a/toolkit/tools/internal/safechroot/chrootinterface.go b/toolkit/tools/internal/safechroot/chrootinterface.go index 1cfe566b899..09c24f103bf 100644 --- a/toolkit/tools/internal/safechroot/chrootinterface.go +++ b/toolkit/tools/internal/safechroot/chrootinterface.go @@ -8,4 +8,5 @@ type ChrootInterface interface { Run(toRun func() error) error UnsafeRun(toRun func() error) error AddFiles(filesToCopy ...FileToCopy) error + AddRPMMacrosFile(macrosFilePath string) error } diff --git a/toolkit/tools/internal/safechroot/dummychroot.go b/toolkit/tools/internal/safechroot/dummychroot.go index f094472e1f5..4894fe3894f 100644 --- a/toolkit/tools/internal/safechroot/dummychroot.go +++ b/toolkit/tools/internal/safechroot/dummychroot.go @@ -23,3 +23,7 @@ func (d *DummyChroot) UnsafeRun(toRun func() error) (err error) { func (d *DummyChroot) AddFiles(filesToCopy ...FileToCopy) (err error) { return AddFilesToDestination(d.RootDir(), filesToCopy...) } + +func (d *DummyChroot) AddRPMMacrosFile(macrosFilePath string) error { + return AddRPMMacrosFile(d, macrosFilePath) +} diff --git a/toolkit/tools/internal/safechroot/safechroot.go b/toolkit/tools/internal/safechroot/safechroot.go index 72c409df33e..0cea950085d 100644 --- a/toolkit/tools/internal/safechroot/safechroot.go +++ b/toolkit/tools/internal/safechroot/safechroot.go @@ -14,9 +14,11 @@ import ( "time" "github.com/microsoft/azurelinux/toolkit/tools/internal/buildpipeline" + "github.com/microsoft/azurelinux/toolkit/tools/internal/directory" "github.com/microsoft/azurelinux/toolkit/tools/internal/file" "github.com/microsoft/azurelinux/toolkit/tools/internal/logger" "github.com/microsoft/azurelinux/toolkit/tools/internal/retry" + "github.com/microsoft/azurelinux/toolkit/tools/internal/rpm" "github.com/microsoft/azurelinux/toolkit/tools/internal/shell" "github.com/microsoft/azurelinux/toolkit/tools/internal/systemdependency" @@ -160,6 +162,60 @@ func NewOverlayMountPoint(chrootDir, source, target, lowerDir, upperDir, workDir return } +func (c *Chroot) AddRPMMacrosFile(macrosFilePath string) error { + return AddRPMMacrosFile(c, macrosFilePath) +} + +func AddRPMMacrosFile(c ChrootInterface, macrosFilePath string) (err error) { + var macroDir string + + doGetMacroDir := func() error { + var macroErr error + + macroDir, macroErr = rpm.GetMacroDir() + if macroErr != nil { + logger.Log.Errorf("Failed to get RPM macro directory: %s", macroErr) + return macroErr + } + + return macroErr + } + + c.Run(doGetMacroDir) + + // Destination path inside the chroot (same path as on the host). + macrosDestDir := filepath.Join(c.RootDir(), macroDir) + macrosDestFile := filepath.Join(macrosDestDir, filepath.Base(macrosFilePath)) + + exists, existsErr := file.PathExists(macrosDestFile) + if existsErr != nil { + logger.Log.Errorf("Failed to check if macros file exists (%s): %s", macrosDestFile, existsErr) + return existsErr + } + + if exists { + logger.Log.Warningf("Macros file (%s) already exists, skipping copy", macrosDestFile) + return nil + } + + // Ensure destination directory exists and copy the file. + mkdirErr := directory.EnsureDirExists(macrosDestDir) + if mkdirErr != nil { + logger.Log.Errorf("Failed to create macros directory inside chroot (%s): %s", macrosDestDir, mkdirErr) + return mkdirErr + } + + copyErr := file.Copy(macrosFilePath, macrosDestFile) + if copyErr != nil { + logger.Log.Errorf("Failed to copy release version macros file into chroot (%s -> %s): %s", macrosFilePath, macrosDestFile, copyErr) + return copyErr + } + + logger.Log.Infof("Copied release version macros file into chroot (%s -> %s)", macrosFilePath, macrosDestFile) + + return +} + // GetSource gets the source device of the mount. func (m *MountPoint) GetSource() string { return m.source @@ -209,7 +265,7 @@ func NewChroot(rootDir string, isExistingDir bool) *Chroot { // This call will block until the chroot initializes successfully. // Only one Chroot will initialize at a given time. func (c *Chroot) Initialize(tarPath string, extraDirectories []string, extraMountPoints []*MountPoint, - includeDefaultMounts bool, + includeDefaultMounts bool, releaseVersionMacrosFile ...string, ) (err error) { // On failed initialization, cleanup all chroot files const leaveChrootOnDisk = false @@ -319,6 +375,16 @@ func (c *Chroot) Initialize(tarPath string, extraDirectories []string, extraMoun activeChroots = append(activeChroots, c) } + // If a release version macros file is provided, copy it into the default RPM macros directory + // inside the chroot so rpmspec/rpmbuild pick it up automatically. + if len(releaseVersionMacrosFile) > 0 && releaseVersionMacrosFile[0] != "" { + err = c.AddRPMMacrosFile(releaseVersionMacrosFile[0]) + if err != nil { + err = fmt.Errorf("failed to add release version macros file to chroot:\n%w", err) + return + } + } + return } diff --git a/toolkit/tools/pkg/licensecheck/licensecheck.go b/toolkit/tools/pkg/licensecheck/licensecheck.go index 2321cee7130..62d62bc61f0 100644 --- a/toolkit/tools/pkg/licensecheck/licensecheck.go +++ b/toolkit/tools/pkg/licensecheck/licensecheck.go @@ -74,7 +74,9 @@ func New(buildDirPath, workerTarPath, rpmDirPath, nameFilePath, exceptionFilePat jobSemaphore: make(chan struct{}, runtime.NumCPU()*2), } - err = newLicenseChecker.simpleToolChroot.InitializeChroot(buildDirPath, chrootName, workerTarPath, rpmDirPath) + noExtraMacrosFile := "" + + err = newLicenseChecker.simpleToolChroot.InitializeChroot(buildDirPath, chrootName, workerTarPath, rpmDirPath, noExtraMacrosFile) if err != nil { err = fmt.Errorf("failed to initialize chroot:\n%w", err) return newLicenseChecker, err diff --git a/toolkit/tools/pkg/rpmssnapshot/rpmssnapshot.go b/toolkit/tools/pkg/rpmssnapshot/rpmssnapshot.go index e92373ecf92..3f000fee67a 100644 --- a/toolkit/tools/pkg/rpmssnapshot/rpmssnapshot.go +++ b/toolkit/tools/pkg/rpmssnapshot/rpmssnapshot.go @@ -59,10 +59,10 @@ type SnapshotGenerator struct { } // New creates a new snapshot generator. If the chroot is created successfully, the caller is responsible for calling CleanUp(). -func New(buildDirPath, workerTarPath, specsDirPath string) (newSnapshotGenerator *SnapshotGenerator, err error) { +func New(buildDirPath, workerTarPath, specsDirPath, releaseVersionMacrosFile string) (newSnapshotGenerator *SnapshotGenerator, err error) { const chrootName = "rpmssnapshot_chroot" newSnapshotGenerator = &SnapshotGenerator{} - err = newSnapshotGenerator.simpleToolChroot.InitializeChroot(buildDirPath, chrootName, workerTarPath, specsDirPath) + err = newSnapshotGenerator.simpleToolChroot.InitializeChroot(buildDirPath, chrootName, workerTarPath, specsDirPath, releaseVersionMacrosFile) return newSnapshotGenerator, err } diff --git a/toolkit/tools/pkg/simpletoolchroot/simpletoolchroot.go b/toolkit/tools/pkg/simpletoolchroot/simpletoolchroot.go index 27ae8c5451a..1a2a510a43c 100644 --- a/toolkit/tools/pkg/simpletoolchroot/simpletoolchroot.go +++ b/toolkit/tools/pkg/simpletoolchroot/simpletoolchroot.go @@ -45,7 +45,7 @@ func (s *SimpleToolChroot) ChrootRelativeMountDir() string { // - chrootName: The name of the chroot to create // - workerTarPath: The path to the tar file containing the worker files // - mountDirPath: The path to the directory to mount, will be mounted to s.ChrootRelativeMountDir() inside the chroot -func (s *SimpleToolChroot) InitializeChroot(buildDir, chrootName, workerTarPath, mountDirPath string) (err error) { +func (s *SimpleToolChroot) InitializeChroot(buildDir, chrootName, workerTarPath, mountDirPath, releaseVersionMacrosFile string) (err error) { const ( existingDir = false ) @@ -57,7 +57,7 @@ func (s *SimpleToolChroot) InitializeChroot(buildDir, chrootName, workerTarPath, extraMountPoints := []*safechroot.MountPoint{ safechroot.NewMountPoint(mountDirPath, chrootMountDirPath, "", safechroot.BindMountPointFlags, ""), } - err = s.chroot.Initialize(workerTarPath, extraDirectories, extraMountPoints, true) + err = s.chroot.Initialize(workerTarPath, extraDirectories, extraMountPoints, true, releaseVersionMacrosFile) if err != nil { err = fmt.Errorf("failed to initialize chroot (%s) inside (%s):\n%w", workerTarPath, chrootDirPath, err) return diff --git a/toolkit/tools/pkg/specarchchecker/specarchchecker.go b/toolkit/tools/pkg/specarchchecker/specarchchecker.go index e51b94c325b..50145a05ea3 100644 --- a/toolkit/tools/pkg/specarchchecker/specarchchecker.go +++ b/toolkit/tools/pkg/specarchchecker/specarchchecker.go @@ -21,10 +21,10 @@ type ArchChecker struct { } // New creates an ArchChecker. If the chroot is created successfully, the caller is responsible for calling CleanUp(). -func New(buildDirPath, workerTarPath, specsDirPath string) (newArchChecker *ArchChecker, err error) { +func New(buildDirPath, workerTarPath, specsDirPath, releaseVersionMacrosFile string) (newArchChecker *ArchChecker, err error) { const chrootName = "specarchchecker_chroot" newArchChecker = &ArchChecker{} - err = newArchChecker.simpleToolChroot.InitializeChroot(buildDirPath, chrootName, workerTarPath, specsDirPath) + err = newArchChecker.simpleToolChroot.InitializeChroot(buildDirPath, chrootName, workerTarPath, specsDirPath, releaseVersionMacrosFile) return newArchChecker, err } diff --git a/toolkit/tools/pkg/specreaderutils/specreaderutil.go b/toolkit/tools/pkg/specreaderutils/specreaderutil.go index e843bd818c7..660c39c2eb3 100644 --- a/toolkit/tools/pkg/specreaderutils/specreaderutil.go +++ b/toolkit/tools/pkg/specreaderutils/specreaderutil.go @@ -50,6 +50,29 @@ var ( } ) +// ChrootConfig holds the configuration for creating a chroot environment. +type ChrootConfig struct { + SrpmsDir string + ReleaseVersionMacrosFile string +} + +// ChrootOption is a functional option for configuring chroot creation. +type ChrootOption func(*ChrootConfig) + +// WithSrpmsDir sets the SRPM directory to mount inside the chroot. +func WithSrpmsDir(srpmsDir string) ChrootOption { + return func(c *ChrootConfig) { + c.SrpmsDir = srpmsDir + } +} + +// WithReleaseVersionMacrosFile sets the release version macros file to copy into the chroot. +func WithReleaseVersionMacrosFile(file string) ChrootOption { + return func(c *ChrootConfig) { + c.ReleaseVersionMacrosFile = file + } +} + // parseResult holds the worker results from parsing a SPEC file. type parseResult struct { packages []*pkgjson.Package @@ -58,7 +81,8 @@ type parseResult struct { // ParseSPECsWrapper wraps parseSPECs to conditionally run it inside a chroot. // If workerTar is non-empty, parsing will occur inside a chroot, otherwise it will run on the host system. -func ParseSPECsWrapper(buildDir, specsDir, rpmsDir, srpmsDir, toolchainDir, distTag, outputFile, workerTar, targetArch string, specListSet map[string]bool, toolchainRPMs []string, workers int, runCheck bool) (err error) { +// releaseVersionMacrosFile, if non-empty, is made available inside the chroot at the same path as on the host. +func ParseSPECsWrapper(buildDir, specsDir, rpmsDir, srpmsDir, toolchainDir, distTag, outputFile, workerTar, releaseVersionMacrosFile, targetArch string, specListSet map[string]bool, toolchainRPMs []string, workers int, runCheck bool) (err error) { var ( chroot *safechroot.Chroot packageRepo *pkgjson.PackageRepo @@ -66,7 +90,7 @@ func ParseSPECsWrapper(buildDir, specsDir, rpmsDir, srpmsDir, toolchainDir, dist if workerTar != "" { const leaveFilesOnDisk = false - chroot, err = createChroot(workerTar, buildDir, specsDir, srpmsDir) + chroot, err = CreateChroot("specparser_chroot", workerTar, buildDir, specsDir, WithSrpmsDir(srpmsDir), WithReleaseVersionMacrosFile(releaseVersionMacrosFile)) if err != nil { return } @@ -125,14 +149,22 @@ func ParseSPECsWrapper(buildDir, specsDir, rpmsDir, srpmsDir, toolchainDir, dist return } -// createChroot creates a chroot to parse SPECs inside of. -func createChroot(workerTar, buildDir, specsDir, srpmsDir string) (chroot *safechroot.Chroot, err error) { +// CreateChroot creates a chroot to parse SPECs inside of. +// Required parameters are chrootName, workerTar, buildDir, and specsDir. +// Optional configuration can be provided via ChrootOption functions: +// - WithSrpmsDir: sets the SRPM directory to mount inside the chroot. +// - WithReleaseVersionMacrosFile: sets the release version macros file to copy into the chroot. +func CreateChroot(chrootName, workerTar, buildDir, specsDir string, opts ...ChrootOption) (chroot *safechroot.Chroot, err error) { const ( - chrootName = "specparser_chroot" existingDir = false leaveFilesOnDisk = false ) + cfg := &ChrootConfig{} + for _, opt := range opts { + opt(cfg) + } + // Mount the specs and srpms directories to an identical path inside the chroot. // Since specreader saves the full paths to specs in its output that grapher will then consume, // the pathing needs to be preserved from the host system. @@ -140,20 +172,23 @@ func createChroot(workerTar, buildDir, specsDir, srpmsDir string) (chroot *safec extraMountPoints := []*safechroot.MountPoint{ safechroot.NewMountPoint(specsDir, specsDir, "", safechroot.BindMountPointFlags, ""), - safechroot.NewMountPoint(srpmsDir, srpmsDir, "", safechroot.BindMountPointFlags, ""), + } + + if cfg.SrpmsDir != "" { + extraMountPoints = append(extraMountPoints, safechroot.NewMountPoint(cfg.SrpmsDir, cfg.SrpmsDir, "", safechroot.BindMountPointFlags, "")) } chrootDir := filepath.Join(buildDir, chrootName) chroot = safechroot.NewChroot(chrootDir, existingDir) - err = chroot.Initialize(workerTar, extraDirectories, extraMountPoints, true) + err = chroot.Initialize(workerTar, extraDirectories, extraMountPoints, true, cfg.ReleaseVersionMacrosFile) if err != nil { return } // If this is not a regular build then copy in all of the SPECs since there are no bind mounts. if !buildpipeline.IsRegularBuild() { - dirsToCopy := []string{specsDir, srpmsDir} + dirsToCopy := []string{specsDir, cfg.SrpmsDir} for _, dir := range dirsToCopy { dirInChroot := filepath.Join(chroot.RootDir(), dir) err = directory.CopyContents(dir, dirInChroot) diff --git a/toolkit/tools/pkgworker/pkgworker.go b/toolkit/tools/pkgworker/pkgworker.go index 62e259d754c..7c0eef8ead5 100644 --- a/toolkit/tools/pkgworker/pkgworker.go +++ b/toolkit/tools/pkgworker/pkgworker.go @@ -37,29 +37,30 @@ const ( ) var ( - app = kingpin.New("pkgworker", "A worker for building packages locally") - srpmFile = exe.InputFlag(app, "Full path to the SRPM to build") - workDir = app.Flag("work-dir", "The directory to create the build folder").Required().String() - workerTar = app.Flag("worker-tar", "Full path to worker_chroot.tar.gz").Required().ExistingFile() - repoFile = app.Flag("repo-file", "Full path to local.repo").Required().ExistingFile() - rpmsDirPath = app.Flag("rpm-dir", "The directory to use as the local repo and to submit RPM packages to").Required().ExistingDir() - srpmsDirPath = app.Flag("srpm-dir", "The output directory for source RPM packages").Required().String() - toolchainDirPath = app.Flag("toolchain-rpms-dir", "Directory that contains already built toolchain RPMs. Should contain a top level directory for each architecture.").Required().ExistingDir() - cacheDir = app.Flag("cache-dir", "The cache directory containing downloaded dependency RPMS from Azure Linux Base").Required().ExistingDir() - basePackageName = app.Flag("base-package-name", "The name of the spec file used to build this package without the extension.").Required().String() - noCleanup = app.Flag("no-cleanup", "Whether or not to delete the chroot folder after the build is done").Bool() - distTag = app.Flag("dist-tag", "The distribution tag the SPEC will be built with.").Required().String() - distroReleaseVersion = app.Flag("distro-release-version", "The distro release version that the SRPM will be built with").Required().String() - distroBuildNumber = app.Flag("distro-build-number", "The distro build number that the SRPM will be built with").Required().String() - rpmmacrosFile = app.Flag("rpmmacros-file", "Optional file path to an rpmmacros file for rpmbuild to use").ExistingFile() - runCheck = app.Flag("run-check", "Run the check during package build").Bool() - packagesToInstall = app.Flag("install-package", "Filepaths to RPM packages that should be installed before building.").Strings() - outArch = app.Flag("out-arch", "Architecture of resulting package").String() - useCcache = app.Flag("use-ccache", "Automatically install and use ccache during package builds").Bool() - ccacheRootDir = app.Flag("ccache-root-dir", "The directory used to store ccache outputs").String() - ccachConfig = app.Flag("ccache-config", "The configuration file for ccache.").String() - maxCPU = app.Flag("max-cpu", "Max number of CPUs used for package building").Default("").String() - timeout = app.Flag("timeout", "Timeout for package building").Required().Duration() + app = kingpin.New("pkgworker", "A worker for building packages locally") + srpmFile = exe.InputFlag(app, "Full path to the SRPM to build") + workDir = app.Flag("work-dir", "The directory to create the build folder").Required().String() + workerTar = app.Flag("worker-tar", "Full path to worker_chroot.tar.gz").Required().ExistingFile() + repoFile = app.Flag("repo-file", "Full path to local.repo").Required().ExistingFile() + rpmsDirPath = app.Flag("rpm-dir", "The directory to use as the local repo and to submit RPM packages to").Required().ExistingDir() + srpmsDirPath = app.Flag("srpm-dir", "The output directory for source RPM packages").Required().String() + toolchainDirPath = app.Flag("toolchain-rpms-dir", "Directory that contains already built toolchain RPMs. Should contain a top level directory for each architecture.").Required().ExistingDir() + cacheDir = app.Flag("cache-dir", "The cache directory containing downloaded dependency RPMS from Azure Linux Base").Required().ExistingDir() + basePackageName = app.Flag("base-package-name", "The name of the spec file used to build this package without the extension.").Required().String() + noCleanup = app.Flag("no-cleanup", "Whether or not to delete the chroot folder after the build is done").Bool() + distTag = app.Flag("dist-tag", "The distribution tag the SPEC will be built with.").Required().String() + distroReleaseVersion = app.Flag("distro-release-version", "The distro release version that the SRPM will be built with").Required().String() + distroBuildNumber = app.Flag("distro-build-number", "The distro build number that the SRPM will be built with").Required().String() + rpmmacrosFile = app.Flag("rpmmacros-file", "Optional file path to an rpmmacros file for rpmbuild to use").ExistingFile() + releaseVersionMacrosFile = app.Flag("versions-macro-file", "File containing release and version macros for all SPECS to use while building.").ExistingFile() + runCheck = app.Flag("run-check", "Run the check during package build").Bool() + packagesToInstall = app.Flag("install-package", "Filepaths to RPM packages that should be installed before building.").Strings() + outArch = app.Flag("out-arch", "Architecture of resulting package").String() + useCcache = app.Flag("use-ccache", "Automatically install and use ccache during package builds").Bool() + ccacheRootDir = app.Flag("ccache-root-dir", "The directory used to store ccache outputs").String() + ccachConfig = app.Flag("ccache-config", "The configuration file for ccache.").String() + maxCPU = app.Flag("max-cpu", "Max number of CPUs used for package building").Default("").String() + timeout = app.Flag("timeout", "Timeout for package building").Required().Duration() logFlags = exe.SetupLogFlags(app) ) @@ -117,7 +118,7 @@ func main() { defines[rpm.MaxCPUDefine] = *maxCPU } - builtRPMs, err := buildSRPMInChroot(chrootDir, rpmsDirAbsPath, toolchainDirAbsPath, *workerTar, *srpmFile, *repoFile, *rpmmacrosFile, *outArch, defines, *noCleanup, *runCheck, *packagesToInstall, ccacheManager, *timeout) + builtRPMs, err := buildSRPMInChroot(chrootDir, rpmsDirAbsPath, toolchainDirAbsPath, *workerTar, *srpmFile, *repoFile, *rpmmacrosFile, *releaseVersionMacrosFile, *outArch, defines, *noCleanup, *runCheck, *packagesToInstall, ccacheManager, *timeout) logger.FatalOnError(err, "Failed to build SRPM '%s'. For details see log file: %s .", *srpmFile, *logFlags.LogFile) // For regular (non-test) package builds: @@ -154,7 +155,7 @@ func isCCacheEnabled(ccacheManager *ccachemanager.CCacheManager) bool { return ccacheManager != nil && ccacheManager.CurrentPkgGroup.Enabled } -func buildSRPMInChroot(chrootDir, rpmDirPath, toolchainDirPath, workerTar, srpmFile, repoFile, rpmmacrosFile, outArch string, defines map[string]string, noCleanup, runCheck bool, packagesToInstall []string, ccacheManager *ccachemanager.CCacheManager, timeout time.Duration) (builtRPMs []string, err error) { +func buildSRPMInChroot(chrootDir, rpmDirPath, toolchainDirPath, workerTar, srpmFile, repoFile, rpmmacrosFile, releaseVersionMacrosFile, outArch string, defines map[string]string, noCleanup, runCheck bool, packagesToInstall []string, ccacheManager *ccachemanager.CCacheManager, timeout time.Duration) (builtRPMs []string, err error) { const ( buildHeartbeatTimeout = 30 * time.Minute @@ -213,7 +214,7 @@ func buildSRPMInChroot(chrootDir, rpmDirPath, toolchainDirPath, workerTar, srpmF extraDirs = append(extraDirs, chrootCcacheDir) } - err = chroot.Initialize(workerTar, extraDirs, mountPoints, true) + err = chroot.Initialize(workerTar, extraDirs, mountPoints, true, releaseVersionMacrosFile) if err != nil { err = fmt.Errorf("failed to initialize chroot:\n%w", err) return diff --git a/toolkit/tools/rpmssnapshot/rpmssnapshot.go b/toolkit/tools/rpmssnapshot/rpmssnapshot.go index d04d701e842..f4f3e2f0ab8 100644 --- a/toolkit/tools/rpmssnapshot/rpmssnapshot.go +++ b/toolkit/tools/rpmssnapshot/rpmssnapshot.go @@ -21,9 +21,10 @@ var ( specsDirPath = exe.InputStringFlag(app, "Path to specs directory.") outputSnapshotPath = exe.OutputFlag(app, "Path to the generated snapshot.") - buildDirPath = app.Flag("build-dir", "Directory to store temporary files.").Required().String() - distTag = app.Flag("dist-tag", "The distribution tag.").Required().String() - workerTar = app.Flag("worker-tar", "Full path to worker_chroot.tar.gz.").Required().ExistingFile() + buildDirPath = app.Flag("build-dir", "Directory to store temporary files.").Required().String() + distTag = app.Flag("dist-tag", "The distribution tag.").Required().String() + workerTar = app.Flag("worker-tar", "Full path to worker_chroot.tar.gz.").Required().ExistingFile() + releaseVersionMacrosFile = app.Flag("versions-macro-file", "File containing release and version macros for all SPECS to use while parsing specs.").ExistingFile() logFlags = exe.SetupLogFlags(app) ) @@ -33,7 +34,7 @@ func main() { kingpin.MustParse(app.Parse(os.Args[1:])) logger.InitBestEffort(logFlags) - snapshotGenerator, err := rpmssnapshot.New(*buildDirPath, *workerTar, *specsDirPath) + snapshotGenerator, err := rpmssnapshot.New(*buildDirPath, *workerTar, *specsDirPath, *releaseVersionMacrosFile) if err != nil { logger.Log.Fatalf("Failed to initialize RPM snapshot generator. Error: %v", err) } diff --git a/toolkit/tools/scheduler/buildagents/chrootagent.go b/toolkit/tools/scheduler/buildagents/chrootagent.go index cf3a79b9a5c..032ee67ea5a 100644 --- a/toolkit/tools/scheduler/buildagents/chrootagent.go +++ b/toolkit/tools/scheduler/buildagents/chrootagent.go @@ -97,8 +97,12 @@ func serializeChrootBuildAgentConfig(config *BuildAgentConfig, basePackageName, fmt.Sprintf("--timeout=%s", allowableRuntime), } - if config.RpmmacrosFile != "" { - serializedArgs = append(serializedArgs, fmt.Sprintf("--rpmmacros-file=%s", config.RpmmacrosFile)) + if config.RPMMacrosFiles != "" { + serializedArgs = append(serializedArgs, fmt.Sprintf("--rpmmacros-file=%s", config.RPMMacrosFiles)) + } + + if config.VersionsMacroFile != "" { + serializedArgs = append(serializedArgs, fmt.Sprintf("--versions-macro-file=%s", config.VersionsMacroFile)) } if config.NoCleanup { diff --git a/toolkit/tools/scheduler/buildagents/definition.go b/toolkit/tools/scheduler/buildagents/definition.go index a502d660753..4919de1f335 100644 --- a/toolkit/tools/scheduler/buildagents/definition.go +++ b/toolkit/tools/scheduler/buildagents/definition.go @@ -25,7 +25,8 @@ type BuildAgentConfig struct { DistTag string DistroReleaseVersion string DistroBuildNumber string - RpmmacrosFile string + RPMMacrosFiles string + VersionsMacroFile string NoCleanup bool UseCcache bool diff --git a/toolkit/tools/scheduler/scheduler.go b/toolkit/tools/scheduler/scheduler.go index 327655caf92..fd3ee3eb8d5 100644 --- a/toolkit/tools/scheduler/scheduler.go +++ b/toolkit/tools/scheduler/scheduler.go @@ -76,6 +76,7 @@ var ( distroReleaseVersion = app.Flag("distro-release-version", "The distro release version that the SRPM will be built with.").Required().String() distroBuildNumber = app.Flag("distro-build-number", "The distro build number that the SRPM will be built with.").Required().String() rpmmacrosFile = app.Flag("rpmmacros-file", "Optional file path to an rpmmacros file for rpmbuild to use.").ExistingFile() + releaseVersionMacrosFile = app.Flag("versions-macro-file", "File containing release and version macros for all SPECS to use while building.").ExistingFile() buildAttempts = app.Flag("build-attempts", "Sets the number of times to try building a package.").Default(defaultBuildAttempts).Int() checkAttempts = app.Flag("check-attempts", "Sets the minimum number of times to test a package if the tests fail.").Default(defaultCheckAttempts).Int() extraLayers = app.Flag("extra-layers", "Sets the number of additional layers in the graph beyond the goal packages to buid.").Default(defaultExtraLayers).Int() @@ -193,7 +194,8 @@ func main() { DistTag: *distTag, DistroReleaseVersion: *distroReleaseVersion, DistroBuildNumber: *distroBuildNumber, - RpmmacrosFile: *rpmmacrosFile, + RPMMacrosFiles: *rpmmacrosFile, + VersionsMacroFile: *releaseVersionMacrosFile, NoCleanup: *noCleanup, UseCcache: *useCcache, diff --git a/toolkit/tools/specarchchecker/specarchchecker.go b/toolkit/tools/specarchchecker/specarchchecker.go index 8a7ccf146ff..1b9723b1682 100644 --- a/toolkit/tools/specarchchecker/specarchchecker.go +++ b/toolkit/tools/specarchchecker/specarchchecker.go @@ -26,9 +26,10 @@ var ( pkgsToBuild = app.Flag("packages", "Space separated list of top-level packages that should be built. Omit this argument to build all packages.").String() pkgsToRebuild = app.Flag("rebuild-packages", "Space separated list of base package names packages that should be rebuilt.").String() - buildDirPath = app.Flag("build-dir", "Directory to store temporary files.").Required().String() - distTag = app.Flag("dist-tag", "The distribution tag.").Required().String() - workerTar = app.Flag("worker-tar", "Full path to worker_chroot.tar.gz.").Required().ExistingFile() + buildDirPath = app.Flag("build-dir", "Directory to store temporary files.").Required().String() + distTag = app.Flag("dist-tag", "The distribution tag.").Required().String() + workerTar = app.Flag("worker-tar", "Full path to worker_chroot.tar.gz.").Required().ExistingFile() + releaseVersionMacrosFile = app.Flag("versions-macro-file", "File containing release and version macros for all SPECS to use while parsing specs.").ExistingFile() testOnly = app.Flag("test-only", "Whether or not to run the filter out specs which don't run tests.").Bool() @@ -48,7 +49,7 @@ func main() { logger.Log.Fatalf("No specs were provided to filter.") } - archChecker, err := specarchchecker.New(*buildDirPath, *workerTar, *specsDirPath) + archChecker, err := specarchchecker.New(*buildDirPath, *workerTar, *specsDirPath, *releaseVersionMacrosFile) if err != nil { logger.Log.Fatalf("Failed to initialize spec arch checker. Error:\n%s", err) } diff --git a/toolkit/tools/specreader/specreader.go b/toolkit/tools/specreader/specreader.go index e1ad389cfc6..7f0208a73df 100644 --- a/toolkit/tools/specreader/specreader.go +++ b/toolkit/tools/specreader/specreader.go @@ -25,23 +25,24 @@ const ( ) var ( - app = kingpin.New("specreader", "A tool to parse spec dependencies into JSON") - specsDir = exe.InputDirFlag(app, "Directory to scan for SPECS") - specList = app.Flag("spec-list", "List of SPECs to parse. If empty will parse all SPECs.").Default("").String() - output = exe.OutputFlag(app, "Output file to export the JSON") - workers = app.Flag("workers", "Number of concurrent goroutines to parse with").Default(defaultWorkerCount).Int() - buildDir = app.Flag("build-dir", "Directory to store temporary files while parsing.").String() - srpmsDir = app.Flag("srpm-dir", "Directory containing SRPMs.").Required().ExistingDir() - rpmsDir = app.Flag("rpm-dir", "Directory containing built RPMs.").Required().ExistingDir() - toolchainManifest = app.Flag("toolchain-manifest", "Path to a list of RPMs which are created by the toolchain. Will mark RPMs from this list as prebuilt.").ExistingFile() - existingToolchainRpmDir = app.Flag("toolchain-rpms-dir", "Directory that contains already built toolchain RPMs. Should contain top level directories for architecture.").Required().ExistingDir() - distTag = app.Flag("dist-tag", "The distribution tag the SPEC will be built with.").Required().String() - workerTar = app.Flag("worker-tar", "Full path to worker_chroot.tar.gz. If this argument is empty, specs will be parsed in the host environment.").ExistingFile() - targetArch = app.Flag("target-arch", "The architecture of the machine the RPM binaries run on").String() - runCheck = app.Flag("run-check", "Whether or not to run the spec file's check section during package build.").Bool() - logFlags = exe.SetupLogFlags(app) - profFlags = exe.SetupProfileFlags(app) - timestampFile = app.Flag("timestamp-file", "File that stores timestamps for this program.").String() + app = kingpin.New("specreader", "A tool to parse spec dependencies into JSON") + specsDir = exe.InputDirFlag(app, "Directory to scan for SPECS") + specList = app.Flag("spec-list", "List of SPECs to parse. If empty will parse all SPECs.").Default("").String() + output = exe.OutputFlag(app, "Output file to export the JSON") + releaseVersionMacrosFile = app.Flag("versions-macro-file", "File containing release and version macros for all SPECS to use while parsing specs.").ExistingFile() + workers = app.Flag("workers", "Number of concurrent goroutines to parse with").Default(defaultWorkerCount).Int() + buildDir = app.Flag("build-dir", "Directory to store temporary files while parsing.").String() + srpmsDir = app.Flag("srpm-dir", "Directory containing SRPMs.").Required().ExistingDir() + rpmsDir = app.Flag("rpm-dir", "Directory containing built RPMs.").Required().ExistingDir() + toolchainManifest = app.Flag("toolchain-manifest", "Path to a list of RPMs which are created by the toolchain. Will mark RPMs from this list as prebuilt.").ExistingFile() + existingToolchainRpmDir = app.Flag("toolchain-rpms-dir", "Directory that contains already built toolchain RPMs. Should contain top level directories for architecture.").Required().ExistingDir() + distTag = app.Flag("dist-tag", "The distribution tag the SPEC will be built with.").Required().String() + workerTar = app.Flag("worker-tar", "Full path to worker_chroot.tar.gz. If this argument is empty, specs will be parsed in the host environment.").ExistingFile() + targetArch = app.Flag("target-arch", "The architecture of the machine the RPM binaries run on").String() + runCheck = app.Flag("run-check", "Whether or not to run the spec file's check section during package build.").Bool() + logFlags = exe.SetupLogFlags(app) + profFlags = exe.SetupProfileFlags(app) + timestampFile = app.Flag("timestamp-file", "File that stores timestamps for this program.").String() ) func main() { @@ -74,6 +75,6 @@ func main() { specsAbsDir, err := filepath.Abs(*specsDir) logger.PanicOnError(err, "Unable to get absolute path for specs directory '%s': %s", *specsDir, err) - err = specreaderutils.ParseSPECsWrapper(*buildDir, specsAbsDir, *rpmsDir, *srpmsDir, *existingToolchainRpmDir, *distTag, *output, *workerTar, *targetArch, specListSet, toolchainRPMs, *workers, *runCheck) + err = specreaderutils.ParseSPECsWrapper(*buildDir, specsAbsDir, *rpmsDir, *srpmsDir, *existingToolchainRpmDir, *distTag, *output, *workerTar, *releaseVersionMacrosFile, *targetArch, specListSet, toolchainRPMs, *workers, *runCheck) logger.PanicOnError(err) } diff --git a/toolkit/tools/srpmpacker/srpmpacker.go b/toolkit/tools/srpmpacker/srpmpacker.go index 8c476f61c3b..54c4e7ff3f3 100644 --- a/toolkit/tools/srpmpacker/srpmpacker.go +++ b/toolkit/tools/srpmpacker/srpmpacker.go @@ -125,10 +125,11 @@ var ( srpmPackList = app.Flag("pack-list", "List of SPECs to pack. If empty will pack all SPECs.").Default("").String() runCheck = app.Flag("run-check", "Whether or not to run the spec file's check section during package build.").Bool() - workers = app.Flag("workers", "Number of concurrent goroutines to parse with.").Default(defaultWorkerCount).Uint() - concurrentNetOps = app.Flag("concurrent-net-ops", "Number of concurrent network operations to perform.").Default(defaultNetOpsCount).Uint() - repackAll = app.Flag("repack", "Rebuild all SRPMs, even if already built.").Bool() - nestedSourcesDir = app.Flag("nested-sources", "Set if for a given SPEC, its sources are contained in a SOURCES directory next to the SPEC file.").Bool() + workers = app.Flag("workers", "Number of concurrent goroutines to parse with.").Default(defaultWorkerCount).Uint() + concurrentNetOps = app.Flag("concurrent-net-ops", "Number of concurrent network operations to perform.").Default(defaultNetOpsCount).Uint() + repackAll = app.Flag("repack", "Rebuild all SRPMs, even if already built.").Bool() + nestedSourcesDir = app.Flag("nested-sources", "Set if for a given SPEC, its sources are contained in a SOURCES directory next to the SPEC file.").Bool() + releaseVersionMacrosFile = app.Flag("versions-macro-file", "File containing release and version macros for all SPECS to use while packing SRPMs.").ExistingFile() // Use String() and not ExistingFile() as the Makefile may pass an empty string if the user did not specify any of these options sourceURL = app.Flag("source-url", "URL to a source server to download SPEC sources from.").String() @@ -215,19 +216,20 @@ func main() { packList, err := packagelist.ParsePackageList(*srpmPackList) logger.PanicOnError(err) - err = createAllSRPMsWrapper(*specsDir, *distTag, *buildDir, *outDir, *workerTar, *workers, *concurrentNetOps, *nestedSourcesDir, *repackAll, *runCheck, packList, templateSrcConfig) + err = createAllSRPMsWrapper(*specsDir, *distTag, *buildDir, *outDir, *workerTar, *releaseVersionMacrosFile, *workers, *concurrentNetOps, *nestedSourcesDir, *repackAll, *runCheck, packList, templateSrcConfig) logger.PanicOnError(err) } // createAllSRPMsWrapper wraps createAllSRPMs to conditionally run it inside a chroot. // If workerTar is non-empty, packing will occur inside a chroot, otherwise it will run on the host system. -func createAllSRPMsWrapper(specsDir, distTag, buildDir, outDir, workerTar string, workers, concurrentNetOps uint, nestedSourcesDir, repackAll, runCheck bool, packList map[string]bool, templateSrcConfig sourceRetrievalConfiguration) (err error) { +// releaseVersionMacrosFile, if non-empty, is made available inside the chroot so rpmbuild can use it while packing SRPMs. +func createAllSRPMsWrapper(specsDir, distTag, buildDir, outDir, workerTar, releaseVersionMacrosFile string, workers, concurrentNetOps uint, nestedSourcesDir, repackAll, runCheck bool, packList map[string]bool, templateSrcConfig sourceRetrievalConfiguration) (err error) { var chroot *safechroot.Chroot originalOutDir := outDir if workerTar != "" { const leaveFilesOnDisk = false useAzureCliAuth := templateSrcConfig.sourceAuthMode == sourceAuthModeAzureCli - chroot, buildDir, outDir, specsDir, err = createChroot(workerTar, buildDir, outDir, specsDir, useAzureCliAuth) + chroot, buildDir, outDir, specsDir, err = createChroot(workerTar, buildDir, outDir, specsDir, releaseVersionMacrosFile, useAzureCliAuth) if err != nil { return } @@ -317,7 +319,7 @@ func findSPECFiles(specsDir string, packList map[string]bool) (specFiles []strin } // createChroot creates a chroot to pack SRPMs inside of. -func createChroot(workerTar, buildDir, outDir, specsDir string, useAzureCliAuth bool) (chroot *safechroot.Chroot, newBuildDir, newOutDir, newSpecsDir string, err error) { +func createChroot(workerTar, buildDir, outDir, specsDir, releaseVersionMacrosFile string, useAzureCliAuth bool) (chroot *safechroot.Chroot, newBuildDir, newOutDir, newSpecsDir string, err error) { const ( chrootName = "srpmpacker_chroot" existingDir = false @@ -354,7 +356,7 @@ func createChroot(workerTar, buildDir, outDir, specsDir string, useAzureCliAuth chrootDir := filepath.Join(buildDir, chrootName) chroot = safechroot.NewChroot(chrootDir, existingDir) - err = chroot.Initialize(workerTar, extraDirectories, extraMountPoints, true) + err = chroot.Initialize(workerTar, extraDirectories, extraMountPoints, true, releaseVersionMacrosFile) if err != nil { return } diff --git a/toolkit/tools/versionsprocessor/versionsprocessor.go b/toolkit/tools/versionsprocessor/versionsprocessor.go new file mode 100644 index 00000000000..ba0d35abdb0 --- /dev/null +++ b/toolkit/tools/versionsprocessor/versionsprocessor.go @@ -0,0 +1,238 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +// versionsprocessor is a tool to generate a macro file of all specs version and release. +// It iterates over all of the SPEC files in the provided directory, gets the version and release for each SPEC, +// and then writes that information to an output file as RPM macros file. +// The output file is then provided to other tools and passed into their respective rpm macros folder so that rpmbuild command automatically recognize it. + +package main + +import ( + "fmt" + "os" + "path/filepath" + "regexp" + "runtime" + "strings" + + "github.com/microsoft/azurelinux/toolkit/tools/internal/exe" + "github.com/microsoft/azurelinux/toolkit/tools/internal/file" + "github.com/microsoft/azurelinux/toolkit/tools/internal/logger" + "github.com/microsoft/azurelinux/toolkit/tools/internal/rpm" + "github.com/microsoft/azurelinux/toolkit/tools/internal/safechroot" + "github.com/microsoft/azurelinux/toolkit/tools/internal/timestamp" + "github.com/microsoft/azurelinux/toolkit/tools/pkg/profile" + "github.com/microsoft/azurelinux/toolkit/tools/pkg/specreaderutils" + + "gopkg.in/alecthomas/kingpin.v2" +) + +var packageVersionRegexp = regexp.MustCompile(`^[^:]+: (?:(.+):)?(.+)-(.+)$`) + +var ( + app = kingpin.New("versionsprocessor", "A tool to generate a macro file of all specs version and release") + specsDir = exe.InputDirFlag(app, "Directory to scan for SPECS") + output = exe.OutputFlag(app, "Output file to export the JSON") + distTag = app.Flag("dist-tag", "The distribution tag the SPEC will be built with.").Required().String() + targetArch = app.Flag("target-arch", "The architecture of the machine the RPM binaries run on").String() + buildDir = app.Flag("build-dir", "Directory to store temporary files while parsing.").String() + logFlags = exe.SetupLogFlags(app) + profFlags = exe.SetupProfileFlags(app) + workerTar = app.Flag("worker-tar", "Full path to worker_chroot.tar.gz. If this argument is empty, specs will be parsed in the host environment.").ExistingFile() + timestampFile = app.Flag("timestamp-file", "File that stores timestamps for this program.").String() + extraFiles = app.Flag("extra-macros-file", "Additional files whose contents will be appended to the output; may be specified multiple times.").ExistingFiles() +) + +func main() { + var ( + chroot *safechroot.Chroot + macrosOutput []string + ) + + app.Version(exe.ToolkitVersion) + kingpin.MustParse(app.Parse(os.Args[1:])) + logger.InitBestEffort(logFlags) + + prof, err := profile.StartProfiling(profFlags) + if err != nil { + logger.Log.Warnf("Could not start profiling: %s", err) + } + defer prof.StopProfiler() + + timestamp.BeginTiming("versionsprocessor", *timestampFile) + defer timestamp.CompleteTiming() + + var buildArch string = *targetArch + + if *targetArch == "" { + buildArch, err = rpm.GetRpmArch(runtime.GOARCH) + if err != nil { + logger.Log.Errorf("Failed to get RPM architecture for GOARCH %s: %s", runtime.GOARCH, err) + return + } + } + + if *workerTar == "" { + logger.Log.Error("No worker tar provided, please provide a worker tar to parse specs in the chroot environment.") + return + } + + const leaveFilesOnDisk = false + chroot, err = specreaderutils.CreateChroot("versionprocessor_chroot", *workerTar, *buildDir, *specsDir) + if err != nil { + logger.PanicOnError("Failed to create chroot") + } + defer chroot.Close(leaveFilesOnDisk) + + doParse := func() error { + // Find all spec files in provided specs dir + allSpecFiles, err := specreaderutils.FindSpecFiles(*specsDir, nil) + if err != nil { + logger.Log.Errorf("Error finding spec files: %s", err) + return err + } + + logger.Log.Infof("Processing version and release for %d spec files into %s", len(allSpecFiles), *output) + + type ProcessResult struct { + macros []string + err error + } + resultsChannel := make(chan ProcessResult, len(allSpecFiles)) + + for _, specFile := range allSpecFiles { + go func(sf string) { + macros, processErr := processSpecFile(sf, buildArch, *distTag, nil) + if processErr != nil { + processErr = fmt.Errorf("error processing spec file (%s): %w", sf, processErr) + } + resultsChannel <- ProcessResult{ + macros: macros, + err: processErr, + } + }(specFile) + } + + for i := 0; i < len(allSpecFiles); i++ { + result := <-resultsChannel + if result.err != nil { + logger.Log.Errorf("%s", result.err) + return result.err + } + macrosOutput = append(macrosOutput, result.macros...) + } + + return err + } + + err = chroot.Run(doParse) + + if err != nil { + logger.Log.Fatalf("Error processing spec files: %s", err) + } + + err = writeExtraFilesToOutput(*extraFiles, macrosOutput, *output) + if err != nil { + logger.Log.Errorf("Failed to write extra files to output: %s", err) + os.Exit(1) + } +} + +func processSpecFile(specFile string, buildArch string, distTag string, macrosOutput []string) (newMacrosOutput []string, err error) { + // Get spec file version-release + + specFileName := filepath.Base(specFile) + + sourceDir := filepath.Dir(specFile) + defines := rpm.DefaultDistroDefines(false, distTag) + + packages, err := rpm.QuerySPEC(specFile, sourceDir, `%{NAME}: %{evr}\n`, buildArch, defines, rpm.QueryHeaderArgument) + + if err != nil { + logger.Log.Errorf("Failed to query spec file (%s). Error: %s", specFileName, err) + return nil, err + } + + for _, packageVersionString := range packages { + + macros, err := processPackageVersionString(packageVersionString, specFileName, distTag) + if err != nil { + logger.Log.Errorf("Error processing package version string: %s", err) + continue + } + + macrosOutput = append(macrosOutput, macros...) + } + + return macrosOutput, nil +} + +func processPackageVersionString(packageVersionString string, specFileName string, distTag string) (macros []string, err error) { + const ( + prefix = "azl" + ) + // the output of the above query is in the format of "packagename: version-release", + // so split by ": " to get the version-release portion we want the second part + releaseVerSplit := packageVersionRegexp.FindStringSubmatch(packageVersionString)[1:] + + if len(releaseVerSplit) <= 2 { + errorString := fmt.Sprintf("Empty version-release format retrieved from spec file (%s)", specFileName) + err = fmt.Errorf(errorString) + logger.Log.Errorf(errorString) + + return []string{""}, err + } + + epoch := releaseVerSplit[0] + version := releaseVerSplit[1] + release := releaseVerSplit[2] + releaseClean := strings.Replace(release, distTag, "", 1) + + // strip out the .spec suffix and replace '-' with '_' as RPM macros cannot have '-' + packageFileNameMacroFormat := strings.Replace(specFileName, ".spec", "", 1) + packageFileNameMacroFormat = strings.ReplaceAll(packageFileNameMacroFormat, "-", "_") + + epochReleaseString := prefix + "_" + packageFileNameMacroFormat + "_epoch" + versionMacroString := prefix + "_" + packageFileNameMacroFormat + "_version" + releaseMacroString := prefix + "_" + packageFileNameMacroFormat + "_release" + + // Generate RPM macro definitions instead of modifying spec files directly. + macros = []string{ + fmt.Sprintf("%%%s %s", versionMacroString, version), + fmt.Sprintf("%%%s %s", releaseMacroString, releaseClean), + } + + // Only append (to the front of the list) if we have an epoch + if epoch != "" { + macros = append([]string{fmt.Sprintf("%%%s %s", epochReleaseString, epoch)}, macros...) + } + + return macros, nil +} + +func writeExtraFilesToOutput(extraFiles []string, macrosOutput []string, output string) (err error) { + // If extra files were provided, append their contents to the output as well. + for _, extraPath := range extraFiles { + if strings.TrimSpace(extraPath) == "" { + continue + } + + contents, readErr := file.Read(extraPath) + if readErr != nil { + logger.Log.Errorf("Failed to read extra macros file (%s): %s", extraPath, readErr) + continue + } + + macrosOutput = append(macrosOutput, contents) + logger.Log.Infof("Appended contents of provided extra macros file (%s) to %s", extraPath, output) + } + + err = file.WriteLines(macrosOutput, output) + if err != nil { + logger.Log.Errorf("Failed to write file (%s)", output) + return err + } + + return nil +} diff --git a/toolkit/tools/versionsprocessor/versionsprocessor_test.go b/toolkit/tools/versionsprocessor/versionsprocessor_test.go new file mode 100644 index 00000000000..a7ba832750b --- /dev/null +++ b/toolkit/tools/versionsprocessor/versionsprocessor_test.go @@ -0,0 +1,493 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +package main + +import ( + "fmt" + "os" + "path/filepath" + "runtime" + "strings" + "testing" + + "github.com/microsoft/azurelinux/toolkit/tools/internal/logger" + "github.com/microsoft/azurelinux/toolkit/tools/internal/rpm" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMain(m *testing.M) { + logger.InitStderrLog() + os.Exit(m.Run()) +} + +// --------------------------------------------------------------------------- +// processPackageVersionString tests +// --------------------------------------------------------------------------- + +func TestProcessPackageVersionString_ValidInput(t *testing.T) { + distTag := ".azl3" + + macros, err := processPackageVersionString("mypackage: 1.2.3-4.azl3", "mypackage.spec", distTag) + require.NoError(t, err) + assert.Contains(t, macros, "%azl_mypackage_version 1.2.3") + assert.Contains(t, macros, "%azl_mypackage_release 4") +} + +func TestProcessPackageVersionString_DashesInSpecName(t *testing.T) { + distTag := ".azl3" + + macros, err := processPackageVersionString("my-cool-pkg: 2.0.0-1.azl3", "my-cool-pkg.spec", distTag) + require.NoError(t, err) + // Dashes should be replaced with underscores in macro names. + assert.Contains(t, macros, "%azl_my_cool_pkg_version 2.0.0") + assert.Contains(t, macros, "%azl_my_cool_pkg_release 1") +} + +func TestProcessPackageVersionString_ReleaseWithoutDistTag(t *testing.T) { + distTag := ".azl3" + + macros, err := processPackageVersionString("pkg: 1.0-5", "pkg.spec", distTag) + require.NoError(t, err) + // When the dist tag is not present in the release, it should remain unchanged. + assert.Contains(t, macros, "%azl_pkg_version 1.0") + assert.Contains(t, macros, "%azl_pkg_release 5") +} + +func TestProcessPackageVersionString_EpochInVersion(t *testing.T) { + distTag := ".azl3" + + macros, err := processPackageVersionString("pkg: 2:3.4.5-6.azl3", "pkg.spec", distTag) + require.NoError(t, err) + assert.Contains(t, macros, "%azl_pkg_epoch 2") + assert.Contains(t, macros, "%azl_pkg_version 3.4.5") + assert.Contains(t, macros, "%azl_pkg_release 6") +} + +func TestProcessPackageVersionString_PanicsOnBadFormat(t *testing.T) { + distTag := ".azl3" + + // Input without the expected "name: version-release" format will cause a panic + // from the regex submatch slice indexing. + assert.Panics(t, func() { + processPackageVersionString("totally-invalid-input", "bad.spec", distTag) + }) +} + +func TestProcessPackageVersionString_EmptyDistTag(t *testing.T) { + distTag := "" + + macros, err := processPackageVersionString("pkg: 1.0-2.azl3", "pkg.spec", distTag) + require.NoError(t, err) + // With an empty dist tag, the release should not be modified. + assert.Contains(t, macros, "%azl_pkg_release 2.azl3") +} + +func TestProcessPackageVersionString_TableDriven(t *testing.T) { + distTag := ".azl3" + + tests := []struct { + name string + input string + specFile string + prefix string + expectedVersion string + expectedRelease string + }{ + { + name: "simple package", + input: "bash: 5.1.8-1.azl3", + specFile: "bash.spec", + prefix: "azl", + expectedVersion: "%azl_bash_version 5.1.8", + expectedRelease: "%azl_bash_release 1", + }, + { + name: "package with underscores already", + input: "python_dateutil: 2.8.2-3.azl3", + specFile: "python_dateutil.spec", + prefix: "azl", + expectedVersion: "%azl_python_dateutil_version 2.8.2", + expectedRelease: "%azl_python_dateutil_release 3", + }, + { + name: "multi digit release", + input: "kernel: 6.6.51.1-9.azl3", + specFile: "kernel.spec", + prefix: "azl", + expectedVersion: "%azl_kernel_version 6.6.51.1", + expectedRelease: "%azl_kernel_release 9", + }, + { + name: "release with extra suffixes after dist tag", + input: "openssl: 3.3.0-1.azl3.1", + specFile: "openssl.spec", + prefix: "azl", + expectedVersion: "%azl_openssl_version 3.3.0", + expectedRelease: "%azl_openssl_release 1.1", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + macros, err := processPackageVersionString(tt.input, tt.specFile, distTag) + require.NoError(t, err) + assert.Contains(t, macros, tt.expectedVersion) + assert.Contains(t, macros, tt.expectedRelease) + }) + } +} + +// --------------------------------------------------------------------------- +// Helper: create a minimal spec file that rpmspec can parse +// --------------------------------------------------------------------------- + +const specTemplate = `Summary: Test spec +Name: %s +Version: %s +Release: %s +License: MIT +URL: https://test.com +Vendor: Microsoft Corporation +Distribution: Azure Linux + +%%description +Test spec. + +%%changelog +* Mon Oct 11 2021 Test User %s-%s +- Test entry. +` + +func createSpecFile(t *testing.T, dir, name, version, release string) string { + t.Helper() + specDir := filepath.Join(dir, name) + err := os.MkdirAll(specDir, 0755) + require.NoError(t, err) + + specPath := filepath.Join(specDir, name+".spec") + content := fmt.Sprintf(specTemplate, name, version, release, version, release) + err = os.WriteFile(specPath, []byte(content), 0644) + require.NoError(t, err) + return specPath +} + +func testBuildArch(t *testing.T) string { + t.Helper() + arch, err := rpm.GetRpmArch(runtime.GOARCH) + require.NoError(t, err) + return arch +} + +// --------------------------------------------------------------------------- +// processSpecFile tests +// --------------------------------------------------------------------------- + +func TestProcessSpecFile_SinglePackage(t *testing.T) { + distTag := ".azl3" + + tmpDir := t.TempDir() + specPath := createSpecFile(t, tmpDir, "testpkg", "1.2.3", "4%{?dist}") + arch := testBuildArch(t) + + macros, err := processSpecFile(specPath, arch, distTag, nil) + require.NoError(t, err) + require.Len(t, macros, 2) + assert.NotContains(t, macros[0], "%azl_testpkg_epoch ") + assert.Contains(t, macros[0], "%azl_testpkg_version 1.2.3") + assert.Contains(t, macros[1], "%azl_testpkg_release 4") +} + +func TestProcessSpecFile_VersionOnly(t *testing.T) { + distTag := ".azl3" + + tmpDir := t.TempDir() + specPath := createSpecFile(t, tmpDir, "simplepkg", "5.0", "1%{?dist}") + arch := testBuildArch(t) + + macros, err := processSpecFile(specPath, arch, distTag, nil) + require.NoError(t, err) + require.Len(t, macros, 2) + assert.Contains(t, macros[0], "%azl_simplepkg_version 5.0") +} + +func TestProcessSpecFile_ReleaseDistTagStripped(t *testing.T) { + distTag := ".azl3" + + tmpDir := t.TempDir() + specPath := createSpecFile(t, tmpDir, "mypkg", "2.0.0", "7%{?dist}") + arch := testBuildArch(t) + + macros, err := processSpecFile(specPath, arch, distTag, nil) + require.NoError(t, err) + require.Len(t, macros, 2) + // The dist tag ".azl3" should be stripped from the release. + assert.Contains(t, macros[1], "%azl_mypkg_release 7") + assert.NotContains(t, macros[1], ".azl3") +} + +func TestProcessSpecFile_DashInPackageName(t *testing.T) { + distTag := ".azl3" + + tmpDir := t.TempDir() + specPath := createSpecFile(t, tmpDir, "my-cool-pkg", "3.1.0", "2%{?dist}") + arch := testBuildArch(t) + + macros, err := processSpecFile(specPath, arch, distTag, nil) + require.NoError(t, err) + require.Len(t, macros, 2) + // The spec file name has dashes, which should become underscores in macro names. + assert.NotContains(t, macros[0], "%azl_my_cool_pkg_epoch ") + assert.Contains(t, macros[0], "%azl_my_cool_pkg_version 3.1.0") + assert.Contains(t, macros[1], "%azl_my_cool_pkg_release 2") +} + +func TestProcessSpecFile_WithSubpackages(t *testing.T) { + distTag := ".azl3" + + tmpDir := t.TempDir() + specDir := filepath.Join(tmpDir, "multipkg") + err := os.MkdirAll(specDir, 0755) + require.NoError(t, err) + + specContent := `Summary: Test spec with subpackages +Name: multipkg +Version: 4.0.0 +Release: 3%{?dist} +License: MIT +URL: https://test.com +Vendor: Microsoft Corporation +Distribution: Azure Linux + +%description +Main package. + +%package devel +Summary: Development files + +%description devel +Dev files. + +%package libs +Summary: Libraries + +%description libs +Libs. + +%changelog +* Mon Oct 11 2021 Test User 4.0.0-3 +- Test entry. +` + specPath := filepath.Join(specDir, "multipkg.spec") + err = os.WriteFile(specPath, []byte(specContent), 0644) + require.NoError(t, err) + + arch := testBuildArch(t) + macros, err := processSpecFile(specPath, arch, distTag, nil) + require.NoError(t, err) + // With --srpm, rpmspec returns the source RPM entry (1 result). + require.Len(t, macros, 2) + + assert.NotContains(t, macros[0], "%azl_multipkg_epoch ") + assert.Contains(t, macros[0], "%azl_multipkg_version 4.0.0") + assert.Contains(t, macros[1], "%azl_multipkg_release 3") +} + +func TestProcessSpecFile_NonexistentFile(t *testing.T) { + distTag := ".azl3" + + arch := testBuildArch(t) + macros, err := processSpecFile("/nonexistent/path/fake.spec", arch, distTag, nil) + assert.Error(t, err) + assert.Nil(t, macros) +} + +func TestProcessSpecFile_InvalidSpec(t *testing.T) { + distTag := ".azl3" + + tmpDir := t.TempDir() + specDir := filepath.Join(tmpDir, "badspec") + err := os.MkdirAll(specDir, 0755) + require.NoError(t, err) + + specPath := filepath.Join(specDir, "badspec.spec") + err = os.WriteFile(specPath, []byte("this is not a valid spec file"), 0644) + require.NoError(t, err) + + arch := testBuildArch(t) + macros, err := processSpecFile(specPath, arch, distTag, nil) + assert.Error(t, err) + assert.Nil(t, macros) +} + +func TestProcessSpecFile_ReleaseWithoutDistMacro(t *testing.T) { + distTag := ".azl3" + + tmpDir := t.TempDir() + // Create spec with a release that has no %{?dist} macro. + specPath := createSpecFile(t, tmpDir, "nodist", "1.0", "5") + arch := testBuildArch(t) + + macros, err := processSpecFile(specPath, arch, distTag, nil) + require.NoError(t, err) + require.Len(t, macros, 2) + // Release should remain as-is since there's no dist tag to strip. + assert.NotContains(t, macros[0], "%azl_nodist_epoch ") + assert.Contains(t, macros[0], "%azl_nodist_version 1.0") + assert.Contains(t, macros[1], "%azl_nodist_release 5") +} + +func TestProcessSpecFile_MultiDigitVersion(t *testing.T) { + distTag := ".azl3" + + tmpDir := t.TempDir() + specPath := createSpecFile(t, tmpDir, "bigver", "10.20.30", "100%{?dist}") + arch := testBuildArch(t) + + macros, err := processSpecFile(specPath, arch, distTag, nil) + require.NoError(t, err) + require.Len(t, macros, 2) + assert.NotContains(t, macros[0], "%azl_bigver_epoch ") + assert.Contains(t, macros[0], "%azl_bigver_version 10.20.30") + assert.Contains(t, macros[1], "%azl_bigver_release 100") +} + +func TestProcessSpecFile_AccumulatesMacros(t *testing.T) { + distTag := ".azl3" + + tmpDir := t.TempDir() + arch := testBuildArch(t) + + // Process a first spec and collect its macros. + specPath1 := createSpecFile(t, tmpDir, "pkga", "1.0", "1%{?dist}") + macros, err := processSpecFile(specPath1, arch, distTag, nil) + require.NoError(t, err) + require.Len(t, macros, 2) + + // Process a second spec, passing the existing macros slice. + specPath2 := createSpecFile(t, tmpDir, "pkgb", "2.0", "2%{?dist}") + macros, err = processSpecFile(specPath2, arch, distTag, macros) + require.NoError(t, err) + // Should now contain macros from both specs. + require.Len(t, macros, 4) + combined := strings.Join(macros, "\n") + assert.Contains(t, combined, "%azl_pkga_version 1.0") + assert.Contains(t, combined, "%azl_pkgb_version 2.0") +} + +// --------------------------------------------------------------------------- +// writeExtraFilesToOutput tests +// --------------------------------------------------------------------------- + +func TestWriteExtraFilesToOutput_NoExtraFiles(t *testing.T) { + tmpDir := t.TempDir() + outputPath := filepath.Join(tmpDir, "output.macros") + macros := []string{"%azl_pkg_version 1.0", "%azl_pkg_release 1"} + + err := writeExtraFilesToOutput(nil, macros, outputPath) + require.NoError(t, err) + + data, err := os.ReadFile(outputPath) + require.NoError(t, err) + content := string(data) + assert.Contains(t, content, "%azl_pkg_version 1.0") + assert.Contains(t, content, "%azl_pkg_release 1") +} + +func TestWriteExtraFilesToOutput_WithExtraFile(t *testing.T) { + tmpDir := t.TempDir() + outputPath := filepath.Join(tmpDir, "output.macros") + + // Create an extra macros file. + extraFile := filepath.Join(tmpDir, "extra.macros") + err := os.WriteFile(extraFile, []byte("%extra_macro value123"), 0644) + require.NoError(t, err) + + macros := []string{"%azl_pkg_version 1.0"} + + err = writeExtraFilesToOutput([]string{extraFile}, macros, outputPath) + require.NoError(t, err) + + data, err := os.ReadFile(outputPath) + require.NoError(t, err) + content := string(data) + assert.Contains(t, content, "%azl_pkg_version 1.0") + assert.Contains(t, content, "%extra_macro value123") +} + +func TestWriteExtraFilesToOutput_MultipleExtraFiles(t *testing.T) { + tmpDir := t.TempDir() + outputPath := filepath.Join(tmpDir, "output.macros") + + extra1 := filepath.Join(tmpDir, "extra1.macros") + extra2 := filepath.Join(tmpDir, "extra2.macros") + err := os.WriteFile(extra1, []byte("%macro_a valA"), 0644) + require.NoError(t, err) + err = os.WriteFile(extra2, []byte("%macro_b valB"), 0644) + require.NoError(t, err) + + macros := []string{"%azl_pkg_version 1.0"} + + err = writeExtraFilesToOutput([]string{extra1, extra2}, macros, outputPath) + require.NoError(t, err) + + data, err := os.ReadFile(outputPath) + require.NoError(t, err) + content := string(data) + assert.Contains(t, content, "%azl_pkg_version 1.0") + assert.Contains(t, content, "%macro_a valA") + assert.Contains(t, content, "%macro_b valB") +} + +func TestWriteExtraFilesToOutput_SkipsEmptyPaths(t *testing.T) { + tmpDir := t.TempDir() + outputPath := filepath.Join(tmpDir, "output.macros") + + macros := []string{"%azl_pkg_version 1.0"} + + err := writeExtraFilesToOutput([]string{"", " ", ""}, macros, outputPath) + require.NoError(t, err) + + data, err := os.ReadFile(outputPath) + require.NoError(t, err) + content := string(data) + // Should only contain the original macros, nothing extra. + assert.Contains(t, content, "%azl_pkg_version 1.0") + lines := strings.Split(strings.TrimSpace(content), "\n") + assert.Equal(t, len(lines), 1) +} + +func TestWriteExtraFilesToOutput_NonexistentExtraFile(t *testing.T) { + tmpDir := t.TempDir() + outputPath := filepath.Join(tmpDir, "output.macros") + + macros := []string{"%azl_pkg_version 1.0"} + + // Should not fail entirely — the bad file is logged and skipped. + err := writeExtraFilesToOutput([]string{"/nonexistent/file.macros"}, macros, outputPath) + require.NoError(t, err) + + data, err := os.ReadFile(outputPath) + require.NoError(t, err) + content := string(data) + assert.Contains(t, content, "%azl_pkg_version 1.0") +} + +func TestWriteExtraFilesToOutput_EmptyMacrosOutput(t *testing.T) { + tmpDir := t.TempDir() + outputPath := filepath.Join(tmpDir, "output.macros") + + err := writeExtraFilesToOutput(nil, nil, outputPath) + require.NoError(t, err) + + data, err := os.ReadFile(outputPath) + require.NoError(t, err) + assert.Equal(t, "", string(data)) +} + +func TestWriteExtraFilesToOutput_InvalidOutputPath(t *testing.T) { + // Writing to a path that cannot be created should return an error. + err := writeExtraFilesToOutput(nil, []string{"macro"}, "/nonexistent/dir/subdir/output.macros") + assert.Error(t, err) +}