Make prefer-local work well

lib/bootstrap/bundler.rb:
1. Added jruby_bundled_specs method to Source::Rubygems that returns specs from JRuby's specifications/ directory
2. Modified specs method to merge jruby_bundled_specs with higher precedence when prefer_local is true
3. Updated install patch to skip installation for gems that already exist in JRuby's bundled gems directory
4. Track @jruby_bundled_specs_dir alongside @jruby_default_specs_dir

lib/pluginmanager/command.rb:
1. Added explicit filter to not touch JRuby's bundled gems in remove_orphan_dependencies!

The behavior now:
- JRuby bundled gems are preferred over remote versions
- If a version constraint requires a newer version, it will be fetched from remote
- No unnecessary copying of already-available gems to vendor/bundle

Author: jsvd

lib/bootstrap/bundler.rb

@@ -20,6 +20,10 @@ module Bundler
     extend self
 
     def patch!
+
+      return if @bundler_patched
+      @bundler_patched = true
+
       # Patch to prevent Bundler to save a .bundle/config file in the root
       # of the application
       ::Bundler::Settings.module_exec do
@@ -40,15 +44,15 @@ def self.reset_paths!
         end
       end
 
-      # When preparing offline packs or generally when installing gems, bundler wants to have `.gem` files 
+      # When preparing offline packs or generally when installing gems, bundler wants to have `.gem` files
       # cached. We ship a default set of gems that inclue all of the unpacked code. During dependency
-      # resolution bundler still wants to ensure`.gem` files exist. This patch updates two paths in bundler where 
+      # resolution bundler still wants to ensure`.gem` files exist. This patch updates two paths in bundler where
       # it natively it would *fail* when a `.gem` file is not found. Instead of failing we force the cache to be
       # updated with a `.gem` file. This preserves the original patch behavior. There is still an open question of
-      # *how* to potentially update the files we vendor or the way we set up bundler to avoid carrying this patch. 
+      # *how* to potentially update the files we vendor or the way we set up bundler to avoid carrying this patch.
       # As of JRuby 9.4.13.0 rubygems (bundler) is at 3.6.3. There have been some releases and changes in bundler code
       # since then but it does not seem to have changed the way it handles gem files. Obviously carrying a patch like this
-      # carries a maintenance burden so prioritizing a packaging solution may be 
+      # carries a maintenance burden so prioritizing a packaging solution may be
       ::Bundler::Source::Rubygems.module_exec do
         def fetch_gem_if_possible(spec, previous_spec = nil)
           path = if spec.remote
@@ -75,18 +79,278 @@ def cache(spec, custom_path = nil)
           raise InstallError, e.message
         end
       end
+
+      #
+      # BACKPORT: Fix `--prefer-local` flag (from rubygems/rubygems commits 607a3bf479, 209b93a, 23047a0)
+      #
+      # The original implementation of --prefer-local was too naive:
+      # 1. It didn't pass prefer_local to Package objects
+      # 2. It returned empty array when no local specs exist (instead of falling back to remote)
+      # 3. It didn't properly handle default gems
+      #
+      # These patches fix:
+      # - PR #7951: Fix `--prefer-local` flag (propagate to packages, add fallback logic)
+      # - PR #8412: Fix `--prefer-local` not respecting default gems
+      # - PR #8484: Fix `bundle install --prefer-local` sometimes installing very old versions
+      #
+
+      # Patch Source base class to add prefer_local! method
+      ::Bundler::Source.class_eval do
+        def prefer_local!
+          # Base implementation - does nothing, subclasses override
+        end
+      end
+
+      # Patch Source::Rubygems to track prefer_local state and handle default_specs properly
+      # Also add support for JRuby bundled gems (gems in vendor/jruby/.../specifications/)
+      ::Bundler::Source::Rubygems.class_eval do
+        # Add prefer_local! method
+        def prefer_local!
+          @prefer_local = true
+        end
+
+        # Return specs from JRuby's bundled gem directory (specifications/, not specifications/default/)
+        # These are gems that ship with JRuby but aren't "default gems" in the Ruby sense
+        def jruby_bundled_specs
+          @jruby_bundled_specs ||= begin
+            idx = ::Bundler::Index.new
+            jruby_specs_dir = LogStash::Bundler.instance_variable_get(:@jruby_bundled_specs_dir)
+            jruby_default_specs_dir = LogStash::Bundler.instance_variable_get(:@jruby_default_specs_dir)
+
+            if jruby_specs_dir && ::File.directory?(jruby_specs_dir)
+              # Get gemspecs from specifications/ but NOT from specifications/default/
+              ::Dir[::File.join(jruby_specs_dir, "*.gemspec")].each do |path|
+                # Skip if this is actually in the default directory
+                next if jruby_default_specs_dir && path.start_with?(jruby_default_specs_dir)
+
+                stub = ::Gem::StubSpecification.gemspec_stub(path, jruby_specs_dir, jruby_specs_dir)
+                # Create a Bundler::StubSpecification from the Gem::StubSpecification
+                bundler_spec = ::Bundler::StubSpecification.from_stub(stub)
+                # Set source to self (the Source::Rubygems instance) - required for materialization
+                bundler_spec.source = self
+                idx << bundler_spec
+              end
+            end
+            idx
+          end
+        end
+
+        # Override specs method to handle prefer_local for default_specs AND jruby_bundled_specs
+        alias_method :original_specs, :specs
+
+        def specs
+          @specs ||= begin
+            # remote_specs usually generates a way larger Index than the other
+            # sources, and large_idx.merge! small_idx is way faster than
+            # small_idx.merge! large_idx.
+            index = @allow_remote ? remote_specs.dup : ::Bundler::Index.new
+            index.merge!(cached_specs) if @allow_cached
+            index.merge!(installed_specs) if @allow_local
+
+            if @allow_local
+              if @prefer_local
+                # With prefer_local, merge jruby_bundled_specs and default_specs so they take precedence
+                # over remote/cached/installed specs. This ensures JRuby's bundled gems are preferred.
+                index.merge!(jruby_bundled_specs)
+                index.merge!(default_specs)
+              else
+                # complete with default specs, only if not already available in the
+                # index through remote, cached, or installed specs
+                index.use(jruby_bundled_specs)
+                index.use(default_specs)
+              end
+            end
+
+            index
+          end
+        end
+      end
+
+      # Patch SourceList to propagate prefer_local! to all sources
+      ::Bundler::SourceList.class_eval do
+        def prefer_local!
+          all_sources.each(&:prefer_local!)
+        end
+      end
+
+      # Patch Definition to call sources.prefer_local! when prefer_local! is called
+      ::Bundler::Definition.class_eval do
+        alias_method :original_prefer_local!, :prefer_local!
+
+        def prefer_local!
+          @prefer_local = true
+          sources.prefer_local!
+        end
+      end
+
+      # Patch Package to add prefer_local support
+      ::Bundler::Resolver::Package.class_eval do
+        def prefer_local?
+          @prefer_local
+        end
+
+        def consider_remote_versions!
+          @prefer_local = false
+        end
+      end
+
+      # Patch Resolver::Base to propagate prefer_local to packages and add include_remote_specs
+      ::Bundler::Resolver::Base.class_eval do
+        alias_method :original_base_initialize, :initialize
+
+        def initialize(source_requirements, dependencies, base, platforms, options)
+          @prefer_local_option = options[:prefer_local]
+          original_base_initialize(source_requirements, dependencies, base, platforms, options)
+        end
+
+        alias_method :original_get_package, :get_package
+
+        def get_package(name)
+          package = original_get_package(name)
+          # Inject prefer_local into packages since older Bundler doesn't pass it through
+          if @prefer_local_option && !package.instance_variable_get(:@prefer_local)
+            package.instance_variable_set(:@prefer_local, true)
+          end
+          package
+        end
+
+        def include_remote_specs(names)
+          names.each do |name|
+            get_package(name).consider_remote_versions!
+          end
+        end
+      end
+
+      # Patch Resolver to fix filter_remote_specs with proper fallback
+      ::Bundler::Resolver.class_eval do
+        # Override filter_remote_specs with the fixed version from Bundler 2.7+
+        # This fixes the issue where --prefer-local would return empty specs
+        # when no local gems are installed, instead of falling back to remote
+        def filter_remote_specs(specs, package)
+          if package.prefer_local?
+            local_specs = specs.select {|s| s.is_a?(::Bundler::StubSpecification) }
+
+            if local_specs.empty?
+              # BACKPORT FIX: If no local specs exist, fall back to remote specs
+              # instead of returning empty array
+              package.consider_remote_versions!
+              specs
+            else
+              local_specs
+            end
+          else
+            specs
+          end
+        end
+      end
+
+      # Patch Source::Rubygems#install to skip installation for default gems and JRuby bundled gems
+      # The original condition `spec.default_gem? && !cached_built_in_gem(...)` has a side effect:
+      # cached_built_in_gem fetches from remote if not in cache. For default gems and JRuby bundled gems,
+      # we should skip installation entirely without needing a cached .gem file.
+      ::Bundler::Source::Rubygems.class_eval do
+        alias_method :original_rubygems_install, :install
+
+        def install(spec, options = {})
+          # For default gems, skip installation entirely - they're already available
+          if spec.default_gem?
+            print_using_message "Using #{version_message(spec, options[:previous_spec])}"
+            return nil
+          end
+
+          # For JRuby bundled gems, also skip installation - they're already available
+          # Check if this exact gem (name + version) exists in JRuby's bundled gems
+          jruby_specs_dir = LogStash::Bundler.instance_variable_get(:@jruby_bundled_specs_dir)
+          if jruby_specs_dir && ::File.directory?(jruby_specs_dir)
+            jruby_gemspec_path = ::File.join(jruby_specs_dir, "#{spec.name}-#{spec.version}.gemspec")
+            if ::File.exist?(jruby_gemspec_path)
+              print_using_message "Using #{version_message(spec, options[:previous_spec])}"
+              return nil
+            end
+          end
+
+          original_rubygems_install(spec, options)
+        end
+      end
     end
 
 
+    # Capture JRuby's default gem directory before paths are changed
+    # This is needed so that default gems (like json) can be found with --prefer-local
+    def preserve_jruby_default_gems_path
+      return @jruby_default_gem_dir if defined?(@jruby_default_gem_dir)
+
+      # The Gradle/JRuby setup already changes Gem.default_dir to a temp path before
+      # this code runs, so we need to construct the actual JRuby path from LOGSTASH_HOME
+      logstash_home = ENV["LOGSTASH_HOME"] || ::File.expand_path("../../..", __FILE__)
+      jruby_gems_dir = ::File.join(logstash_home, "vendor", "jruby", "lib", "ruby", "gems", "shared")
+      jruby_default_specs = ::File.join(jruby_gems_dir, "specifications", "default")
+      jruby_bundled_specs = ::File.join(jruby_gems_dir, "specifications")
+
+      if ::File.directory?(jruby_default_specs)
+        @jruby_default_gem_dir = jruby_gems_dir
+        @jruby_default_specs_dir = jruby_default_specs
+        @jruby_bundled_specs_dir = jruby_bundled_specs
+      else
+        # Fall back to Gem.default_dir if vendor/jruby doesn't exist
+        @jruby_default_gem_dir = ::Gem.default_dir
+        @jruby_default_specs_dir = ::Gem.default_specifications_dir
+        @jruby_bundled_specs_dir = nil
+      end
+
+      @jruby_default_gem_dir
+    end
+
+    # Patch Gem::Specification.default_stubs to also look in JRuby's original default specs directory
+    # This is needed because Gem.default_specifications_dir only returns a single path,
+    # and after Gem.paths = ENV it points to Logstash's gem home, not JRuby's installation
+    def patch_default_stubs!
+      return if @default_stubs_patched || !defined?(@jruby_default_specs_dir) || @jruby_default_specs_dir.nil?
+      @default_stubs_patched = true
+
+      jruby_specs_dir = @jruby_default_specs_dir
+
+      ::Gem::Specification.singleton_class.class_eval do
+        alias_method :original_default_stubs, :default_stubs
+
+        define_method(:default_stubs) do |pattern = "*.gemspec"|
+          # Get stubs from the current default_specifications_dir
+          stubs = original_default_stubs(pattern)
+
+          # Also look in JRuby's original default specs directory if it exists and is different
+          if jruby_specs_dir && ::File.directory?(jruby_specs_dir) && jruby_specs_dir != ::Gem.default_specifications_dir
+            base_dir = jruby_specs_dir
+            ::Dir[::File.join(base_dir, pattern)].each do |path|
+              # Use default_gemspec_stub to mark these as default gems (default_gem = true)
+              stub = ::Gem::StubSpecification.default_gemspec_stub(path, base_dir, base_dir)
+              stubs << stub unless stubs.any? { |s| s.name == stub.name && s.version == stub.version }
+            end
+          end
+
+          stubs
+        end
+      end
+    end
+
     # prepare bundler's environment variables, but do not invoke ::Bundler::setup
     def prepare(options = {})
       options = {:without => [:development]}.merge(options)
       options[:without] = Array(options[:without])
 
+      # Capture JRuby default gems path BEFORE clearing
+      jruby_gem_dir = preserve_jruby_default_gems_path
+
       ::Gem.clear_paths
-      ENV['GEM_HOME'] = ENV['GEM_PATH'] = Environment.logstash_gem_home
+      # Include both Logstash gem home AND JRuby's default gem directory in GEM_PATH
+      # This ensures default gems can be discovered by Gem::Specification.default_stubs
+      gem_path = [Environment.logstash_gem_home, jruby_gem_dir].compact.uniq.join(::File::PATH_SEPARATOR)
+      ENV['GEM_HOME'] = Environment.logstash_gem_home
+      ENV['GEM_PATH'] = gem_path
       ::Gem.paths = ENV
 
+      # Patch default_stubs to also look in JRuby's original location
+      patch_default_stubs!
+
       # set BUNDLE_GEMFILE ENV before requiring bundler to avoid bundler recurse and load unrelated Gemfile(s)
       ENV["BUNDLE_GEMFILE"] = Environment::GEMFILE_PATH
 
@@ -127,9 +391,21 @@ def invoke!(options = {})
                  :jobs => 12, :all => false, :package => false, :without => [:development]}.merge(options)
       options[:without] = Array(options[:without])
       options[:update] = Array(options[:update]) if options[:update]
+
+      # Capture JRuby default gems path BEFORE clearing
+      jruby_gem_dir = preserve_jruby_default_gems_path
+
       ::Gem.clear_paths
-      ENV['GEM_HOME'] = ENV['GEM_PATH'] = LogStash::Environment.logstash_gem_home
+      # Include both Logstash gem home AND JRuby's default gem directory in GEM_PATH
+      # This ensures default gems can be discovered by Gem::Specification.default_stubs
+      gem_path = [LogStash::Environment.logstash_gem_home, jruby_gem_dir].compact.uniq.join(::File::PATH_SEPARATOR)
+      ENV['GEM_HOME'] = LogStash::Environment.logstash_gem_home
+      ENV['GEM_PATH'] = gem_path
       ::Gem.paths = ENV
+
+      # Patch default_stubs to also look in JRuby's original location
+      patch_default_stubs!
+
       # set BUNDLE_GEMFILE ENV before requiring bundler to avoid bundler recurse and load unrelated Gemfile(s).
       # in the context of calling Bundler::CLI this is not really required since Bundler::CLI will look at
       # Bundler.settings[:gemfile] unlike Bundler.setup. For the sake of consistency and defensive/future proofing, let's keep it here.

lib/pluginmanager/bundler/logstash_uninstall.rb

@@ -35,6 +35,7 @@ def initialize(gemfile_path, lockfile_path)
     end
 
     def uninstall!(gems_to_remove)
+
       gems_to_remove = Array(gems_to_remove)
 
       unsatisfied_dependency_mapping = Dsl.evaluate(gemfile_path, lockfile_path, {}).specs.each_with_object({}) do |spec, memo|

lib/pluginmanager/command.rb

@@ -50,9 +50,14 @@ def remove_unused_locally_installed_gems!
 
   def remove_orphan_dependencies!
     locked_gem_names = ::Bundler::LockfileParser.new(File.read(LogStash::Environment::LOCKFILE)).specs.map(&:full_name).to_set
+    bundle_path = LogStash::Environment::BUNDLE_DIR
+    # JRuby bundled gems path - never touch these
+    jruby_gems_path = File.join(LogStash::Environment::LOGSTASH_HOME, "vendor", "jruby", "lib", "ruby", "gems")
     orphan_gem_specs = ::Gem::Specification.each
                                            .reject(&:stubbed?) # skipped stubbed (uninstalled) gems
                                            .reject(&:default_gem?) # don't touch jruby-included default gems
+                                           .reject { |spec| spec.full_gem_path.start_with?(jruby_gems_path) } # don't touch jruby bundled gems
+                                           .select { |spec| spec.full_gem_path.start_with?(bundle_path) } # only gems in bundle path
                                            .reject{ |spec| locked_gem_names.include?(spec.full_name) }
                                            .sort
 

rakelib/plugin.rake

@@ -90,7 +90,6 @@ namespace "plugin" do
     task.reenable # Allow this task to be run again
   end # task "install"
 
-
   task "clean-duplicate-gems" do
     shared_gems_path = File.join(LogStash::Environment::LOGSTASH_HOME,
                                   'vendor/jruby/lib/ruby/gems/shared/gems')
@@ -133,14 +132,15 @@ namespace "plugin" do
 
     # Remove default gem gemspecs only
     default_duplicates.each do |gem_name|
-      # For stdlib default gems we only remove the gemspecs as removing the source code 
+      # For stdlib default gems we only remove the gemspecs as removing the source code
       # files results in code loading errors and ruby warnings
       FileUtils.rm_rf(Dir.glob("#{default_gemspecs_path}/#{gem_name}-*.gemspec"))
     end
-    
+
     task.reenable
   end
 
+
   task "install-default" => "bootstrap" do
     puts("[plugin:install-default] Installing default plugins")