Benchmark revamp + run benchmark as part of CI (#3176)

* Organize benchmark gems in a group

Inspired by ruby/json#606

* Reorganize benchmark scripts

Having these in a folder helps because
we can document experiment results in it as well.

And we can edit the require script to raise an error if it takes
longer than a threshold to load faker.

Co-Authored-By: Thiago Araujo <thd.araujo@gmail.com>

* Strict permissions for gh workflows

* Skip eval and avoid generating all generators inside of the benchmark execution

Looping over the constants instead of using `eval` is a more secure approach.
Plus, build the list of generators outside so that we're only benchmarking generator execution.

Co-Authored-By: Thiago Araujo <thd.araujo@gmail.com>

* This was a one-time experiment

* Don't need to keep this around anymore after comparing YML vs JSON

---------

Co-authored-by: Thiago Araujo <thd.araujo@gmail.com>

Author: stefannibrasil

.github/workflows/bench.yml

@@ -0,0 +1,29 @@
+name: Bench
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+permissions:
+  contents: read
+  pull-requests: read
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    name: Benchmarks
+    steps:
+    - uses: actions/checkout@v6
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: '3.4'
+    - name: Install dependencies
+      run: bundle install
+
+    - name: Generators
+      run: RUBYOPT="-W0" bundle exec ruby benchmark/generators.rb
+    - name: Require
+      run: RUBYOPT="-W0" bundle exec ruby benchmark/require.rb
+

.github/workflows/ruby.yml

@@ -15,6 +15,7 @@ on:
 
 permissions:
   contents: read
+  pull-requests: read
 
 jobs:
   lint:
@@ -53,7 +54,9 @@ jobs:
         ruby-version: ${{ matrix.ruby }}
 
     - name: Install dependencies
-      run: bundle install
+      run: |
+        bundle config --without benchmark
+        bundle install
 
     - name: Run tests
       run: bundle exec rake test

.rubocop.yml

@@ -50,6 +50,11 @@ Naming/VariableNumber:
   Description: Use the configured style when numbering symbols, methods and variables.
   Enabled: false
 
+Security/Eval:
+  Description: The use of eval represents a serious security risk.
+  Exclude:
+    - 'lib/faker/default/json.rb'
+
 Style/AsciiComments:
   Description: This cop checks for non-ascii (non-English) characters in comments.
   Exclude:
@@ -102,11 +107,6 @@ Style/RegexpLiteral:
   - mixed
   AllowInnerSlashes: false
 
-Security/Eval:
-  Description: The use of eval represents a serious security risk.
-  Exclude:
-    - 'lib/faker/default/json.rb'
-
 Style/IfUnlessModifier:
   Description: Checks for `if` and `unless` statements that would fit on one line if written as modifier `if`/`unless`. The cop also checks for modifier `if`/`unless` lines that exceed the maximum line length.
   Enabled: false

Gemfile

@@ -5,7 +5,6 @@ source 'https://rubygems.org'
 # Specify your gem's dependencies in faker.gemspec
 gemspec
 
-gem 'benchmark'
 gem 'irb'
 gem 'minitest', '5.27.0'
 gem 'pry', '0.16.0'
@@ -18,3 +17,8 @@ gem 'simplecov', '0.22.0'
 gem 'test-unit', '3.7.7'
 gem 'timecop', '0.9.10'
 gem 'yard', '0.9.38'
+
+group :benchmark do
+  gem 'benchmark'
+  gem 'benchmark-ips'
+end

Gemfile.lock

@@ -9,6 +9,7 @@ GEM
   specs:
     ast (2.4.3)
     benchmark (0.5.0)
+    benchmark-ips (2.14.0)
     coderay (1.1.3)
     concurrent-ruby (1.3.6)
     date (3.5.1)
@@ -96,6 +97,7 @@ PLATFORMS
 
 DEPENDENCIES
   benchmark
+  benchmark-ips
   faker!
   irb
   minitest (= 5.27.0)

benchmark/generators.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require 'benchmark/ips'
+require 'faker'
+
+def generators
+  constants.flat_map do |klass|
+    subclass_methods(klass)
+  end
+end
+
+def constants
+  Faker.constants.delete_if do |subclass|
+    %i[
+      Base
+      Cat
+      Char
+      Base58
+      Config
+      Date
+      Deprecator
+      Dog
+      Religion
+      Time
+      VERSION
+    ].include?(subclass)
+  end.sort
+end
+
+def subclass_methods(subclass)
+  klass = Faker.const_get(subclass)
+
+  public_methods = klass.public_methods(false) - Faker::Base.public_methods(false)
+
+  public_methods.sort.map do |method|
+    [klass, method]
+  end
+end
+
+all_generators = generators
+
+Benchmark.ips do |x|
+  x.report("Number of generators: #{all_generators.count}") do
+    all_generators.each { |klass, generator| klass.send(generator) }
+  end
+end

benchmark/require.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+require 'benchmark'
+
+ms = Benchmark.realtime do
+  require 'faker'
+end * 1000
+
+puts "took #{ms}ms to load"