diff --git a/.github/workflows/sentry_sidekiq_test.yml b/.github/workflows/sentry_sidekiq_test.yml
index bb09d281d..a8c01a9f7 100644
--- a/.github/workflows/sentry_sidekiq_test.yml
+++ b/.github/workflows/sentry_sidekiq_test.yml
@@ -40,4 +40,4 @@ jobs:
     - name: Run specs
       run: |
         bundle install --jobs 4 --retry 3
-        bundle exec rake
+        make test
diff --git a/sentry-sidekiq/Makefile b/sentry-sidekiq/Makefile
index 62ad039eb..2e9d7ea98 100644
--- a/sentry-sidekiq/Makefile
+++ b/sentry-sidekiq/Makefile
@@ -1,3 +1,7 @@
 build:
 	bundle install
 	gem build sentry-sidekiq.gemspec
+
+test:
+	bundle exec rspec
+	WITH_SENTRY_RAILS=1 bundle exec rspec spec/sentry/rails_spec.rb
diff --git a/sentry-sidekiq/spec/sentry/rails_spec.rb b/sentry-sidekiq/spec/sentry/rails_spec.rb
index deb94b062..903f8ab38 100644
--- a/sentry-sidekiq/spec/sentry/rails_spec.rb
+++ b/sentry-sidekiq/spec/sentry/rails_spec.rb
@@ -1,3 +1,5 @@
+return unless ENV["WITH_SENTRY_RAILS"]
+
 require "rails"
 require "sentry-rails"
 require "spec_helper"