Maintain versioned API docs at the website.

.github/workflows/release.yaml

@@ -18,7 +18,8 @@ permissions:
   contents: read
 
 jobs:
-  push:
+  release:
+    name: Release Gems
     # Limit who can run run this action to core maintainers.
     if: github.repository == 'block/elasticgraph' && contains('["myronmarston", "BrianSigafoos-SQ"]', github.actor)
     runs-on: ubuntu-latest
@@ -62,8 +63,21 @@ jobs:
           git update-index --skip-worktree Rakefile
           echo "BUNDLE_GEMFILE=config/release/Gemfile" >> "$GITHUB_ENV"
 
+      - name: Archive Docs (for non-prerelease versions)
+        run: |
+          if [[ ! "${{ inputs.version }}" =~ [a-zA-Z] ]]; then
+            bundle exec rake "site:archive_docs[${{ inputs.version }}]"
+            git add config/site/archived_docs/v${{ inputs.version }}.tar.gz
+            git commit -m "Archive docs for v${{ inputs.version }}"
+          fi
+
       - name: Bump the ElasticGraph version
-        run: bundle exec rake bump_version[${{ inputs.version }}]
+        run: |
+          bundle exec rake bump_version[${{ inputs.version }}]
+          if [[ ! "${{ inputs.version }}" =~ [a-zA-Z] ]]; then
+            # For non-prerelease versions, squash the doc archive commit with the version bump
+            git reset --soft HEAD~2 && git commit -C ORIG_HEAD
+          fi
 
       # Note: we put this after bumping the version because really bumping the version is safe to do in dry-run mode.
       - name: Enable Dry Run Mode

config/release/Gemfile

@@ -10,3 +10,4 @@ source "https://rubygems.org"
 
 gem "gem-release", "~> 2.2"
 gem "rake", "~> 13.2"
+gem "yard", "~> 0.9", ">= 0.9.37"

config/release/Gemfile.lock

@@ -3,6 +3,7 @@ GEM
   specs:
     gem-release (2.2.4)
     rake (13.2.1)
+    yard (0.9.37)
 
 PLATFORMS
   ruby
@@ -11,6 +12,7 @@ PLATFORMS
 DEPENDENCIES
   gem-release (~> 2.2)
   rake (~> 13.2)
+  yard (~> 0.9, >= 0.9.37)
 
 BUNDLED WITH
    2.5.22

config/release/Rakefile

@@ -15,6 +15,9 @@ require "gem/release"
 require "#{project_root}/elasticgraph-support/lib/elastic_graph/version"
 require "#{project_root}/script/list_eg_gems"
 
+# Load tasks from config/site/Rakefile
+load "#{project_root}/config/site/Rakefile"
+
 desc "Bumps the ElasticGraph version to the specified new version number"
 task :bump_version, [:version] do |_, args|
   version = args.fetch(:version)

config/site/Rakefile

@@ -6,16 +6,19 @@
 #
 # frozen_string_literal: true
 
-require "elastic_graph/local/rake_tasks"
-require "elastic_graph/query_registry/rake_tasks"
+# Note: we need to avoid loading anything here except rake and standard library things, because
+# our release workflow loads this with a minimal Gemfile that lacks most of our dependencies.
 require "pathname"
+require "rake/tasklib"
 
 module ElasticGraph
   class SiteRakeTasks < ::Rake::TaskLib
     SITE_CONFIG_DIR = ::Pathname.new(__dir__)
     REPO_ROOT = SITE_CONFIG_DIR.parent.parent
     SITE_SOURCE_DIR = SITE_CONFIG_DIR / "src"
-    YARD_OUTPUT_DIR = SITE_SOURCE_DIR / "docs" / "main"
+    DOCS_DIR = SITE_SOURCE_DIR / "docs"
+    ARCHIVED_DOCS_DIR = SITE_CONFIG_DIR / "archived_docs"
+    YARD_OUTPUT_DIR = DOCS_DIR / "main"
     JEKYLL_SITE_DIR = SITE_CONFIG_DIR / "_site"
     JEKYLL_DATA_DIR = SITE_SOURCE_DIR / "_data"
     EXAMPLE_SCHEMA_FILES_BY_NAME = SITE_CONFIG_DIR.glob("examples/*/schema.rb").to_h do |file|
@@ -41,7 +44,7 @@ module ElasticGraph
 
     def initialize
       namespace :site do
-        task :build_docs do
+        task build_docs: [:unpack_doc_archives] do
           # Clean the docs output directory
           FileUtils.rm_rf(YARD_OUTPUT_DIR)
           run_yard_doc_ignoring_expected_warnings
@@ -134,6 +137,47 @@ module ElasticGraph
           sh "bundle exec jekyll build #{common_jekyll_args}"
         end
 
+        desc "Build YARD docs and create versioned archive"
+        task :archive_docs, [:version] => [:build_docs] do |_, args|
+          abort "Version argument is required (e.g., rake site:archive_docs[1.0.0])" unless args[:version]
+
+          archive_name = "v#{args[:version]}.tar.gz"
+          archive_path = ARCHIVED_DOCS_DIR / archive_name
+
+          FileUtils.mkdir_p(ARCHIVED_DOCS_DIR)
+
+          Dir.chdir(YARD_OUTPUT_DIR) do
+            # --no-xattrs is necessary to ensure that apple-specific attributes are excluded from the archive.
+            # When unpacking the archives on linux system, it produces warnings if we allow apple's extended
+            # attributes to be included.
+            sh "tar --no-xattrs -czf #{archive_path} ."
+          end
+
+          puts "Documentation archive created at: #{archive_path}"
+        end
+
+        desc "Unpack all documentation archives into src/docs"
+        task :unpack_doc_archives do
+          # Clean the docs directory
+          FileUtils.rm_rf(DOCS_DIR)
+
+          Dir.glob(ARCHIVED_DOCS_DIR / "*.tar.gz").sort.each do |archive|
+            version_name = File.basename(archive, ".tar.gz")
+            target_dir = DOCS_DIR / version_name
+
+            # Remove existing directory if it exists
+            FileUtils.rm_rf(target_dir)
+            FileUtils.mkdir_p(target_dir)
+
+            puts "Unpacking #{version_name} documentation..."
+            Dir.chdir(target_dir) do
+              sh "tar -xzf #{archive}"
+            end
+          end
+
+          puts "All documentation archives have been unpacked"
+        end
+
         desc "Serve Jekyll site locally"
         task serve: [:build_docs, :build_css, "examples:compile_queries"] do
           require "filewatcher"
@@ -212,34 +256,40 @@ module ElasticGraph
           end
         end
 
-        namespace :examples do
-          task compile_queries: EXAMPLE_SCHEMA_FILES_BY_NAME.keys.map { |schema| "#{schema}:compile_queries" }
+        # When we are releasing gems, these dependencies aren't available.
+        unless ENV["BUNDLE_GEMFILE"].to_s.end_with?("config/release/Gemfile")
+          require "elastic_graph/local/rake_tasks"
+          require "elastic_graph/query_registry/rake_tasks"
 
-          EXAMPLE_SCHEMA_FILES_BY_NAME.each do |schema_name, schema_file|
-            example_dir = schema_file.parent
-            settings_file = example_dir / "local_settings.yaml"
-            queries_dir = example_dir / "queries"
+          namespace :examples do
+            task compile_queries: EXAMPLE_SCHEMA_FILES_BY_NAME.keys.map { |schema| "#{schema}:compile_queries" }
 
-            namespace schema_name do
-              ::ElasticGraph::Local::RakeTasks.new(local_config_yaml: settings_file, path_to_schema: schema_file) do |tasks|
-                tasks.opensearch_versions = []
-                tasks.enforce_json_schema_version = false
-              end
-              ::ElasticGraph::QueryRegistry::RakeTasks.from_yaml_file(settings_file, queries_dir)
+            EXAMPLE_SCHEMA_FILES_BY_NAME.each do |schema_name, schema_file|
+              example_dir = schema_file.parent
+              settings_file = example_dir / "local_settings.yaml"
+              queries_dir = example_dir / "queries"
+
+              namespace schema_name do
+                ::ElasticGraph::Local::RakeTasks.new(local_config_yaml: settings_file, path_to_schema: schema_file) do |tasks|
+                  tasks.opensearch_versions = []
+                  tasks.enforce_json_schema_version = false
+                end
+                ::ElasticGraph::QueryRegistry::RakeTasks.from_yaml_file(settings_file, queries_dir)
 
-              task "query_registry:validate_queries" => ["schema_artifacts:dump", "query_registry:dump_variables:all"]
+                task "query_registry:validate_queries" => ["schema_artifacts:dump", "query_registry:dump_variables:all"]
 
-              task compile_queries: "query_registry:validate_queries" do
-                queries_by_name_by_category = queries_dir.children.to_h do |category_path|
-                  queries_by_name = category_path.glob("*.graphql").to_h do |query_path|
-                    [query_path.basename.sub_ext("").to_s, query_path.read.strip]
+                task compile_queries: "query_registry:validate_queries" do
+                  queries_by_name_by_category = queries_dir.children.to_h do |category_path|
+                    queries_by_name = category_path.glob("*.graphql").to_h do |query_path|
+                      [query_path.basename.sub_ext("").to_s, query_path.read.strip]
+                    end
+
+                    [category_path.basename.to_s, queries_by_name]
                   end
 
-                  [category_path.basename.to_s, queries_by_name]
+                  ::FileUtils.mkdir_p JEKYLL_DATA_DIR
+                  ::File.write(::File.join(JEKYLL_DATA_DIR, "#{schema_name}_queries.yaml"), ::YAML.dump(queries_by_name_by_category))
                 end
-
-                ::FileUtils.mkdir_p JEKYLL_DATA_DIR
-                ::File.write(::File.join(JEKYLL_DATA_DIR, "#{schema_name}_queries.yaml"), ::YAML.dump(queries_by_name_by_category))
               end
             end
           end

config/site/src/_includes/navbar.html

@@ -5,7 +5,33 @@
       <a href="{{ '/' | relative_url }}" class="font-bold hover:underline">ElasticGraph</a>
       <a href="{{ '/getting-started' | relative_url }}" class="{{ site.style.link }}">Get Started</a>
       <a href="{{ '/query-api' | relative_url }}" class="{{ site.style.link }}">Query API</a>
-      <a href="{{ '/docs/main' | relative_url }}" class="{{ site.style.link }}">Docs</a>
+
+      <!-- Docs dropdown -->
+      <div class="relative group">
+        <button class="{{ site.style.link }} flex items-center">
+          Docs
+          <svg class="w-4 h-4 ml-1" fill="none" stroke="currentColor" viewBox="0 0 24 24">
+            <path stroke-linecap="round" stroke-linejoin="round" stroke-width="2" d="M19 9l-7 7-7-7"></path>
+          </svg>
+        </button>
+        <!-- Added pt-2 to create space for hover and moved mt-2 to inner div -->
+        <div class="absolute left-0 pt-2 w-48 hidden group-hover:block z-50">
+          <div class="rounded-md shadow-lg bg-white dark:bg-gray-700 ring-1 ring-black ring-opacity-5">
+            <div class="py-1">
+              {% for version in site.data.doc_versions.versions %}
+                <a href="{{ '/docs/' | append: version | relative_url }}"
+                   class="block px-4 py-2 text-sm text-gray-700 dark:text-gray-200 hover:bg-gray-100 dark:hover:bg-gray-600">
+                  {% if version == 'main' %}
+                    Development (main)
+                  {% else %}
+                    {{ version }}
+                  {% endif %}
+                </a>
+              {% endfor %}
+            </div>
+          </div>
+        </div>
+      </div>
 
     </div>
     <div>

config/site/src/_plugins/doc_versions.rb

@@ -0,0 +1,27 @@
+# Copyright 2024 Block, Inc.
+#
+# Use of this source code is governed by an MIT-style
+# license that can be found in the LICENSE file or at
+# https://opensource.org/licenses/MIT.
+#
+# frozen_string_literal: true
+
+module Jekyll
+  class DocVersions < Generator
+    def generate(site)
+      # Get all subdirectories in src/docs
+      docs_dir = File.join(site.source, "docs")
+      versions = if Dir.exist?(docs_dir)
+        Dir.entries(docs_dir)
+          .select { |f| File.directory?(File.join(docs_dir, f)) && f !~ /^\./ }
+          .sort_by { |v| (v == "main") ? "0" : Gem::Version.new(v.delete_prefix("v")) }
+          .reverse
+      else
+        []
+      end
+
+      # Add the data to site.data
+      site.data["doc_versions"] = {"versions" => versions, "latest_version" => versions.first}
+    end
+  end
+end

config/site/src/getting-started.md

@@ -181,8 +181,9 @@ can be used in your publishing system to validate the indexing payloads or for c
 
 ## Resources
 
-- **ElasticGraph Documentation**: [{{ '/docs/main' | absolute_url }}]({{ '/docs/main' | relative_url }})
-- **GraphQL Introduction**: [https://graphql.org/learn/](https://graphql.org/learn/)
+- **[ElasticGraph Query API Documentation]({% link query-api.md %})**
+- **[ElasticGraph Ruby API Documentation]({{ '/docs/' | append: site.data.doc_versions.latest_version | relative_url }})**
+- **[GraphQL Introduction](https://graphql.org/learn/)**
 
 ## Feedback
 

elasticgraph-admin/elasticgraph-admin.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |spec|
   spec.metadata = {
     "bug_tracker_uri" => "https://github.com/block/elasticgraph/issues",
     "changelog_uri" => "https://github.com/block/elasticgraph/releases/tag/v#{ElasticGraph::VERSION}",
-    "documentation_uri" => "https://block.github.io/elasticgraph/docs/main/",
+    "documentation_uri" => "https://block.github.io/elasticgraph/docs/v#{ElasticGraph::VERSION}/",
     "homepage_uri" => "https://block.github.io/elasticgraph/",
     "source_code_uri" => "https://github.com/block/elasticgraph/tree/v#{ElasticGraph::VERSION}/#{spec.name}",
     "gem_category" => "core" # used by script/update_codebase_overview

elasticgraph-admin_lambda/elasticgraph-admin_lambda.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |spec|
   spec.metadata = {
     "bug_tracker_uri" => "https://github.com/block/elasticgraph/issues",
     "changelog_uri" => "https://github.com/block/elasticgraph/releases/tag/v#{ElasticGraph::VERSION}",
-    "documentation_uri" => "https://block.github.io/elasticgraph/docs/main/",
+    "documentation_uri" => "https://block.github.io/elasticgraph/docs/v#{ElasticGraph::VERSION}/",
     "homepage_uri" => "https://block.github.io/elasticgraph/",
     "source_code_uri" => "https://github.com/block/elasticgraph/tree/v#{ElasticGraph::VERSION}/#{spec.name}",
     "gem_category" => "lambda" # used by script/update_codebase_overview

elasticgraph-apollo/elasticgraph-apollo.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |spec|
   spec.metadata = {
     "bug_tracker_uri" => "https://github.com/block/elasticgraph/issues",
     "changelog_uri" => "https://github.com/block/elasticgraph/releases/tag/v#{ElasticGraph::VERSION}",
-    "documentation_uri" => "https://block.github.io/elasticgraph/docs/main/",
+    "documentation_uri" => "https://block.github.io/elasticgraph/docs/v#{ElasticGraph::VERSION}/",
     "homepage_uri" => "https://block.github.io/elasticgraph/",
     "source_code_uri" => "https://github.com/block/elasticgraph/tree/v#{ElasticGraph::VERSION}/#{spec.name}",
     "gem_category" => "extension" # used by script/update_codebase_overview

elasticgraph-datastore_core/elasticgraph-datastore_core.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |spec|
   spec.metadata = {
     "bug_tracker_uri" => "https://github.com/block/elasticgraph/issues",
     "changelog_uri" => "https://github.com/block/elasticgraph/releases/tag/v#{ElasticGraph::VERSION}",
-    "documentation_uri" => "https://block.github.io/elasticgraph/docs/main/",
+    "documentation_uri" => "https://block.github.io/elasticgraph/docs/v#{ElasticGraph::VERSION}/",
     "homepage_uri" => "https://block.github.io/elasticgraph/",
     "source_code_uri" => "https://github.com/block/elasticgraph/tree/v#{ElasticGraph::VERSION}/#{spec.name}",
     "gem_category" => "core" # used by script/update_codebase_overview

elasticgraph-elasticsearch/elasticgraph-elasticsearch.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |spec|
   spec.metadata = {
     "bug_tracker_uri" => "https://github.com/block/elasticgraph/issues",
     "changelog_uri" => "https://github.com/block/elasticgraph/releases/tag/v#{ElasticGraph::VERSION}",
-    "documentation_uri" => "https://block.github.io/elasticgraph/docs/main/",
+    "documentation_uri" => "https://block.github.io/elasticgraph/docs/v#{ElasticGraph::VERSION}/",
     "homepage_uri" => "https://block.github.io/elasticgraph/",
     "source_code_uri" => "https://github.com/block/elasticgraph/tree/v#{ElasticGraph::VERSION}/#{spec.name}",
     "gem_category" => "datastore_adapter" # used by script/update_codebase_overview

elasticgraph-graphql/elasticgraph-graphql.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |spec|
   spec.metadata = {
     "bug_tracker_uri" => "https://github.com/block/elasticgraph/issues",
     "changelog_uri" => "https://github.com/block/elasticgraph/releases/tag/v#{ElasticGraph::VERSION}",
-    "documentation_uri" => "https://block.github.io/elasticgraph/docs/main/",
+    "documentation_uri" => "https://block.github.io/elasticgraph/docs/v#{ElasticGraph::VERSION}/",
     "homepage_uri" => "https://block.github.io/elasticgraph/",
     "source_code_uri" => "https://github.com/block/elasticgraph/tree/v#{ElasticGraph::VERSION}/#{spec.name}",
     "gem_category" => "core" # used by script/update_codebase_overview

elasticgraph-graphql_lambda/elasticgraph-graphql_lambda.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |spec|
   spec.metadata = {
     "bug_tracker_uri" => "https://github.com/block/elasticgraph/issues",
     "changelog_uri" => "https://github.com/block/elasticgraph/releases/tag/v#{ElasticGraph::VERSION}",
-    "documentation_uri" => "https://block.github.io/elasticgraph/docs/main/",
+    "documentation_uri" => "https://block.github.io/elasticgraph/docs/v#{ElasticGraph::VERSION}/",
     "homepage_uri" => "https://block.github.io/elasticgraph/",
     "source_code_uri" => "https://github.com/block/elasticgraph/tree/v#{ElasticGraph::VERSION}/#{spec.name}",
     "gem_category" => "lambda" # used by script/update_codebase_overview

elasticgraph-health_check/elasticgraph-health_check.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |spec|
   spec.metadata = {
     "bug_tracker_uri" => "https://github.com/block/elasticgraph/issues",
     "changelog_uri" => "https://github.com/block/elasticgraph/releases/tag/v#{ElasticGraph::VERSION}",
-    "documentation_uri" => "https://block.github.io/elasticgraph/docs/main/",
+    "documentation_uri" => "https://block.github.io/elasticgraph/docs/v#{ElasticGraph::VERSION}/",
     "homepage_uri" => "https://block.github.io/elasticgraph/",
     "source_code_uri" => "https://github.com/block/elasticgraph/tree/v#{ElasticGraph::VERSION}/#{spec.name}",
     "gem_category" => "extension" # used by script/update_codebase_overview

elasticgraph-indexer/elasticgraph-indexer.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |spec|
   spec.metadata = {
     "bug_tracker_uri" => "https://github.com/block/elasticgraph/issues",
     "changelog_uri" => "https://github.com/block/elasticgraph/releases/tag/v#{ElasticGraph::VERSION}",
-    "documentation_uri" => "https://block.github.io/elasticgraph/docs/main/",
+    "documentation_uri" => "https://block.github.io/elasticgraph/docs/v#{ElasticGraph::VERSION}/",
     "homepage_uri" => "https://block.github.io/elasticgraph/",
     "source_code_uri" => "https://github.com/block/elasticgraph/tree/v#{ElasticGraph::VERSION}/#{spec.name}",
     "gem_category" => "core" # used by script/update_codebase_overview

elasticgraph-indexer_autoscaler_lambda/elasticgraph-indexer_autoscaler_lambda.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |spec|
   spec.metadata = {
     "bug_tracker_uri" => "https://github.com/block/elasticgraph/issues",
     "changelog_uri" => "https://github.com/block/elasticgraph/releases/tag/v#{ElasticGraph::VERSION}",
-    "documentation_uri" => "https://block.github.io/elasticgraph/docs/main/",
+    "documentation_uri" => "https://block.github.io/elasticgraph/docs/v#{ElasticGraph::VERSION}/",
     "homepage_uri" => "https://block.github.io/elasticgraph/",
     "source_code_uri" => "https://github.com/block/elasticgraph/tree/v#{ElasticGraph::VERSION}/#{spec.name}",
     "gem_category" => "lambda" # used by script/update_codebase_overview

elasticgraph-indexer_lambda/elasticgraph-indexer_lambda.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |spec|
   spec.metadata = {
     "bug_tracker_uri" => "https://github.com/block/elasticgraph/issues",
     "changelog_uri" => "https://github.com/block/elasticgraph/releases/tag/v#{ElasticGraph::VERSION}",
-    "documentation_uri" => "https://block.github.io/elasticgraph/docs/main/",
+    "documentation_uri" => "https://block.github.io/elasticgraph/docs/v#{ElasticGraph::VERSION}/",
     "homepage_uri" => "https://block.github.io/elasticgraph/",
     "source_code_uri" => "https://github.com/block/elasticgraph/tree/v#{ElasticGraph::VERSION}/#{spec.name}",
     "gem_category" => "lambda" # used by script/update_codebase_overview

elasticgraph-json_schema/elasticgraph-json_schema.gemspec

@@ -22,7 +22,7 @@ Gem::Specification.new do |spec|
   spec.metadata = {
     "bug_tracker_uri" => "https://github.com/block/elasticgraph/issues",
     "changelog_uri" => "https://github.com/block/elasticgraph/releases/tag/v#{ElasticGraph::VERSION}",
-    "documentation_uri" => "https://block.github.io/elasticgraph/docs/main/",
+    "documentation_uri" => "https://block.github.io/elasticgraph/docs/v#{ElasticGraph::VERSION}/",
     "homepage_uri" => "https://block.github.io/elasticgraph/",
     "source_code_uri" => "https://github.com/block/elasticgraph/tree/v#{ElasticGraph::VERSION}/#{spec.name}",
     "gem_category" => "core" # used by script/update_codebase_overview