Alternative missing value approach to subaggregations

elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/date_histogram_grouping.rb

@@ -53,6 +53,10 @@ def inner_meta
           INNER_META
         end
 
+        def handles_missing_values?
+          false
+        end
+
         INNER_META = {
           # On a date histogram aggregation, the `key` is formatted as a number (milliseconds since epoch). We
           # need it formatted as a string, which `key_as_string` provides.

elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/field_term_grouping.rb

@@ -11,12 +11,29 @@
 module ElasticGraph
   class GraphQL
     module Aggregation
-      class FieldTermGrouping < Support::MemoizableData.define(:field_path)
-        # @dynamic field_path
+      class FieldTermGrouping < Support::MemoizableData.define(:field_path, :field)
+        # @dynamic field_path, field
         include TermGrouping
 
-        private
+        # Random 18 bytes converted to base64. Used 18 bytes instead of 16 to avoid base64 padding.
+        # Since it uses more bytes, this has a lower probability of collission than a 16 byte random UUID.
+        MISSING_STRING_PLACEHOLDER = "f1TXKoApWwIG3U8ks9vVduvU"
+        MISSING_NUMERIC_PLACEHOLDER = "NaN"
+
+        def missing_value_placeholder
+          unwrapped_type = field.type.unwrap_fully
+          case unwrapped_type.name
+          when "String", "ID"
+            MISSING_STRING_PLACEHOLDER
+          when "Int", "JsonSafeLong", "Float"
+            MISSING_NUMERIC_PLACEHOLDER
+          else
+            unwrapped_type.enum? ? MISSING_STRING_PLACEHOLDER : nil
+          end
+        end
 
+        private
+
         def terms_subclause
           {"field" => encoded_index_field_path}
         end

elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/non_composite_grouping_adapter.rb

@@ -65,29 +65,36 @@ def grouping_detail(groupings, query)
           def format_buckets(sub_agg, buckets_path, parent_key_fields: {}, parent_key_values: [])
             agg_with_buckets = sub_agg.dig(*buckets_path)
 
-            missing_bucket = {
-              # Doc counts in missing value buckets are always perfectly accurate.
-              "doc_count_error_upper_bound" => 0
-            }.merge(sub_agg.dig(*missing_bucket_path_from(buckets_path))) # : ::Hash[::String, untyped]
 
             meta = agg_with_buckets.fetch("meta")
 
             grouping_field_names = meta.fetch("grouping_fields") # provides the names of the fields being grouped on
             key_path = meta.fetch("key_path") # indicates whether we want to get the key values from `key` or `key_as_string`.
             sub_buckets_path = meta["buckets_path"] # buckets_path is optional, so we don't use fetch.
+            missing_values = meta["missing_values"] # missing_values is optional, so we don't use fetch.
             merge_into_bucket = meta.fetch("merge_into_bucket")
 
             raw_buckets = agg_with_buckets.fetch("buckets") # : ::Array[::Hash[::String, untyped]]
 
             # If the missing bucket is non-empty, include it. This matches the behavior of composite aggregations
             # when the `missing_bucket` option is used.
-            raw_buckets += [missing_bucket] if missing_bucket.fetch("doc_count") > 0
+            missing_bucket = sub_agg.dig(*missing_bucket_path_from(buckets_path))
+            raw_buckets << {
+              # Doc counts in missing value buckets are always perfectly accurate.
+              "doc_count_error_upper_bound" => 0
+            }.merge(missing_bucket) if missing_bucket["doc_count"] > 0
 
             raw_buckets.flat_map do |raw_bucket|
               # The key will either be a single value (e.g. `47`) if we used a `terms`/`date_histogram` aggregation,
               # or a tuple of values (e.g. `[47, "abc"]`) if we used a `multi_terms` aggregation. Here we convert it
               # to the form needed for resolving `grouped_by` fields: a hash like `{"size" => 47, "tag" => "abc"}`.
               key_values = Array(raw_bucket.dig(*key_path))
+
+              # if missing_values are present, then for each element in key_values, replace with nil if it matches the placeholder value from missing_values
+              key_values = key_values.each_with_index.map do |value, index|
+                value == missing_values[index] ? nil : value
+              end if missing_values
+
               key_fields_hash = grouping_field_names.zip(key_values).to_h
 
               # If we have multiple levels of aggregations, we need to merge the key fields hash with the key fields from the parent levels.

elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/query.rb

@@ -149,32 +149,34 @@ def wrap_with_grouping(grouping, query:)
             "grouping_fields" => [agg_key]
           })
 
+          extra_inner_meta["missing_values"] = [grouping.missing_value_placeholder] if grouping.handles_missing_values?
+
           inner_agg_hash = {
             "aggs" => (clauses unless (clauses || {}).empty?),
             "meta" => meta.merge(extra_inner_meta)
           }.compact
 
-          missing_bucket_inner_agg_hash = inner_agg_hash.key?("aggs") ? inner_agg_hash : {} # : ::Hash[::String, untyped]
-
-          AggregationDetail.new(
-            {
-              agg_key => grouping.non_composite_clause_for(query).merge(inner_agg_hash),
-
-              # Here we include a `missing` aggregation as a sibling to the main grouping aggregation. We do this
-              # so that we get a bucket of documents that have `null` values for the field we are grouping on, in
-              # order to provide the same behavior as the `CompositeGroupingAdapter` (which uses the built-in
-              # `missing_bucket` option).
-              #
-              # To work correctly, we need to include this `missing` aggregation as a sibling at _every_ level of
-              # the aggregation structure, and the `missing` aggregation needs the same child aggregations as the
-              # main grouping aggregation has. Given the recursive nature of how this is applied, this results in
-              # a fairly complex structure, even though conceptually the idea behind this isn't _too_ bad.
-              Key.missing_value_bucket_key(agg_key) => {
-                "missing" => {"field" => grouping.encoded_index_field_path}
-              }.merge(missing_bucket_inner_agg_hash)
-            },
-            {"buckets_path" => [agg_key]}
-          )
+          wrapped_clauses = {
+              agg_key => grouping.non_composite_clause_for(query).merge(inner_agg_hash)
+          }
+
+          unless grouping.handles_missing_values?
+            # Here we include a `missing` aggregation as a sibling to the main grouping aggregation. We do this
+            # so that we get a bucket of documents that have `null` values for the field we are grouping on, in
+            # order to provide the same behavior as the `CompositeGroupingAdapter` (which uses the built-in
+            # `missing_bucket` option).
+            #
+            # To work correctly, we need to include this `missing` aggregation as a sibling at _every_ level of
+            # the aggregation structure, and the `missing` aggregation needs the same child aggregations as the
+            # main grouping aggregation has. Given the recursive nature of how this is applied, this results in
+            # a fairly complex structure, even though conceptually the idea behind this isn't _too_ bad.
+            missing_bucket_inner_agg_hash = inner_agg_hash.key?("aggs") ? inner_agg_hash : {} # : ::Hash[::String, untyped]
+            wrapped_clauses[Key.missing_value_bucket_key(agg_key)] = {
+              "missing" => {"field" => grouping.encoded_index_field_path}
+            }.merge(missing_bucket_inner_agg_hash)
+          end
+
+          AggregationDetail.new(wrapped_clauses, {"buckets_path" => [agg_key]})
         end
       end
     end

elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/query_adapter.rb

@@ -211,7 +211,7 @@ def build_groupings_from(node_node, aggregation_name, from_field_path: [])
               date_time_groupings_from(field_path: field_path, node: node)
             elsif !field.type.object?
               # Non-date/time grouping
-              [FieldTermGrouping.new(field_path: field_path)]
+              [FieldTermGrouping.new(field_path: field_path, field: field)]
             end
           end
         end

elasticgraph-graphql/lib/elastic_graph/graphql/aggregation/term_grouping.rb

@@ -31,6 +31,7 @@ def composite_clause(grouping_options: {})
 
         def non_composite_clause_for(query)
           clause_value = work_around_elasticsearch_bug(terms_subclause)
+          clause_value["missing"] = missing_value_placeholder if handles_missing_values?
           {
             "terms" => clause_value.merge({
               "size" => query.paginator.requested_page_size,
@@ -39,6 +40,14 @@ def non_composite_clause_for(query)
           }
         end
 
+        def missing_value_placeholder
+          nil
+        end
+
+        def handles_missing_values?
+          missing_value_placeholder != nil
+        end
+
         INNER_META = {"key_path" => ["key"], "merge_into_bucket" => {}}
 
         def inner_meta