diff --git a/gigl/common/data/dataloaders.py b/gigl/common/data/dataloaders.py index 824e5225d..67152c898 100644 --- a/gigl/common/data/dataloaders.py +++ b/gigl/common/data/dataloaders.py @@ -398,16 +398,6 @@ def load_as_torch_tensors( feature_spec_dict[entity_key] = tf.io.FixedLenFeature( shape=[], dtype=tf.int64 ) - if ( - packed_feature_key is not None - and packed_feature_key not in feature_spec_dict - ): - logger.info( - f"Injecting packed feature key {packed_feature_key} into feature spec dictionary with value `tf.io.FixedLenFeature(shape=[], dtype=tf.string)`" - ) - feature_spec_dict[packed_feature_key] = tf.io.FixedLenFeature( - shape=[], dtype=tf.string - ) else: id_concat_axis = 1 proccess_id_tensor = lambda t: tf.stack( @@ -433,6 +423,17 @@ def load_as_torch_tensors( shape=[], dtype=tf.int64 ) + if ( + packed_feature_key is not None + and packed_feature_key not in feature_spec_dict + ): + logger.info( + f"Injecting packed feature key {packed_feature_key} into feature spec dictionary with value `tf.io.FixedLenFeature(shape=[], dtype=tf.string)`" + ) + feature_spec_dict[packed_feature_key] = tf.io.FixedLenFeature( + shape=[], dtype=tf.string + ) + uris = self._partition_children_uris( serialized_tf_record_info.tfrecord_uri_prefix, serialized_tf_record_info.tfrecord_uri_pattern, diff --git a/gigl/common/data/load_torch_tensors.py b/gigl/common/data/load_torch_tensors.py index 3a1174888..667bc348d 100644 --- a/gigl/common/data/load_torch_tensors.py +++ b/gigl/common/data/load_torch_tensors.py @@ -1,7 +1,7 @@ import time import traceback -from dataclasses import dataclass -from typing import MutableMapping, Optional, Union +from dataclasses import dataclass, replace +from typing import MutableMapping, Optional, Union, cast import torch import torch.multiprocessing as mp @@ -119,6 +119,134 @@ class SerializedGraphMetadata: node_quantization_metadata: Optional[ Union[FeatureQuantizationMetadata, dict[NodeType, FeatureQuantizationMetadata]] ] = None + edge_quantization_metadata: Optional[ + Union[FeatureQuantizationMetadata, dict[EdgeType, FeatureQuantizationMetadata]] + ] = None + + +def _validate_weight_edge_feature_name( + edge_entity_info: Union[ + SerializedTFRecordInfo, dict[EdgeType, SerializedTFRecordInfo] + ], + weight_edge_feat_name: Optional[Union[str, dict[EdgeType, str]]], +) -> None: + if weight_edge_feat_name is None: + return + + configured_weights: list[tuple[EdgeType, str, SerializedTFRecordInfo]] + if isinstance(edge_entity_info, SerializedTFRecordInfo): + if not isinstance(weight_edge_feat_name, str): + raise ValueError("weight_edge_feat_name must be str for homogeneous graph") + edge_type = DEFAULT_HOMOGENEOUS_EDGE_TYPE + configured_weights = [(edge_type, weight_edge_feat_name, edge_entity_info)] + else: + if isinstance(weight_edge_feat_name, str): + if len(edge_entity_info) != 1: + raise ValueError( + "weight_edge_feat_name must be dict[EdgeType, str] for heterogeneous graph with multiple edge types" + ) + edge_type, serialized_info = next(iter(edge_entity_info.items())) + configured_weights = [(edge_type, weight_edge_feat_name, serialized_info)] + else: + unknown_edge_types = set(weight_edge_feat_name) - set(edge_entity_info) + if unknown_edge_types: + raise ValueError( + f"weight_edge_feat_name contains unknown edge types: {unknown_edge_types}" + ) + configured_weights = [ + (edge_type, feature_name, edge_entity_info[edge_type]) + for edge_type, feature_name in weight_edge_feat_name.items() + ] + + for edge_type, feature_name, serialized_info in configured_weights: + if feature_name not in serialized_info.feature_keys: + raise ValueError( + f"Sampling-weight field '{feature_name}' for edge type {edge_type} must be an unquantized raw edge feature." + ) + + +def remove_sampling_weight_from_edge_quantization_metadata( + serialized_graph_metadata: SerializedGraphMetadata, + weight_edge_feat_name: Optional[Union[str, dict[EdgeType, str]]], +) -> Optional[ + Union[FeatureQuantizationMetadata, dict[EdgeType, FeatureQuantizationMetadata]] +]: + """Remove separately stored sampling weights from edge reconstruction metadata. + + TFRecord loading removes the sampling-weight column from raw edge features + before registering it with the weighted sampler. The resulting metadata + must describe the remaining model features so batch reconstruction scatters + raw and dequantized columns into the correct positions. + + Args: + serialized_graph_metadata: Serialized edge schema and quantization metadata. + weight_edge_feat_name: Raw scalar feature configured as sampling weights. + + Returns: + Quantization metadata for the model-facing edge features. + """ + quantization_metadata = serialized_graph_metadata.edge_quantization_metadata + if quantization_metadata is None or weight_edge_feat_name is None: + return quantization_metadata + + if isinstance(serialized_graph_metadata.edge_entity_info, SerializedTFRecordInfo): + assert isinstance(quantization_metadata, FeatureQuantizationMetadata) + assert isinstance(weight_edge_feat_name, str) + edge_info_by_type: dict[EdgeType, SerializedTFRecordInfo] = { + DEFAULT_HOMOGENEOUS_EDGE_TYPE: serialized_graph_metadata.edge_entity_info + } + metadata_by_type: dict[EdgeType, FeatureQuantizationMetadata] = { + DEFAULT_HOMOGENEOUS_EDGE_TYPE: quantization_metadata + } + weight_by_type: dict[EdgeType, str] = { + DEFAULT_HOMOGENEOUS_EDGE_TYPE: weight_edge_feat_name + } + is_homogeneous = True + else: + assert isinstance(quantization_metadata, dict) + edge_info_by_type: dict[EdgeType, SerializedTFRecordInfo] = ( + serialized_graph_metadata.edge_entity_info + ) + metadata_by_type: dict[EdgeType, FeatureQuantizationMetadata] = cast( + dict[EdgeType, FeatureQuantizationMetadata], quantization_metadata + ) + if isinstance(weight_edge_feat_name, str): + edge_type = next(iter(edge_info_by_type)) + weight_by_type: dict[EdgeType, str] = {edge_type: weight_edge_feat_name} + else: + weight_by_type: dict[EdgeType, str] = weight_edge_feat_name + is_homogeneous = False + + adjusted_metadata: dict[EdgeType, FeatureQuantizationMetadata] = {} + for edge_type, metadata in metadata_by_type.items(): + weight_feature_name = weight_by_type.get(edge_type) + if weight_feature_name is None: + adjusted_metadata[edge_type] = metadata + continue + + edge_info = edge_info_by_type[edge_type] + raw_column_offset = 0 + for feature_name in edge_info.feature_keys: + if feature_name == weight_feature_name: + break + feature_spec = edge_info.feature_spec[feature_name] + raw_column_offset += feature_spec.shape[-1] if feature_spec.shape else 1 + weight_logical_index = metadata.raw_feature_indices[raw_column_offset] + adjusted_quantized_feature_indices = tuple( + quantized_feature_index - 1 + if quantized_feature_index > weight_logical_index + else quantized_feature_index + for quantized_feature_index in metadata.quantized_feature_indices + ) + adjusted_metadata[edge_type] = replace( + metadata, + feature_dim=metadata.feature_dim - 1, + quantized_feature_indices=adjusted_quantized_feature_indices, + ) + + if is_homogeneous: + return adjusted_metadata[DEFAULT_HOMOGENEOUS_EDGE_TYPE] + return adjusted_metadata def _data_loading_process( @@ -199,14 +327,6 @@ def _data_loading_process( raise NotImplementedError( "Label keys are not supported for edge entities" ) - if ( - serialized_entity_tf_record_info.packed_feature_key is not None - and not serialized_entity_tf_record_info.is_node_entity - ): - # TODO(quantization): Support feature quantization for edge features. - raise NotImplementedError( - "Packed feature keys are not supported for edge entities" - ) loaded_entity = tf_record_dataloader.load_as_torch_tensors( serialized_tf_record_info=serialized_entity_tf_record_info, tf_dataset_options=tf_dataset_options, @@ -396,6 +516,11 @@ def load_torch_tensors_from_tf_record( loaded_graph_tensors (LoadedGraphTensors): Unpartitioned Graph Tensors """ + _validate_weight_edge_feature_name( + edge_entity_info=serialized_graph_metadata.edge_entity_info, + weight_edge_feat_name=weight_edge_feat_name, + ) + logger.info(f"Rank {rank} starting loading torch tensors from serialized info ...") start_time = time.time() @@ -525,6 +650,9 @@ def load_torch_tensors_from_tf_record( edge_index = edge_output_dict[_ID_FMT.format(entity=_EDGE_KEY)] edge_features = edge_output_dict.get(_FEATURE_FMT.format(entity=_EDGE_KEY), None) + edge_quantized_features = edge_output_dict.get( + _PACKED_FEATURE_FMT.format(entity=_EDGE_KEY), None + ) edge_weights = edge_output_dict.get(_EDGE_WEIGHTS_KEY, None) positive_labels = edge_output_dict.get( @@ -552,6 +680,7 @@ def load_torch_tensors_from_tf_record( node_labels=node_labels, edge_index=edge_index, edge_features=edge_features, + edge_quantized_features=edge_quantized_features, positive_label=positive_labels, negative_label=negative_labels, edge_weights=edge_weights, diff --git a/gigl/common/utils/feature_quantization/README.md b/gigl/common/utils/feature_quantization/README.md index 181548b31..b8ece0cdc 100644 --- a/gigl/common/utils/feature_quantization/README.md +++ b/gigl/common/utils/feature_quantization/README.md @@ -22,11 +22,11 @@ this as a useful tradeoff for GiGL. The built-in flow is: 1. The data preprocessor computes feature summary statistics offline. -2. The preprocessor quantizes selected scalar feature columns with NumPy. +2. The preprocessor quantizes selected scalar node or main-edge feature columns with NumPy. 3. The packed `uint8` feature sidecar is written to TFRecords. 4. Distributed dataset construction partitions and samples the packed bytes. 5. The dataloader collate path dequantizes sampled packed features with Torch. -6. Dequantized columns are scattered back into the logical `x` feature matrix. +6. Dequantized columns are scattered back into the logical `x` or `edge_attr` feature matrix. The NumPy/Torch split is intentional: diff --git a/gigl/distributed/base_dist_loader.py b/gigl/distributed/base_dist_loader.py index ea2e91bfc..afe8d564f 100644 --- a/gigl/distributed/base_dist_loader.py +++ b/gigl/distributed/base_dist_loader.py @@ -246,6 +246,7 @@ def __init__( self._node_feature_info = dataset_schema.node_feature_info self._edge_feature_info = dataset_schema.edge_feature_info self._node_quantization_metadata = dataset_schema.node_quantization_metadata + self._edge_quantization_metadata = dataset_schema.edge_quantization_metadata self._sampler_options = sampler_options self._non_blocking_transfers = non_blocking_transfers @@ -435,7 +436,10 @@ def create_sampling_config( batch_size=batch_size, shuffle=shuffle, drop_last=drop_last, - with_edge=dataset_schema.edge_feature_info is not None, + with_edge=( + dataset_schema.edge_feature_info is not None + or dataset_schema.edge_quantization_metadata is not None + ), collect_features=True, with_neg=False, with_weight=with_weight, diff --git a/gigl/distributed/base_sampler.py b/gigl/distributed/base_sampler.py index 67ab6d183..e92f2cf3d 100644 --- a/gigl/distributed/base_sampler.py +++ b/gigl/distributed/base_sampler.py @@ -19,6 +19,7 @@ from gigl.common.logger import Logger from gigl.distributed.sampler import ( + EDGE_PACKED_FEATURES_METADATA_KEY, NEGATIVE_LABEL_METADATA_KEY, NODE_PACKED_FEATURES_METADATA_KEY, POSITIVE_LABEL_METADATA_KEY, @@ -116,6 +117,7 @@ def __init__(self, *args, **kwargs) -> None: self._sampling_error_sent: bool = False self.dist_node_quantized_feature: Optional[DistFeature] = None + self.dist_edge_quantized_feature: Optional[DistFeature] = None if ( self.collect_features and data is not None @@ -132,6 +134,20 @@ def __init__(self, *args, **kwargs) -> None: rpc_router=self.rpc_router, device=self.device, ) + if ( + self.collect_features + and data is not None + and getattr(data, "edge_quantized_features", None) is not None + ): + self.dist_edge_quantized_feature = DistFeature( + data.num_partitions, + data.partition_idx, + data.edge_quantized_features, + data.edge_pb, + local_only=False, + rpc_router=self.rpc_router, + device=self.device, + ) def _prepare_sample_loop_inputs( self, @@ -436,6 +452,8 @@ async def _collate_fn( ) if self.dist_edge_feature is not None and self.with_edge: for etype in self.edge_types: + if etype not in self.dist_edge_feature.local_feature: + continue if self.edge_dir == "in": eids = result_map.get( f"{as_str(reverse_edge_type(etype))}.eids", None @@ -451,6 +469,26 @@ async def _collate_fn( futs[result_key] = wrap_torch_future( self.dist_edge_feature.async_get(eids, etype) ) + if self.dist_edge_quantized_feature is not None and self.with_edge: + for etype in self.edge_types: + # Like node features, an edge partition book covers every + # edge type while a feature store may register only some. + if etype not in self.dist_edge_quantized_feature.local_feature: + continue + result_edge_type = ( + reverse_edge_type(etype) if self.edge_dir == "in" else etype + ) + eids = result_map.get(f"{as_str(result_edge_type)}.eids") + if eids is not None: + eids = eids.to(torch.long) + # GLT maps incoming wire edge types back to the dataset edge + # type during collation. Metadata bypasses that mapping, so its + # transport key must already match the final output store. + futs[f"#META.{EDGE_PACKED_FEATURES_METADATA_KEY}.{etype}"] = ( + wrap_torch_future( + self.dist_edge_quantized_feature.async_get(eids, etype) + ) + ) if output.batch is not None: for ntype, batch in output.batch.items(): result_map[f"{as_str(ntype)}.batch"] = batch @@ -490,6 +528,10 @@ async def _collate_fn( futs["efeats"] = wrap_torch_future( self.dist_edge_feature.async_get(eids) ) + if self.dist_edge_quantized_feature is not None: + futs[f"#META.{EDGE_PACKED_FEATURES_METADATA_KEY}"] = wrap_torch_future( + self.dist_edge_quantized_feature.async_get(result_map["eids"]) + ) if output.batch is not None: result_map["batch"] = output.batch diff --git a/gigl/distributed/dataset_factory.py b/gigl/distributed/dataset_factory.py index 1a5f859b5..0ffa3c462 100644 --- a/gigl/distributed/dataset_factory.py +++ b/gigl/distributed/dataset_factory.py @@ -25,6 +25,7 @@ SerializedGraphMetadata, TFDatasetOptions, load_torch_tensors_from_tf_record, + remove_sampling_weight_from_edge_quantization_metadata, ) from gigl.common.logger import Logger from gigl.common.utils.decorator import tf_on_cpu @@ -194,6 +195,10 @@ def _load_and_build_partitioned_dataset( partitioner.register_edge_features( edge_features=loaded_graph_tensors.edge_features ) + if loaded_graph_tensors.edge_quantized_features is not None: + partitioner.register_edge_quantized_features( + edge_quantized_features=loaded_graph_tensors.edge_quantized_features + ) if loaded_graph_tensors.positive_label is not None: partitioner.register_labels( label_edge_index=loaded_graph_tensors.positive_label, is_positive=True @@ -212,6 +217,7 @@ def _load_and_build_partitioned_dataset( loaded_graph_tensors.node_quantized_features, loaded_graph_tensors.edge_index, loaded_graph_tensors.edge_features, + loaded_graph_tensors.edge_quantized_features, loaded_graph_tensors.edge_weights, loaded_graph_tensors.positive_label, loaded_graph_tensors.negative_label, @@ -227,6 +233,10 @@ def _load_and_build_partitioned_dataset( world_size=world_size, edge_dir=edge_dir, node_quantization_metadata=serialized_graph_metadata.node_quantization_metadata, + edge_quantization_metadata=remove_sampling_weight_from_edge_quantization_metadata( + serialized_graph_metadata=serialized_graph_metadata, + weight_edge_feat_name=weight_edge_feat_name, + ), ) dataset.build( diff --git a/gigl/distributed/dist_ablp_neighborloader.py b/gigl/distributed/dist_ablp_neighborloader.py index 10638c330..b15d96dff 100644 --- a/gigl/distributed/dist_ablp_neighborloader.py +++ b/gigl/distributed/dist_ablp_neighborloader.py @@ -38,6 +38,7 @@ extract_edge_type_metadata, extract_metadata, labeled_to_homogeneous, + materialize_quantized_edge_features, materialize_quantized_node_features, set_missing_features, shard_nodes_by_process, @@ -609,6 +610,7 @@ def _setup_for_colocated( node_feature_info=dataset.node_feature_info, edge_feature_info=dataset.edge_feature_info, node_quantization_metadata=dataset.node_quantization_metadata, + edge_quantization_metadata=dataset.edge_quantization_metadata, edge_dir=dataset.edge_dir, ), ) @@ -799,6 +801,7 @@ def _setup_for_graph_store( node_feature_info=node_feature_info, edge_feature_info=edge_feature_info, node_quantization_metadata=dataset.fetch_node_quantization_metadata(), + edge_quantization_metadata=dataset.fetch_edge_quantization_metadata(), edge_dir=edge_dir, ), backend_key, @@ -972,6 +975,11 @@ def _collate_fn(self, msg: SampleMessage) -> Union[Data, HeteroData]: metadata=metadata, node_quantization_metadata=self._node_quantization_metadata, ) + data, metadata = materialize_quantized_edge_features( + data=data, + metadata=metadata, + edge_quantization_metadata=self._edge_quantization_metadata, + ) # Attach any remaining metadata (e.g. custom user-defined keys) directly onto the # data object so downstream code can access them via attribute lookup. diff --git a/gigl/distributed/dist_dataset.py b/gigl/distributed/dist_dataset.py index a41c0e12a..3ad3a7167 100644 --- a/gigl/distributed/dist_dataset.py +++ b/gigl/distributed/dist_dataset.py @@ -4,7 +4,7 @@ import time from collections.abc import Mapping from multiprocessing.reduction import ForkingPickler -from typing import Literal, Optional, Tuple, TypeVar, Union, overload +from typing import Literal, Optional, Tuple, TypeVar, Union, cast, overload import graphlearn_torch as glt import torch @@ -58,6 +58,9 @@ def __init__( node_quantized_feature_partition: Optional[ Union[Feature, dict[NodeType, Feature]] ] = None, + edge_quantized_feature_partition: Optional[ + Union[Feature, dict[EdgeType, Feature]] + ] = None, edge_feature_partition: Optional[ Union[Feature, dict[EdgeType, Feature]] ] = None, @@ -87,6 +90,11 @@ def __init__( dict[NodeType, FeatureQuantizationMetadata], ] ] = None, + edge_quantization_metadata: Optional[ + Union[ + FeatureQuantizationMetadata, dict[EdgeType, FeatureQuantizationMetadata] + ] + ] = None, edge_feature_info: Optional[ Union[FeatureInfo, dict[EdgeType, FeatureInfo]] ] = None, @@ -166,6 +174,8 @@ def __init__( self._node_quantized_features = node_quantized_feature_partition self._node_quantization_metadata = node_quantization_metadata + self._edge_quantized_features = edge_quantized_feature_partition + self._edge_quantization_metadata = edge_quantization_metadata self._degree_tensor: Optional[ Union[torch.Tensor, dict[NodeType, torch.Tensor]] @@ -253,6 +263,13 @@ def edge_features( ): self._edge_features = new_edge_features + @property + def edge_quantized_features( + self, + ) -> Optional[Union[Feature, dict[EdgeType, Feature]]]: + """Packed uint8 main-edge feature sidecar.""" + return self._edge_quantized_features + @property def node_pb( self, @@ -340,6 +357,15 @@ def node_quantization_metadata( ]: return self._node_quantization_metadata + @property + def edge_quantization_metadata( + self, + ) -> Optional[ + Union[FeatureQuantizationMetadata, dict[EdgeType, FeatureQuantizationMetadata]] + ]: + """Metadata required to materialize packed main-edge features.""" + return self._edge_quantization_metadata + @property def edge_feature_info( self, @@ -899,6 +925,43 @@ def _initialize_edge_features( ) logger.info(f"Initialized edge features for homogeneous graph to dataset") + def _initialize_edge_quantized_features( + self, + edge_partition_book: Union[PartitionBook, dict[EdgeType, PartitionBook]], + partitioned_edge_quantized_features: Optional[ + Union[FeaturePartitionData, dict[EdgeType, FeaturePartitionData]] + ], + ) -> None: + """Initialize packed uint8 main-edge feature storage.""" + features, id_to_index = _prepare_feature_data( + partition_book=edge_partition_book, + partitioned_data=partitioned_edge_quantized_features, + ) + if features is None or id_to_index is None: + logger.info("Found no packed quantized edge features to initialize") + return + if isinstance(features, Mapping): + assert isinstance(id_to_index, Mapping) + features = cast(dict[EdgeType, torch.Tensor], features) + id_to_index = cast(dict[EdgeType, torch.Tensor], id_to_index) + self._edge_quantized_features = { + edge_type: Feature( + feature_tensor=features_per_edge_type, + id2index=id_to_index[edge_type], + with_gpu=False, + dtype=torch.uint8, + ) + for edge_type, features_per_edge_type in features.items() + } + else: + assert not isinstance(id_to_index, Mapping) + self._edge_quantized_features = Feature( + feature_tensor=features, + id2index=id_to_index, + with_gpu=False, + dtype=torch.uint8, + ) + def build( self, partition_output: PartitionOutput, @@ -1011,6 +1074,13 @@ def build( partition_output.partitioned_edge_features = None gc.collect() + self._initialize_edge_quantized_features( + edge_partition_book=partition_output.edge_partition_book, + partitioned_edge_quantized_features=partition_output.partitioned_edge_quantized_features, + ) + partition_output.partitioned_edge_quantized_features = None + gc.collect() + self._node_partition_book = partition_output.node_partition_book self._edge_partition_book = partition_output.edge_partition_book @@ -1037,6 +1107,7 @@ def share_ipc( Optional[Union[Feature, dict[NodeType, Feature]]], Optional[Union[Feature, dict[NodeType, Feature]]], Optional[Union[Feature, dict[EdgeType, Feature]]], + Optional[Union[Feature, dict[EdgeType, Feature]]], Optional[Union[Feature, dict[NodeType, Feature]]], Optional[Union[PartitionBook, dict[NodeType, PartitionBook]]], Optional[Union[PartitionBook, dict[EdgeType, PartitionBook]]], @@ -1053,6 +1124,12 @@ def share_ipc( dict[NodeType, FeatureQuantizationMetadata], ] ], + Optional[ + Union[ + FeatureQuantizationMetadata, + dict[EdgeType, FeatureQuantizationMetadata], + ] + ], Optional[Union[FeatureInfo, dict[EdgeType, FeatureInfo]]], Optional[Union[torch.Tensor, dict[NodeType, torch.Tensor]]], Optional[int], @@ -1067,6 +1144,7 @@ def share_ipc( Optional[Union[Graph, dict[EdgeType, Graph]]]: Partitioned Graph Data Optional[Union[Feature, dict[NodeType, Feature]]]: Partitioned Node Feature Data Optional[Union[Feature, dict[NodeType, Feature]]]: Partitioned packed uint8 node feature data + Optional[Union[Feature, dict[EdgeType, Feature]]]: Partitioned packed uint8 edge feature data Optional[Union[Feature, dict[EdgeType, Feature]]]: Partitioned Edge Feature Data Optional[Union[Feature, dict[NodeType, Feature]]]: Node labels on the current machine. Will be a dict if heterogeneous. Optional[Union[torch.Tensor, dict[NodeType, torch.Tensor]]]: Node Partition Book Tensor @@ -1079,6 +1157,7 @@ def share_ipc( Optional[Union[int, dict[NodeType, int]]]: Number of test nodes on the current machine. Will be a dict if heterogeneous. Optional[Union[FeatureInfo, dict[NodeType, FeatureInfo]]]: Node feature dim and its data type, will be a dict if heterogeneous Optional[Union[FeatureQuantizationMetadata, dict[NodeType, FeatureQuantizationMetadata]]]: Node quantization metadata. + Optional[Union[FeatureQuantizationMetadata, dict[EdgeType, FeatureQuantizationMetadata]]]: Edge quantization metadata. Optional[Union[FeatureInfo, dict[EdgeType, FeatureInfo]]]: Edge feature dim and its data type, will be a dict if heterogeneous Optional[Union[torch.Tensor, dict[NodeType, torch.Tensor]]]: Degree tensors Optional[int]: Optional per-anchor label cap for ABLP label fetching @@ -1100,6 +1179,7 @@ def share_ipc( self._graph, self._node_features, self._node_quantized_features, + self._edge_quantized_features, self._edge_features, self._node_labels, self._node_partition_book, @@ -1112,6 +1192,7 @@ def share_ipc( self._num_test, # Additional field unique to DistDataset class self._node_feature_info, # Additional field unique to DistDataset class self._node_quantization_metadata, # Additional field unique to DistDataset class + self._edge_quantization_metadata, # Additional field unique to DistDataset class self._edge_feature_info, # Additional field unique to DistDataset class self._degree_tensor, # Additional field unique to DistDataset class self._max_labels_per_anchor_node, # Additional field unique to DistDataset class @@ -1348,6 +1429,9 @@ def _rebuild_distributed_dataset( Optional[ Union[Feature, dict[NodeType, Feature]] ], # Partitioned packed uint8 node feature data + Optional[ + Union[Feature, dict[EdgeType, Feature]] + ], # Partitioned packed uint8 edge feature data Optional[ Union[Feature, dict[EdgeType, Feature]] ], # Partitioned Edge Feature Data @@ -1377,6 +1461,12 @@ def _rebuild_distributed_dataset( dict[NodeType, FeatureQuantizationMetadata], ] ], # Node quantization metadata + Optional[ + Union[ + FeatureQuantizationMetadata, + dict[EdgeType, FeatureQuantizationMetadata], + ] + ], # Edge quantization metadata Optional[ Union[FeatureInfo, dict[EdgeType, FeatureInfo]] ], # Edge feature dim and its data type diff --git a/gigl/distributed/dist_partitioner.py b/gigl/distributed/dist_partitioner.py index 04de8ce72..9549c9e3f 100644 --- a/gigl/distributed/dist_partitioner.py +++ b/gigl/distributed/dist_partitioner.py @@ -208,6 +208,8 @@ def __init__( self._edge_ids: Optional[dict[EdgeType, tuple[int, int]]] = None self._edge_feat: Optional[dict[EdgeType, torch.Tensor]] = None self._edge_feat_dim: Optional[dict[EdgeType, int]] = None + self._edge_quantized_feat: Optional[dict[EdgeType, torch.Tensor]] = None + self._edge_quantized_feat_dim: Optional[dict[EdgeType, int]] = None self._edge_weights: Optional[dict[EdgeType, torch.Tensor]] = None # TODO (mkolodner-sc): Deprecate the need for explicitly storing labels are part of this class, leveraging @@ -669,6 +671,36 @@ def register_edge_features( for edge_type in input_edge_features: self._edge_feat_dim[edge_type] = input_edge_features[edge_type].shape[1] + def register_edge_quantized_features( + self, edge_quantized_features: Union[torch.Tensor, dict[EdgeType, torch.Tensor]] + ) -> None: + """Register packed uint8 main-edge features for co-partitioning.""" + + self._assert_and_get_rpc_setup() + if self._edge_quantized_feat is not None: + raise ValueError( + "Edge quantized features have already been registered. Cannot re-register edge quantized feature data." + ) + logger.info("Registering Edge Quantized Features ...") + + input_edge_quantized_features = ( + self._convert_edge_entity_to_heterogeneous_format( + input_edge_entity=edge_quantized_features + ) + ) + + assert input_edge_quantized_features, ( + "Edge quantized features is an empty dictionary. Please provide edge quantized features to register." + ) + + self._edge_quantized_feat = convert_to_tensor( + input_edge_quantized_features, dtype=torch.uint8 + ) + self._edge_quantized_feat_dim = { + edge_type: features.shape[1] + for edge_type, features in input_edge_quantized_features.items() + } + def register_edge_weights( self, edge_weights: Union[torch.Tensor, dict[EdgeType, torch.Tensor]] ) -> None: @@ -1201,7 +1233,10 @@ def _partition_edge_index_and_edge_features( node_partition_book: dict[NodeType, PartitionBook], edge_type: EdgeType, ) -> Tuple[ - GraphPartitionData, Optional[FeaturePartitionData], Optional[PartitionBook] + GraphPartitionData, + Optional[FeaturePartitionData], + Optional[FeaturePartitionData], + Optional[PartitionBook], ]: r"""Partition graph topology and edge features of a specific edge type. If there are no edge features for the current edge type, both the returned edge feature and edge partition book will be None. @@ -1213,6 +1248,7 @@ def _partition_edge_index_and_edge_features( Returns: GraphPartitionData: The graph data of the current partition. Optional[FeaturePartitionData]: The edge features on the current partition, will be None if there are no edge features for the current edge type + Optional[FeaturePartitionData]: The quantized edge features on the current partition, will be None if there are no quantized edge features for the current edge type Optional[PartitionBook]: The partition book of graph edges, will be None if there are no edge features for the current edge type """ @@ -1225,11 +1261,17 @@ def _partition_edge_index_and_edge_features( ), "Must have registered edges prior to partitioning them" has_edge_feats = self._edge_feat is not None and edge_type in self._edge_feat + has_edge_quantized_feats = ( + self._edge_quantized_feat is not None + and edge_type in self._edge_quantized_feat + ) has_weights_for_edge_type = ( self._edge_weights is not None and edge_type in self._edge_weights ) # Need a partition book if we have features or weights to reindex. - should_generate_partition_book = has_edge_feats or has_weights_for_edge_type + should_generate_partition_book = ( + has_edge_feats or has_edge_quantized_feats or has_weights_for_edge_type + ) # Partitioning Edge Indices @@ -1283,12 +1325,12 @@ def _edge_pfn(_, chunk_range): gc.collect() - # Partition edge features and weights together in a single pass, + # Partition edge features, packed features, and weights together in a single pass, # mirroring how node features and labels are co-partitioned. - # Input tuple layout: (edge_feat?, edge_weights?, edge_ids) - # IDs are always at r[-1]; features at r[0]; weights at r[1] when - # features are also present, else r[0]. + # Input tuple layout: (edge_feat?, edge_quantized_feat?, edge_weights?, edge_ids) + # IDs are always last; optional tensor indices are recorded when appended. current_feat_part: Optional[FeaturePartitionData] = None + current_quantized_feat_part: Optional[FeaturePartitionData] = None partitioned_weights: Optional[torch.Tensor] = None partitioned_edge_ids: Optional[torch.Tensor] = None @@ -1309,6 +1351,8 @@ def _edge_pfn(_, chunk_range): edge_feat: Optional[torch.Tensor] = None edge_feat_dim: Optional[int] = None edge_weights_tensor: Optional[torch.Tensor] = None + edge_quantized_features: Optional[torch.Tensor] = None + edge_quantized_feature_dim: Optional[int] = None if has_edge_feats: assert self._edge_feat is not None and edge_type in self._edge_feat assert ( @@ -1316,30 +1360,38 @@ def _edge_pfn(_, chunk_range): ) edge_feat = self._edge_feat[edge_type] edge_feat_dim = self._edge_feat_dim[edge_type] + if has_edge_quantized_feats: + assert self._edge_quantized_feat is not None + assert self._edge_quantized_feat_dim is not None + edge_quantized_features = self._edge_quantized_feat[edge_type] + edge_quantized_feature_dim = self._edge_quantized_feat_dim[edge_type] if has_weights_for_edge_type: assert self._edge_weights is not None edge_weights_tensor = self._edge_weights[edge_type] input_parts: list[torch.Tensor] = [] + feat_idx: Optional[int] = None if edge_feat is not None: + feat_idx = len(input_parts) input_parts.append(edge_feat) + quantized_feat_idx: Optional[int] = None + if edge_quantized_features is not None: + quantized_feat_idx = len(input_parts) + input_parts.append(edge_quantized_features) + weight_idx: Optional[int] = None if edge_weights_tensor is not None: + weight_idx = len(input_parts) input_parts.append(edge_weights_tensor) input_parts.append(edge_ids) - # Positional indices: features first, weights next, ids always last. - feat_idx: Optional[int] = 0 if has_edge_feats else None - weight_idx: Optional[int] = None - if has_weights_for_edge_type: - weight_idx = 1 if has_edge_feats else 0 - + # Recorded indices keep result unpacking aligned with optional inputs. def _edge_feat_weight_pfn( ids_chunk: torch.Tensor, _: object ) -> torch.Tensor: assert edge_partition_book is not None return edge_partition_book[ids_chunk] - # Each result tuple contains (edge_feat?, edge_weights?, edge_ids). + # Each result tuple preserves the input tuple layout. feat_weight_res_list, _ = self._partition_by_chunk( input_data=tuple(input_parts), rank_indices=edge_ids, @@ -1360,6 +1412,21 @@ def _edge_feat_weight_pfn( if len(self._edge_feat) == 0 and len(self._edge_feat_dim) == 0: self._edge_feat = None self._edge_feat_dim = None + if has_edge_quantized_feats: + assert edge_quantized_features is not None + assert self._edge_quantized_feat is not None + assert self._edge_quantized_feat_dim is not None + del edge_quantized_features + del ( + self._edge_quantized_feat[edge_type], + self._edge_quantized_feat_dim[edge_type], + ) + if ( + len(self._edge_quantized_feat) == 0 + and len(self._edge_quantized_feat_dim) == 0 + ): + self._edge_quantized_feat = None + self._edge_quantized_feat_dim = None if has_weights_for_edge_type: assert edge_weights_tensor is not None assert self._edge_weights is not None @@ -1377,6 +1444,14 @@ def _edge_feat_weight_pfn( feats=torch.empty(0, edge_feat_dim), ids=partitioned_edge_ids, ) + if has_edge_quantized_feats: + assert edge_quantized_feature_dim is not None + current_quantized_feat_part = FeaturePartitionData( + feats=torch.empty( + 0, edge_quantized_feature_dim, dtype=torch.uint8 + ), + ids=partitioned_edge_ids, + ) if has_weights_for_edge_type: partitioned_weights = torch.empty(0) else: @@ -1387,6 +1462,14 @@ def _edge_feat_weight_pfn( feats=torch.cat([r[feat_idx] for r in feat_weight_res_list]), ids=partitioned_edge_ids, ) + if has_edge_quantized_feats: + assert quantized_feat_idx is not None + current_quantized_feat_part = FeaturePartitionData( + feats=torch.cat( + [r[quantized_feat_idx] for r in feat_weight_res_list] + ), + ids=partitioned_edge_ids, + ) if has_weights_for_edge_type: assert weight_idx is not None partitioned_weights = torch.cat( @@ -1410,7 +1493,12 @@ def _edge_feat_weight_pfn( weights=partitioned_weights, ) - return current_graph_part, current_feat_part, edge_partition_book + return ( + current_graph_part, + current_feat_part, + current_quantized_feat_part, + edge_partition_book, + ) def _partition_label_edge_index( self, @@ -1683,11 +1771,15 @@ def partition_edge_index_and_edge_features( self, node_partition_book: Union[PartitionBook, dict[NodeType, PartitionBook]] ) -> Union[ Tuple[ - GraphPartitionData, Optional[FeaturePartitionData], Optional[PartitionBook] + GraphPartitionData, + Optional[FeaturePartitionData], + Optional[FeaturePartitionData], + Optional[PartitionBook], ], Tuple[ dict[EdgeType, GraphPartitionData], Optional[dict[EdgeType, FeaturePartitionData]], + Optional[dict[EdgeType, FeaturePartitionData]], Optional[dict[EdgeType, PartitionBook]], ], ]: @@ -1698,8 +1790,8 @@ def partition_edge_index_and_edge_features( node_partition_book (Union[PartitionBook, dict[NodeType, PartitionBook]]): The computed Node Partition Book Returns: Union[ - Tuple[GraphPartitionData, FeaturePartitionData, PartitionBook], - Tuple[dict[EdgeType, GraphPartitionData], Optional[dict[EdgeType, FeaturePartitionData]], Optional[dict[EdgeType, PartitionBook]]], + Tuple[GraphPartitionData, Optional[FeaturePartitionData], Optional[FeaturePartitionData], Optional[PartitionBook]], + Tuple[dict[EdgeType, GraphPartitionData], Optional[dict[EdgeType, FeaturePartitionData]], Optional[dict[EdgeType, FeaturePartitionData]], Optional[dict[EdgeType, PartitionBook]]], ]: Partitioned Graph Data, Feature Data, and corresponding edge partition book, is a dictionary if heterogeneous. The second and third elements of this tuple are only present if there are edge features to partition, and are None otherwise. @@ -1748,21 +1840,27 @@ def partition_edge_index_and_edge_features( edge_partition_book: dict[EdgeType, PartitionBook] = {} partitioned_edge_index: dict[EdgeType, GraphPartitionData] = {} partitioned_edge_features: dict[EdgeType, FeaturePartitionData] = {} + partitioned_edge_quantized_features: dict[EdgeType, FeaturePartitionData] = {} for edge_type in self._edge_types: ( partitioned_edge_index_per_edge_type, partitioned_edge_features_per_edge_type, + partitioned_edge_quantized_features_per_edge_type, edge_partition_book_per_edge_type, ) = self._partition_edge_index_and_edge_features( node_partition_book=transformed_node_partition_book, edge_type=edge_type ) partitioned_edge_index[edge_type] = partitioned_edge_index_per_edge_type - if partitioned_edge_features_per_edge_type is not None: - assert edge_partition_book_per_edge_type is not None + if edge_partition_book_per_edge_type is not None: edge_partition_book[edge_type] = edge_partition_book_per_edge_type + if partitioned_edge_features_per_edge_type is not None: partitioned_edge_features[edge_type] = ( partitioned_edge_features_per_edge_type ) + if partitioned_edge_quantized_features_per_edge_type is not None: + partitioned_edge_quantized_features[edge_type] = ( + partitioned_edge_quantized_features_per_edge_type + ) elapsed_time = time.time() - start_time logger.info(f"Edge Partitioning finished, took {elapsed_time:.3f}s") @@ -1784,6 +1882,9 @@ def partition_edge_index_and_edge_features( to_homogeneous(partitioned_edge_features) if partitioned_edge_features else None, + to_homogeneous(partitioned_edge_quantized_features) + if partitioned_edge_quantized_features + else None, to_homogeneous(edge_partition_book) if edge_partition_book else None, ) else: @@ -1791,6 +1892,11 @@ def partition_edge_index_and_edge_features( return ( partitioned_edge_index, partitioned_edge_features if partitioned_edge_features else None, + ( + partitioned_edge_quantized_features + if partitioned_edge_quantized_features + else None + ), edge_partition_book if edge_partition_book else None, ) @@ -1889,6 +1995,7 @@ def partition( ( partitioned_edge_index, partitioned_edge_features, + partitioned_edge_quantized_features, edge_partition_book, ) = self.partition_edge_index_and_edge_features( node_partition_book=node_partition_book @@ -1936,6 +2043,7 @@ def partition( partitioned_node_features=partitioned_node_features, partitioned_node_quantized_features=partitioned_node_quantized_features, partitioned_edge_features=partitioned_edge_features, + partitioned_edge_quantized_features=partitioned_edge_quantized_features, partitioned_positive_labels=partitioned_positive_edge_index, partitioned_negative_labels=partitioned_negative_edge_index, partitioned_node_labels=partitioned_node_labels, diff --git a/gigl/distributed/dist_range_partitioner.py b/gigl/distributed/dist_range_partitioner.py index b7b0754f7..170d5de09 100644 --- a/gigl/distributed/dist_range_partitioner.py +++ b/gigl/distributed/dist_range_partitioner.py @@ -215,7 +215,10 @@ def _partition_edge_index_and_edge_features( node_partition_book: dict[NodeType, PartitionBook], edge_type: EdgeType, ) -> tuple[ - GraphPartitionData, Optional[FeaturePartitionData], Optional[PartitionBook] + GraphPartitionData, + Optional[FeaturePartitionData], + Optional[FeaturePartitionData], + Optional[PartitionBook], ]: """ Partition graph topology of a specific edge type. For range-based partitioning, we partition @@ -232,6 +235,7 @@ def _partition_edge_index_and_edge_features( Returns: GraphPartitionData: The graph data of the current partition. Optional[FeaturePartitionData]: The edge features on the current partition, will be None if there are no edge features for the current edge type + Optional[FeaturePartitionData]: The quantized edge features on the current partition, will be None if there are no quantized edge features for the current edge type Optional[PartitionBook]: The partition book of graph edges, will be None if there are no edge features for the current edge type """ @@ -243,6 +247,10 @@ def _partition_edge_index_and_edge_features( edge_index = self._edge_index[edge_type] has_edge_feats = self._edge_feat is not None and edge_type in self._edge_feat + has_edge_quantized_feats = ( + self._edge_quantized_feat is not None + and edge_type in self._edge_quantized_feat + ) has_edge_weights = ( self._edge_weights is not None and edge_type in self._edge_weights ) @@ -255,24 +263,35 @@ def _partition_edge_index_and_edge_features( edge_feat: Optional[torch.Tensor] = None edge_feat_dim: Optional[int] = None edge_weights_tensor: Optional[torch.Tensor] = None + edge_quantized_features: Optional[torch.Tensor] = None + edge_quantized_feature_dim: Optional[int] = None if has_edge_feats: assert self._edge_feat is not None and self._edge_feat_dim is not None assert edge_type in self._edge_feat_dim edge_feat = self._edge_feat[edge_type] edge_feat_dim = self._edge_feat_dim[edge_type] + if has_edge_quantized_feats: + assert self._edge_quantized_feat is not None + assert self._edge_quantized_feat_dim is not None + edge_quantized_features = self._edge_quantized_feat[edge_type] + edge_quantized_feature_dim = self._edge_quantized_feat_dim[edge_type] if has_edge_weights: assert self._edge_weights is not None edge_weights_tensor = self._edge_weights[edge_type] - # Build input_data tuple: (src, dst[, feat][, weights]) - # Track the index of each optional tensor so we can unpack res_list correctly. + # Build input_data as (src, dst[, feat][, packed feat][, weights]). + # Recorded indices keep result unpacking aligned with optional inputs. input_parts: list[torch.Tensor] = [edge_index[0], edge_index[1]] feat_idx: Optional[int] = None weight_idx: Optional[int] = None if edge_feat is not None: feat_idx = len(input_parts) input_parts.append(edge_feat) + quantized_feat_idx: Optional[int] = None + if edge_quantized_features is not None: + quantized_feat_idx = len(input_parts) + input_parts.append(edge_quantized_features) if edge_weights_tensor is not None: weight_idx = len(input_parts) input_parts.append(edge_weights_tensor) @@ -301,6 +320,15 @@ def edge_partition_fn(rank_indices, _): del self._edge_feat[edge_type], self._edge_feat_dim[edge_type] if self._edge_weights is not None and edge_type in self._edge_weights: del self._edge_weights[edge_type] + if ( + self._edge_quantized_feat is not None + and edge_type in self._edge_quantized_feat + ): + assert self._edge_quantized_feat_dim is not None + del ( + self._edge_quantized_feat[edge_type], + self._edge_quantized_feat_dim[edge_type], + ) # We check if edge_index or edge_feat dict is empty after deleting the tensor. If so, we set these fields to None. if not self._edge_index: @@ -310,6 +338,9 @@ def edge_partition_fn(rank_indices, _): self._edge_feat_dim = None if self._edge_weights is not None and not self._edge_weights: self._edge_weights = None + if self._edge_quantized_feat is not None and not self._edge_quantized_feat: + self._edge_quantized_feat = None + self._edge_quantized_feat_dim = None gc.collect() @@ -319,6 +350,11 @@ def edge_partition_fn(rank_indices, _): torch.empty(0, edge_feat_dim) if edge_feat_dim is not None else None ) partitioned_weights = torch.empty(0) if has_edge_weights else None + partitioned_edge_quantized_features = ( + torch.empty(0, edge_quantized_feature_dim, dtype=torch.uint8) + if edge_quantized_feature_dim is not None + else None + ) else: partitioned_edge_index = torch.stack( ( @@ -337,12 +373,17 @@ def edge_partition_fn(rank_indices, _): if weight_idx is not None else None ) + partitioned_edge_quantized_features = ( + torch.cat([r[quantized_feat_idx] for r in res_list]) + if quantized_feat_idx is not None + else None + ) res_list.clear() gc.collect() - # Generate range-based edge partition book and infer edge IDs. - # Only needed when edge features are present — weights use positional IDs. + # Generate range-based edge partition book and infer edge IDs for every + # sidecar that requires sampled edge lookup. num_edges_on_each_rank: list[tuple[int, int]] = sorted( all_gather((self._rank, partitioned_edge_index.size(1))).values(), key=lambda x: x[0], @@ -354,21 +395,26 @@ def edge_partition_fn(rank_indices, _): partition_ranges.append((start, end)) start = end - if edge_feat_dim is not None: + if ( + edge_feat_dim is not None + or edge_quantized_feature_dim is not None + or has_edge_weights + ): edge_partition_book = RangePartitionBook( partition_ranges=partition_ranges, partition_idx=self._rank ) partitioned_edge_ids = get_ids_on_rank( partition_book=edge_partition_book, rank=self._rank ) - assert partitioned_edge_features is not None current_graph_part = GraphPartitionData( edge_index=partitioned_edge_index, edge_ids=partitioned_edge_ids, weights=partitioned_weights, ) - current_feat_part = FeaturePartitionData( - feats=partitioned_edge_features, ids=None + current_feat_part = ( + FeaturePartitionData(feats=partitioned_edge_features, ids=None) + if partitioned_edge_features is not None + else None ) logger.info( f"Got edge range-based partition book for edge type {edge_type} on rank {self._rank} with partition bounds: {edge_partition_book.partition_bounds}" @@ -386,17 +432,31 @@ def edge_partition_fn(rank_indices, _): f"Edge Index and Feature Partitioning for edge type {edge_type} finished, took {time.time() - start_time:.3f}s" ) - return current_graph_part, current_feat_part, edge_partition_book + current_quantized_feat_part = ( + FeaturePartitionData(feats=partitioned_edge_quantized_features, ids=None) + if partitioned_edge_quantized_features is not None + else None + ) + return ( + current_graph_part, + current_feat_part, + current_quantized_feat_part, + edge_partition_book, + ) def partition_edge_index_and_edge_features( self, node_partition_book: Union[PartitionBook, dict[NodeType, PartitionBook]] ) -> Union[ tuple[ - GraphPartitionData, Optional[FeaturePartitionData], Optional[PartitionBook] + GraphPartitionData, + Optional[FeaturePartitionData], + Optional[FeaturePartitionData], + Optional[PartitionBook], ], tuple[ dict[EdgeType, GraphPartitionData], Optional[dict[EdgeType, FeaturePartitionData]], + Optional[dict[EdgeType, FeaturePartitionData]], Optional[dict[EdgeType, PartitionBook]], ], ]: @@ -408,10 +468,11 @@ def partition_edge_index_and_edge_features( Args: node_partition_book (Union[PartitionBook, dict[NodeType, PartitionBook]]): The computed Node Partition Book + Returns: Union[ - Tuple[GraphPartitionData, Optional[FeaturePartitionData], Optional[PartitionBook]], - Tuple[dict[EdgeType, GraphPartitionData], Optional[dict[EdgeType, FeaturePartitionData]], Optional[dict[EdgeType, PartitionBook]]], + Tuple[GraphPartitionData, Optional[FeaturePartitionData], Optional[FeaturePartitionData], Optional[PartitionBook]], + Tuple[dict[EdgeType, GraphPartitionData], Optional[dict[EdgeType, FeaturePartitionData]], Optional[dict[EdgeType, FeaturePartitionData]], Optional[dict[EdgeType, PartitionBook]]], ]: Partitioned Graph Data, Feature Data, and corresponding edge partition book, is a dictionary if heterogeneous. """ @@ -448,21 +509,27 @@ def partition_edge_index_and_edge_features( edge_partition_book: dict[EdgeType, PartitionBook] = {} partitioned_edge_index: dict[EdgeType, GraphPartitionData] = {} partitioned_edge_features: dict[EdgeType, FeaturePartitionData] = {} + partitioned_edge_quantized_features: dict[EdgeType, FeaturePartitionData] = {} for edge_type in self._edge_types: ( partitioned_edge_index_per_edge_type, partitioned_edge_features_per_edge_type, + partitioned_edge_quantized_features_per_edge_type, edge_partition_book_per_edge_type, ) = self._partition_edge_index_and_edge_features( node_partition_book=transformed_node_partition_book, edge_type=edge_type ) partitioned_edge_index[edge_type] = partitioned_edge_index_per_edge_type - if partitioned_edge_features_per_edge_type is not None: - assert edge_partition_book_per_edge_type is not None + if edge_partition_book_per_edge_type is not None: edge_partition_book[edge_type] = edge_partition_book_per_edge_type + if partitioned_edge_features_per_edge_type is not None: partitioned_edge_features[edge_type] = ( partitioned_edge_features_per_edge_type ) + if partitioned_edge_quantized_features_per_edge_type is not None: + partitioned_edge_quantized_features[edge_type] = ( + partitioned_edge_quantized_features_per_edge_type + ) elapsed_time = time.time() - start_time logger.info(f"Edge Partitioning finished, took {elapsed_time:.3f}s") @@ -481,6 +548,9 @@ def partition_edge_index_and_edge_features( to_homogeneous(partitioned_edge_features) if partitioned_edge_features else None, + to_homogeneous(partitioned_edge_quantized_features) + if partitioned_edge_quantized_features + else None, to_homogeneous(edge_partition_book) if edge_partition_book else None, ) else: @@ -488,5 +558,10 @@ def partition_edge_index_and_edge_features( return ( partitioned_edge_index, partitioned_edge_features if partitioned_edge_features else None, + ( + partitioned_edge_quantized_features + if partitioned_edge_quantized_features + else None + ), edge_partition_book if edge_partition_book else None, ) diff --git a/gigl/distributed/distributed_neighborloader.py b/gigl/distributed/distributed_neighborloader.py index 3effa01c3..6764cd582 100644 --- a/gigl/distributed/distributed_neighborloader.py +++ b/gigl/distributed/distributed_neighborloader.py @@ -29,6 +29,7 @@ SamplingClusterSetup, extract_metadata, labeled_to_homogeneous, + materialize_quantized_edge_features, materialize_quantized_node_features, set_missing_features, shard_nodes_by_process, @@ -413,6 +414,7 @@ def _setup_for_graph_store( node_feature_info=node_feature_info, edge_feature_info=edge_feature_info, node_quantization_metadata=dataset.fetch_node_quantization_metadata(), + edge_quantization_metadata=dataset.fetch_edge_quantization_metadata(), edge_dir=dataset.fetch_edge_dir(), ), backend_key, @@ -531,6 +533,7 @@ def _setup_for_colocated( node_feature_info=dataset.node_feature_info, edge_feature_info=dataset.edge_feature_info, node_quantization_metadata=dataset.node_quantization_metadata, + edge_quantization_metadata=dataset.edge_quantization_metadata, edge_dir=dataset.edge_dir, ), ) @@ -564,6 +567,11 @@ def _collate_fn(self, msg: SampleMessage) -> Union[Data, HeteroData]: metadata=metadata, node_quantization_metadata=self._node_quantization_metadata, ) + data, metadata = materialize_quantized_edge_features( + data=data, + metadata=metadata, + edge_quantization_metadata=self._edge_quantization_metadata, + ) # Attach any remaining metadata (e.g. custom user-defined keys) directly onto the # data object so downstream code can access them via attribute lookup. diff --git a/gigl/distributed/graph_store/dist_server.py b/gigl/distributed/graph_store/dist_server.py index 0a92d959d..b608be433 100644 --- a/gigl/distributed/graph_store/dist_server.py +++ b/gigl/distributed/graph_store/dist_server.py @@ -418,6 +418,14 @@ def get_node_quantization_metadata( """Get node feature quantization metadata from the dataset.""" return self.dataset.node_quantization_metadata + def get_edge_quantization_metadata( + self, + ) -> Union[ + FeatureQuantizationMetadata, dict[EdgeType, FeatureQuantizationMetadata], None + ]: + """Get main-edge feature quantization metadata from the dataset.""" + return self.dataset.edge_quantization_metadata + def get_edge_feature_info( self, ) -> Union[FeatureInfo, dict[EdgeType, FeatureInfo], None]: diff --git a/gigl/distributed/graph_store/remote_dist_dataset.py b/gigl/distributed/graph_store/remote_dist_dataset.py index 81609961b..127078fca 100644 --- a/gigl/distributed/graph_store/remote_dist_dataset.py +++ b/gigl/distributed/graph_store/remote_dist_dataset.py @@ -80,6 +80,14 @@ def fetch_node_quantization_metadata( DistServer.get_node_quantization_metadata, ) + def fetch_edge_quantization_metadata( + self, + ) -> Union[ + FeatureQuantizationMetadata, dict[EdgeType, FeatureQuantizationMetadata], None + ]: + """Fetch main-edge feature quantization metadata from storage.""" + return request_server(0, DistServer.get_edge_quantization_metadata) + def fetch_edge_feature_info( self, ) -> Union[FeatureInfo, dict[EdgeType, FeatureInfo], None]: diff --git a/gigl/distributed/sampler.py b/gigl/distributed/sampler.py index 7789c6731..1e01ee85f 100644 --- a/gigl/distributed/sampler.py +++ b/gigl/distributed/sampler.py @@ -9,6 +9,7 @@ POSITIVE_LABEL_METADATA_KEY: Final[str] = "gigl_positive_labels." NEGATIVE_LABEL_METADATA_KEY: Final[str] = "gigl_negative_labels." NODE_PACKED_FEATURES_METADATA_KEY: Final[str] = "node_packed_features" +EDGE_PACKED_FEATURES_METADATA_KEY: Final[str] = "edge_packed_features" class ABLPNodeSamplerInput(NodeSamplerInput): diff --git a/gigl/distributed/utils/neighborloader.py b/gigl/distributed/utils/neighborloader.py index 6b81306ee..4f316298b 100644 --- a/gigl/distributed/utils/neighborloader.py +++ b/gigl/distributed/utils/neighborloader.py @@ -10,13 +10,17 @@ import torch from graphlearn_torch.channel import SampleMessage from torch_geometric.data import Data, HeteroData -from torch_geometric.data.storage import NodeStorage +from torch_geometric.data.storage import EdgeStorage, NodeStorage from torch_geometric.typing import EdgeType, NodeType from gigl.common.logger import Logger from gigl.common.utils.feature_quantization.torch_ops import dequantize_torch_tensor -from gigl.distributed.sampler import NODE_PACKED_FEATURES_METADATA_KEY +from gigl.distributed.sampler import ( + EDGE_PACKED_FEATURES_METADATA_KEY, + NODE_PACKED_FEATURES_METADATA_KEY, +) from gigl.types.graph import ( + DEFAULT_HOMOGENEOUS_EDGE_TYPE, DEFAULT_HOMOGENEOUS_NODE_TYPE, FeatureInfo, FeatureQuantizationIndexTensors, @@ -59,6 +63,9 @@ class DatasetSchema: node_quantization_metadata: Optional[ Union[FeatureQuantizationMetadata, dict[NodeType, FeatureQuantizationMetadata]] ] = None + edge_quantization_metadata: Optional[ + Union[FeatureQuantizationMetadata, dict[EdgeType, FeatureQuantizationMetadata]] + ] = None def patch_fanout_for_sampling( @@ -337,6 +344,49 @@ def set_missing_features( return data +def _materialize_quantized_features( + store: Union[Data, NodeStorage, EdgeStorage], + packed_features: torch.Tensor, + quantization_metadata: FeatureQuantizationMetadata, + feature_attribute: Literal["x", "edge_attr"], +) -> None: + """Reconstruct and assign quantized features for one PyG feature store. + + Args: + store: PyG data or typed storage receiving the reconstructed features. + packed_features: Quantized feature columns for sampled graph entities. + quantization_metadata: Column layout and dequantization metadata. + feature_attribute: PyG attribute that stores the feature tensor. + + Raises: + ValueError: If expected raw feature columns are absent or have an + unexpected dimension. + """ + dequantized = dequantize_torch_tensor( + packed_features, metadata=quantization_metadata + ) + raw_features = getattr(store, feature_attribute, None) + materialized_features = dequantized.new_empty( + (dequantized.size(0), quantization_metadata.feature_dim) + ) + scatter_idx: FeatureQuantizationIndexTensors = ( + quantization_metadata.scatter_index_tensors(materialized_features.device) + ) + materialized_features[:, scatter_idx.quantized] = dequantized + + if raw_features is None and quantization_metadata.raw_feature_dim: + raise ValueError( + f"Missing {quantization_metadata.raw_feature_dim} unquantized features" + ) + if raw_features is not None: + if raw_features.size(1) != quantization_metadata.raw_feature_dim: + raise ValueError( + f"Expected {quantization_metadata.raw_feature_dim} raw features before dequantization, got {raw_features.size(1)}" + ) + materialized_features[:, scatter_idx.raw] = raw_features + setattr(store, feature_attribute, materialized_features) + + def materialize_quantized_node_features( data: _GraphType, metadata: dict[str, torch.Tensor], @@ -371,48 +421,6 @@ def materialize_quantized_node_features( if node_quantization_metadata is None: return data, metadata - def materialize( - store: Union[Data, NodeStorage], - packed_features: torch.Tensor, - quantization_metadata: FeatureQuantizationMetadata, - ) -> None: - """Reconstruct and assign node features for one PyG node store. - - Args: - store: Node store receiving the reconstructed ``x`` tensor. - packed_features: Quantized feature columns for the sampled nodes. - quantization_metadata: Column layout and dequantization metadata. - - Raises: - ValueError: If expected raw feature columns are absent or have an - unexpected dimension. - """ - dequantized = dequantize_torch_tensor( - packed_features, metadata=quantization_metadata - ) - x = getattr(store, "x", None) - out = dequantized.new_empty( - (dequantized.size(0), quantization_metadata.feature_dim) - ) - scatter_idx: FeatureQuantizationIndexTensors = ( - quantization_metadata.scatter_index_tensors(out.device) - ) - out[:, scatter_idx.quantized] = dequantized - - if x is None and quantization_metadata.raw_feature_dim: - raise ValueError( - f"Missing {quantization_metadata.raw_feature_dim} unquantized features" - ) - if x is not None: - if x.size(1) != quantization_metadata.raw_feature_dim: - raise ValueError( - "Expected " - f"{quantization_metadata.raw_feature_dim} raw node features " - f"before dequantization, got {x.size(1)}" - ) - out[:, scatter_idx.raw] = x - store.x = out - if isinstance(data, Data): if isinstance(node_quantization_metadata, dict): raise ValueError("Expect scalar quantization metadata for homogeneous data") @@ -430,7 +438,12 @@ def materialize( raise ValueError( f"Missing packed quantized features in metadata keys {NODE_PACKED_FEATURES_METADATA_KEY} or {labeled_homogeneous_packed_features_key}" ) - materialize(data, packed_features, node_quantization_metadata) + _materialize_quantized_features( + data, + packed_features, + node_quantization_metadata, + feature_attribute="x", + ) else: if not isinstance(node_quantization_metadata, dict): raise ValueError("Expected per-node-type metadata for heterogeneous data.") @@ -442,7 +455,94 @@ def materialize( packed_features = metadata.pop(metadata_key, None) if packed_features is None: continue - materialize(data[node_type], packed_features, quantization_metadata) + _materialize_quantized_features( + data[node_type], + packed_features, + quantization_metadata, + feature_attribute="x", + ) + + return data, metadata + + +def materialize_quantized_edge_features( + data: _GraphType, + metadata: dict[str, torch.Tensor], + edge_quantization_metadata: Optional[ + Union[FeatureQuantizationMetadata, dict[EdgeType, FeatureQuantizationMetadata]] + ], +) -> tuple[_GraphType, dict[str, torch.Tensor]]: + """Materialize packed quantized edge features into PyG edge feature tensors. + + Reconstructs each edge feature tensor in its original column order by + dequantizing packed features and combining them with any unquantized + feature columns already present in ``data``. Consumed packed-feature + entries are removed from ``metadata``. + + Args: + data: Homogeneous or heterogeneous sampled graph containing raw edge + feature columns. + metadata: Sample metadata containing packed edge feature tensors. + edge_quantization_metadata: Quantization metadata for the graph's edge + features. Homogeneous graphs require a single value; heterogeneous + graphs require metadata for each edge type. + + Returns: + A tuple containing the graph with reconstructed edge features and the + remaining sample metadata. + + Raises: + ValueError: If the graph and quantization metadata shapes do not match, + required packed features are missing, or raw feature dimensions are + inconsistent. + """ + if edge_quantization_metadata is None: + return data, metadata + + if isinstance(data, Data): + if isinstance(edge_quantization_metadata, dict): + raise ValueError("Expect scalar quantization metadata for homogeneous data") + packed_features = metadata.pop(EDGE_PACKED_FEATURES_METADATA_KEY, None) + labeled_homogeneous_packed_features_key = ( + f"{EDGE_PACKED_FEATURES_METADATA_KEY}.{DEFAULT_HOMOGENEOUS_EDGE_TYPE}" + ) + if packed_features is None: + # Labeled homogeneous graphs are sampled as heterogeneous graphs, so + # the packed-feature transport key retains the default edge type. + packed_features = metadata.pop( + labeled_homogeneous_packed_features_key, None + ) + if packed_features is None: + raise ValueError( + f"Missing packed quantized features in metadata keys {EDGE_PACKED_FEATURES_METADATA_KEY} or {labeled_homogeneous_packed_features_key}" + ) + _materialize_quantized_features( + data, + packed_features, + edge_quantization_metadata, + feature_attribute="edge_attr", + ) + else: + if not isinstance(edge_quantization_metadata, dict): + raise ValueError("Expected per-edge-type metadata for heterogeneous data.") + edge_quantization_metadata = cast( + dict[EdgeType, FeatureQuantizationMetadata], edge_quantization_metadata + ) + for edge_type, quantization_metadata in edge_quantization_metadata.items(): + metadata_key = f"{EDGE_PACKED_FEATURES_METADATA_KEY}.{edge_type}" + packed_features = metadata.pop(metadata_key, None) + if packed_features is None: + if edge_type not in data.edge_types or data[edge_type].num_edges == 0: + continue + raise ValueError( + f"Missing packed quantized edge features for sampled edge type {edge_type}" + ) + _materialize_quantized_features( + data[edge_type], + packed_features, + quantization_metadata, + feature_attribute="edge_attr", + ) return data, metadata diff --git a/gigl/distributed/utils/serialized_graph_metadata_translator.py b/gigl/distributed/utils/serialized_graph_metadata_translator.py index 36ad31c52..25fb26882 100644 --- a/gigl/distributed/utils/serialized_graph_metadata_translator.py +++ b/gigl/distributed/utils/serialized_graph_metadata_translator.py @@ -33,18 +33,11 @@ def _build_serialized_tfrecord_entity_info( entity_key (Union[str, Tuple[str, str]]): Entity key to register to SerializedTFRecordInfo, is a str if Node entity or Tuple[str, str] if Edge entity tfrecord_uri_pattern (str): Regex pattern for loading serialized tf records quantization_metadata (Optional[FeatureQuantizationMetadata]): Quantization - metadata for a node entity, when its features are quantized. + metadata for a node or main-edge entity when its features are quantized. Returns: SerializedTFRecordInfo: Stored metadata for current entity """ if quantization_metadata is not None: - if not isinstance( - preprocessed_metadata, PreprocessedMetadata.NodeMetadataOutput - ): - # TODO(quantization): Support edge feature quantization. - raise NotImplementedError( - "Feature quantization is not supported for edge entities." - ) packed_feature_key = ( preprocessed_metadata.quantized_feature_metadata.packed_feature_key ) @@ -146,6 +139,7 @@ def convert_pb_to_serialized_graph_metadata( positive_label_entity_info: dict[EdgeType, SerializedTFRecordInfo] = {} negative_label_entity_info: dict[EdgeType, SerializedTFRecordInfo] = {} node_quantization_metadata: dict[NodeType, FeatureQuantizationMetadata] = {} + edge_quantization_metadata: dict[EdgeType, FeatureQuantizationMetadata] = {} preprocessed_metadata_pb = preprocessed_metadata_pb_wrapper.preprocessed_metadata_pb @@ -202,11 +196,19 @@ def convert_pb_to_serialized_graph_metadata( edge_feature_spec_dict = preprocessed_metadata_pb_wrapper.condensed_edge_type_to_feature_schema_map[ condensed_edge_type ].feature_spec + if edge_metadata.main_edge_info.HasField("quantized_feature_metadata"): + edge_quantization_metadata[edge_type] = ( + _build_feature_quantization_metadata( + quantized_metadata=edge_metadata.main_edge_info.quantized_feature_metadata, + feature_dim=edge_metadata.main_edge_info.feature_dim, + ) + ) edge_entity_info[edge_type] = _build_serialized_tfrecord_entity_info( preprocessed_metadata=edge_metadata.main_edge_info, feature_spec_dict=edge_feature_spec_dict, entity_key=edge_key, tfrecord_uri_pattern=tfrecord_uri_pattern, + quantization_metadata=edge_quantization_metadata.get(edge_type), ) if edge_metadata.HasField("positive_edge_info"): @@ -251,6 +253,9 @@ def convert_pb_to_serialized_graph_metadata( node_quantization_metadata=to_homogeneous(node_quantization_metadata) if len(node_quantization_metadata) > 0 else None, + edge_quantization_metadata=to_homogeneous(edge_quantization_metadata) + if len(edge_quantization_metadata) > 0 + else None, ) else: return SerializedGraphMetadata( @@ -265,4 +270,7 @@ def convert_pb_to_serialized_graph_metadata( node_quantization_metadata=node_quantization_metadata if len(node_quantization_metadata) > 0 else None, + edge_quantization_metadata=edge_quantization_metadata + if len(edge_quantization_metadata) > 0 + else None, ) diff --git a/gigl/src/data_preprocessor/data_preprocessor.py b/gigl/src/data_preprocessor/data_preprocessor.py index 9d84a8b42..9c9151993 100644 --- a/gigl/src/data_preprocessor/data_preprocessor.py +++ b/gigl/src/data_preprocessor/data_preprocessor.py @@ -74,6 +74,38 @@ logger = Logger() +def _load_feature_quantization_metadata_pb( + metadata_path: str, entity_description: str +) -> preprocessed_metadata_pb2.PreprocessedMetadata.FeatureQuantizationMetadata: + if not tf.io.gfile.exists(metadata_path): + raise RuntimeError( + f"Quantization metadata was expected for {entity_description}, " + f"but was not produced at {metadata_path}." + ) + logger.info( + f"Loading {entity_description} quantization metadata from {metadata_path}" + ) + with tf.io.gfile.GFile(metadata_path) as metadata_file: + metadata = json.loads(metadata_file.read()) + logger.info(f"Loaded {entity_description} quantization metadata {metadata}") + + quantization_metadata = ( + preprocessed_metadata_pb2.PreprocessedMetadata.FeatureQuantizationMetadata( + packed_feature_key=metadata["packed_feature_key"], + quantized_feature_indices=metadata["quantized_feature_indices"], + ) + ) + bits = metadata["bits"] + if bits == 1: + quantization_metadata.single_bit_state.neg_mean = metadata["neg_mean"] + quantization_metadata.single_bit_state.pos_mean = metadata["pos_mean"] + else: + quantization_metadata.multi_bit_state.bits = bits + quantization_metadata.multi_bit_state.clip_min = metadata["clip_min"] + quantization_metadata.multi_bit_state.clip_max = metadata["clip_max"] + return quantization_metadata + + class PreprocessedMetadataReferences(NamedTuple): node_data: dict[NodeDataReference, TransformedFeaturesInfo] edge_data: dict[EdgeDataReference, TransformedFeaturesInfo] @@ -216,13 +248,18 @@ def __preprocess_single_data_reference( f"Got {type(data_reference)}." ) - if isinstance(preprocessing_spec, NodeDataPreprocessingSpec): + if isinstance( + preprocessing_spec, (NodeDataPreprocessingSpec, EdgeDataPreprocessingSpec) + ): feature_quantization_enabled = ( preprocessing_spec.feature_quantization_spec is not None ) - else: - # TODO(quantization): Support quantization for edge features. - feature_quantization_enabled = False + if ( + isinstance(data_reference, EdgeDataReference) + and feature_quantization_enabled + and data_reference.edge_usage_type != EdgeUsageType.MAIN + ): + raise ValueError("Feature quantization is supported only for main edges.") transformed_features_info = TransformedFeaturesInfo( applied_task_identifier=self.applied_task_identifier, @@ -428,7 +465,7 @@ def _generate_edge_metadata_info_pb( transformed_features_info: TransformedFeaturesInfo, enumerated_edge_metadata: EnumeratorEdgeTypeMetadata, ) -> preprocessed_metadata_pb2.PreprocessedMetadata.EdgeMetadataInfo: - return preprocessed_metadata_pb2.PreprocessedMetadata.EdgeMetadataInfo( + output = preprocessed_metadata_pb2.PreprocessedMetadata.EdgeMetadataInfo( tfrecord_uri_prefix=transformed_features_info.transformed_features_file_prefix.uri, schema_uri=transformed_features_info.transformed_features_schema_path.uri, feature_keys=transformed_features_info.features_outputs, @@ -437,6 +474,13 @@ def _generate_edge_metadata_info_pb( feature_dim=transformed_features_info.feature_dim_output, transform_fn_assets_uri=transformed_features_info.transformed_features_transform_fn_assets_path.uri, ) + if transformed_features_info.feature_quantization_enabled: + quantization_metadata = _load_feature_quantization_metadata_pb( + metadata_path=transformed_features_info.feature_quantization_metadata_path.uri, + entity_description=f"edge type {transformed_features_info.entity_type}", + ) + output.quantized_feature_metadata.CopyFrom(quantization_metadata) + return output def generate_preprocessed_metadata_pb( self, @@ -492,30 +536,10 @@ def generate_preprocessed_metadata_pb( transform_fn_assets_uri=node_transformed_features_info.transformed_features_transform_fn_assets_path.uri, ) if node_transformed_features_info.feature_quantization_enabled: - metadata_path = node_transformed_features_info.feature_quantization_metadata_path.uri - if not tf.io.gfile.exists(metadata_path): - raise RuntimeError( - f"Quantization metadata was expected for node type {node_type}, " - f"but was not produced at {metadata_path}." - ) - logger.info(f"Loading node quantization metadata from {metadata_path}") - with tf.io.gfile.GFile(metadata_path) as f: - metadata = json.loads(f.read()) - logger.info(f"Loaded node quantization metadata {metadata}") - bits = metadata["bits"] - quantized_feature_metadata_pb = preprocessed_metadata_pb2.PreprocessedMetadata.FeatureQuantizationMetadata( - packed_feature_key=metadata["packed_feature_key"], - quantized_feature_indices=metadata["quantized_feature_indices"], + quantized_feature_metadata_pb = _load_feature_quantization_metadata_pb( + metadata_path=node_transformed_features_info.feature_quantization_metadata_path.uri, + entity_description=f"node type {node_type}", ) - if bits == 1: - single_bit_state = quantized_feature_metadata_pb.single_bit_state - single_bit_state.neg_mean = metadata["neg_mean"] - single_bit_state.pos_mean = metadata["pos_mean"] - else: - multi_bit_state = quantized_feature_metadata_pb.multi_bit_state - multi_bit_state.bits = bits - multi_bit_state.clip_min = metadata["clip_min"] - multi_bit_state.clip_max = metadata["clip_max"] node_metadata_output_pb.quantized_feature_metadata.CopyFrom( quantized_feature_metadata_pb ) @@ -782,6 +806,7 @@ def inner() -> FeatureSpecDict: pretrained_tft_model_uri=input_edge_preprocessing_spec.pretrained_tft_model_uri, features_outputs=input_edge_preprocessing_spec.features_outputs, labels_outputs=input_edge_preprocessing_spec.labels_outputs, + feature_quantization_spec=input_edge_preprocessing_spec.feature_quantization_spec, ) enumerated_edge_refs_to_preprocessing_specs[ enumerated_edge_metadata.enumerated_edge_data_reference diff --git a/gigl/src/data_preprocessor/lib/transform/feature_quantization.py b/gigl/src/data_preprocessor/lib/transform/feature_quantization.py index db7ecf500..0e1b7b3b2 100644 --- a/gigl/src/data_preprocessor/lib/transform/feature_quantization.py +++ b/gigl/src/data_preprocessor/lib/transform/feature_quantization.py @@ -15,7 +15,8 @@ from gigl.src.data_preprocessor.lib.types import FeatureQuantizationSpec logger = Logger() -_NODE_PACKED_FEATURE_KEY: Final[str] = "node_packed_features" +NODE_PACKED_FEATURE_KEY: Final[str] = "node_packed_features" +EDGE_PACKED_FEATURE_KEY: Final[str] = "edge_packed_features" _SignStats: TypeAlias = tuple[float, int, float, int] @@ -25,11 +26,14 @@ def apply_feature_quantization_transform( logical_feature_keys: list[str], quantization_spec: FeatureQuantizationSpec, quantization_metadata_path: str, + packed_feature_key: str, ) -> tuple[beam.PCollection[pa.RecordBatch], DatasetMetadata | beam.pvalue.AsSingleton]: """Quantizes selected feature columns and bit-packs each record's values. - Stores the packed bytes in ``node_packed_features`` and computes global - quantization statistics with Beam. + Stores packed bytes under ``packed_feature_key`` and computes global + quantization statistics with Beam. Node preprocessing uses + ``node_packed_features``; main-edge preprocessing uses + ``edge_packed_features``. Side Effects: Writes the quantization statistics JSON that ``data_preprocessor.py`` @@ -46,12 +50,17 @@ def apply_feature_quantization_transform( logical_feature_keys: Logical feature columns in original feature-vector order. quantization_spec: Feature keys and bit width to quantize. quantization_metadata_path: Destination for the quantization statistics JSON. + packed_feature_key: Reserved physical field used for packed values. Returns: Quantized RecordBatches and eager or deferred physical I/O metadata. That metadata removes quantized feature columns and adds - ``node_packed_features``. It affects serialized-record I/O only; the + ``packed_feature_key``. It affects serialized-record I/O only; the logical model schema remains unchanged. + + Raises: + ValueError: If the reserved packed key already exists, a selected feature + is absent or non-scalar, or feature values cannot be quantized. """ missing = set(quantization_spec.feature_keys) - set(logical_feature_keys) if missing: @@ -73,6 +82,7 @@ def apply_feature_quantization_transform( quantization_spec=quantization_spec, logical_feature_keys=logical_feature_keys, logical_metadata=metadata_for_json, + packed_feature_key=packed_feature_key, ) | "Write quantization stats" >> beam.io.WriteToText( @@ -86,19 +96,24 @@ def apply_feature_quantization_transform( _quantize_record_batch, quantization_spec=quantization_spec, quantization_stats=beam.pvalue.AsSingleton(quantization_stats), + packed_feature_key=packed_feature_key, ) ) if logical_metadata_is_eager: physical_feature_metadata = DatasetMetadata( - _apply_quantization_schema(logical_metadata.schema, quantization_spec) + _apply_quantization_schema( + logical_metadata.schema, quantization_spec, packed_feature_key + ) ) else: physical_feature_metadata = logical_metadata | ( "Apply feature quantization schema" >> beam.Map( lambda metadata, quantization_spec: DatasetMetadata( - _apply_quantization_schema(metadata.schema, quantization_spec) + _apply_quantization_schema( + metadata.schema, quantization_spec, packed_feature_key + ) ), quantization_spec=quantization_spec, ) @@ -139,7 +154,12 @@ def _quantize_record_batch( batch: pa.RecordBatch, quantization_spec: FeatureQuantizationSpec, quantization_stats: dict[str, float], + packed_feature_key: str, ) -> pa.RecordBatch: + if packed_feature_key in batch.schema.names: + raise ValueError( + f"Reserved packed feature key {packed_feature_key} already exists in the logical schema." + ) feature_matrix = _build_feature_matrix(batch, quantization_spec.feature_keys) if quantization_spec.bits == 1: packed = quantize_ndarray(feature_matrix, bits=quantization_spec.bits) @@ -161,7 +181,7 @@ def _quantize_record_batch( arrays.append( pa.array([[row.tobytes()] for row in packed], type=pa.list_(pa.binary())) ) - names.append(_NODE_PACKED_FEATURE_KEY) + names.append(packed_feature_key) return pa.RecordBatch.from_arrays(arrays, names=names) @@ -170,9 +190,10 @@ def _quantization_stats_to_json( quantization_spec: FeatureQuantizationSpec, logical_feature_keys: list[str], logical_metadata: DatasetMetadata, + packed_feature_key: str, ) -> str: metadata = { - "packed_feature_key": _NODE_PACKED_FEATURE_KEY, + "packed_feature_key": packed_feature_key, "quantized_feature_indices": _quantized_feature_indices( logical_metadata, logical_feature_keys, quantization_spec.feature_keys ), @@ -204,9 +225,15 @@ def _quantized_feature_indices( def _apply_quantization_schema( - schema: schema_pb2.Schema, quantization_spec: FeatureQuantizationSpec + schema: schema_pb2.Schema, + quantization_spec: FeatureQuantizationSpec, + packed_feature_key: str, ) -> schema_pb2.Schema: - drop_keys = set(quantization_spec.feature_keys) | {_NODE_PACKED_FEATURE_KEY} + if any(feature.name == packed_feature_key for feature in schema.feature): + raise ValueError( + f"Reserved packed feature key {packed_feature_key} already exists in the logical schema." + ) + drop_keys = set(quantization_spec.feature_keys) quantized_schema = schema_pb2.Schema() quantized_schema.CopyFrom(schema) del quantized_schema.feature[:] @@ -214,14 +241,14 @@ def _apply_quantization_schema( feature for feature in schema.feature if feature.name not in drop_keys ) packed_feature = quantized_schema.feature.add() - packed_feature.name = _NODE_PACKED_FEATURE_KEY + packed_feature.name = packed_feature_key packed_feature.type = schema_pb2.BYTES packed_feature.value_count.min = 1 packed_feature.value_count.max = 1 logger.info( f"Updated transformed schema for feature quantization: dropped " f"{len(quantization_spec.feature_keys)} features and added bytes feature " - f"{_NODE_PACKED_FEATURE_KEY}." + f"{packed_feature_key}." ) return quantized_schema diff --git a/gigl/src/data_preprocessor/lib/transform/utils.py b/gigl/src/data_preprocessor/lib/transform/utils.py index 07bfeaf7c..39a3ccc7b 100644 --- a/gigl/src/data_preprocessor/lib/transform/utils.py +++ b/gigl/src/data_preprocessor/lib/transform/utils.py @@ -28,6 +28,8 @@ NodeDataReference, ) from gigl.src.data_preprocessor.lib.transform.feature_quantization import ( + EDGE_PACKED_FEATURE_KEY, + NODE_PACKED_FEATURE_KEY, apply_feature_quantization_transform, ) from gigl.src.data_preprocessor.lib.transform.tf_value_encoder import TFValueEncoder @@ -372,9 +374,15 @@ def get_load_data_and_transform_pipeline_component( else analyzed_transform_fn[1].deferred_metadata # type: ignore ) quantization_spec: FeatureQuantizationSpec | None = None - if isinstance(preprocessing_spec, NodeDataPreprocessingSpec): + if isinstance( + preprocessing_spec, (NodeDataPreprocessingSpec, EdgeDataPreprocessingSpec) + ): quantization_spec = preprocessing_spec.feature_quantization_spec if quantization_spec is not None: + if isinstance(preprocessing_spec, EdgeDataPreprocessingSpec): + packed_feature_key = EDGE_PACKED_FEATURE_KEY + else: + packed_feature_key = NODE_PACKED_FEATURE_KEY transformed_features, resolved_transformed_metadata = ( apply_feature_quantization_transform( logical_features=transformed_features, @@ -384,6 +392,7 @@ def get_load_data_and_transform_pipeline_component( ), quantization_spec=quantization_spec, quantization_metadata_path=transformed_features_info.feature_quantization_metadata_path.uri, + packed_feature_key=packed_feature_key, ) ) diff --git a/gigl/src/data_preprocessor/lib/types.py b/gigl/src/data_preprocessor/lib/types.py index 014f7cbc0..8af220e4d 100644 --- a/gigl/src/data_preprocessor/lib/types.py +++ b/gigl/src/data_preprocessor/lib/types.py @@ -120,6 +120,7 @@ class EdgeDataPreprocessingSpec(NamedTuple): pretrained_tft_model_uri: Optional[Uri] = None features_outputs: Optional[list[str]] = None labels_outputs: Optional[list[str]] = None + feature_quantization_spec: Optional[FeatureQuantizationSpec] = None def __repr__(self) -> str: return f"""EdgeDataPreprocessingSpec( diff --git a/gigl/types/graph.py b/gigl/types/graph.py index 849f7708a..eb501f0d7 100644 --- a/gigl/types/graph.py +++ b/gigl/types/graph.py @@ -105,6 +105,9 @@ class PartitionOutput: partitioned_node_quantized_features: Optional[ Union[FeaturePartitionData, dict[NodeType, FeaturePartitionData]] ] = None + partitioned_edge_quantized_features: Optional[ + Union[FeaturePartitionData, dict[EdgeType, FeaturePartitionData]] + ] = None @dataclass(frozen=True) @@ -236,6 +239,9 @@ class LoadedGraphTensors: node_quantized_features: Optional[ Union[torch.Tensor, dict[NodeType, torch.Tensor]] ] = None + edge_quantized_features: Optional[ + Union[torch.Tensor, dict[EdgeType, torch.Tensor]] + ] = None def treat_labels_as_edges(self, edge_dir: Literal["in", "out"]) -> None: """ @@ -337,6 +343,9 @@ def treat_labels_as_edges(self, edge_dir: Literal["in", "out"]) -> None: self.node_quantized_features = to_heterogeneous_node( self.node_quantized_features ) + self.edge_quantized_features = to_heterogeneous_edge( + self.edge_quantized_features + ) self.edge_index = edge_index_with_labels self.edge_features = to_heterogeneous_edge(self.edge_features) self.edge_weights = to_heterogeneous_edge(self.edge_weights) diff --git a/proto/snapchat/research/gbml/preprocessed_metadata.proto b/proto/snapchat/research/gbml/preprocessed_metadata.proto index d7dfe3469..661c91d85 100644 --- a/proto/snapchat/research/gbml/preprocessed_metadata.proto +++ b/proto/snapchat/research/gbml/preprocessed_metadata.proto @@ -71,6 +71,8 @@ message PreprocessedMetadata{ optional uint32 feature_dim = 6; // Contains categorical feature vocabularies string transform_fn_assets_uri = 7; + // Optional quantized main-edge feature metadata. + FeatureQuantizationMetadata quantized_feature_metadata = 8; } // Houses metadata about edge TFTransform output from DataPreprocessor. diff --git a/scala/common/src/main/scala/snapchat/research/gbml/preprocessed_metadata/PreprocessedMetadata.scala b/scala/common/src/main/scala/snapchat/research/gbml/preprocessed_metadata/PreprocessedMetadata.scala index 7a9012ffa..9ddc933a7 100644 --- a/scala/common/src/main/scala/snapchat/research/gbml/preprocessed_metadata/PreprocessedMetadata.scala +++ b/scala/common/src/main/scala/snapchat/research/gbml/preprocessed_metadata/PreprocessedMetadata.scala @@ -1125,6 +1125,8 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r * Feature dimension after preprocessing * @param transformFnAssetsUri * Contains categorical feature vocabularies + * @param quantizedFeatureMetadata + * Optional quantized main-edge feature metadata. */ @SerialVersionUID(0L) final case class EdgeMetadataInfo( @@ -1135,6 +1137,7 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r enumeratedEdgeDataBqTable: _root_.scala.Predef.String = "", featureDim: _root_.scala.Option[_root_.scala.Int] = _root_.scala.None, transformFnAssetsUri: _root_.scala.Predef.String = "", + quantizedFeatureMetadata: _root_.scala.Option[snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata] = _root_.scala.None, unknownFields: _root_.scalapb.UnknownFieldSet = _root_.scalapb.UnknownFieldSet.empty ) extends scalapb.GeneratedMessage with scalapb.lenses.Updatable[EdgeMetadataInfo] { @transient @@ -1181,6 +1184,10 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r __size += _root_.com.google.protobuf.CodedOutputStream.computeStringSize(7, __value) } }; + if (quantizedFeatureMetadata.isDefined) { + val __value = quantizedFeatureMetadata.get + __size += 1 + _root_.com.google.protobuf.CodedOutputStream.computeUInt32SizeNoTag(__value.serializedSize) + __value.serializedSize + }; __size += unknownFields.serializedSize __size } @@ -1230,6 +1237,12 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r _output__.writeString(7, __v) } }; + quantizedFeatureMetadata.foreach { __v => + val __m = __v + _output__.writeTag(8, 2) + _output__.writeUInt32NoTag(__m.serializedSize) + __m.writeTo(_output__) + }; unknownFields.writeTo(_output__) } def clearFeatureKeys = copy(featureKeys = _root_.scala.Seq.empty) @@ -1247,6 +1260,9 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r def clearFeatureDim: EdgeMetadataInfo = copy(featureDim = _root_.scala.None) def withFeatureDim(__v: _root_.scala.Int): EdgeMetadataInfo = copy(featureDim = Option(__v)) def withTransformFnAssetsUri(__v: _root_.scala.Predef.String): EdgeMetadataInfo = copy(transformFnAssetsUri = __v) + def getQuantizedFeatureMetadata: snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata = quantizedFeatureMetadata.getOrElse(snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata.defaultInstance) + def clearQuantizedFeatureMetadata: EdgeMetadataInfo = copy(quantizedFeatureMetadata = _root_.scala.None) + def withQuantizedFeatureMetadata(__v: snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata): EdgeMetadataInfo = copy(quantizedFeatureMetadata = Option(__v)) def withUnknownFields(__v: _root_.scalapb.UnknownFieldSet) = copy(unknownFields = __v) def discardUnknownFields = copy(unknownFields = _root_.scalapb.UnknownFieldSet.empty) def getFieldByNumber(__fieldNumber: _root_.scala.Int): _root_.scala.Any = { @@ -1270,6 +1286,7 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r val __t = transformFnAssetsUri if (__t != "") __t else null } + case 8 => quantizedFeatureMetadata.orNull } } def getField(__field: _root_.scalapb.descriptors.FieldDescriptor): _root_.scalapb.descriptors.PValue = { @@ -1282,6 +1299,7 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r case 5 => _root_.scalapb.descriptors.PString(enumeratedEdgeDataBqTable) case 6 => featureDim.map(_root_.scalapb.descriptors.PInt(_)).getOrElse(_root_.scalapb.descriptors.PEmpty) case 7 => _root_.scalapb.descriptors.PString(transformFnAssetsUri) + case 8 => quantizedFeatureMetadata.map(_.toPMessage).getOrElse(_root_.scalapb.descriptors.PEmpty) } } def toProtoString: _root_.scala.Predef.String = _root_.scalapb.TextFormat.printToUnicodeString(this) @@ -1299,6 +1317,7 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r var __enumeratedEdgeDataBqTable: _root_.scala.Predef.String = "" var __featureDim: _root_.scala.Option[_root_.scala.Int] = _root_.scala.None var __transformFnAssetsUri: _root_.scala.Predef.String = "" + var __quantizedFeatureMetadata: _root_.scala.Option[snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata] = _root_.scala.None var `_unknownFields__`: _root_.scalapb.UnknownFieldSet.Builder = null var _done__ = false while (!_done__) { @@ -1319,6 +1338,8 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r __featureDim = Option(_input__.readUInt32()) case 58 => __transformFnAssetsUri = _input__.readStringRequireUtf8() + case 66 => + __quantizedFeatureMetadata = Option(__quantizedFeatureMetadata.fold(_root_.scalapb.LiteParser.readMessage[snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata](_input__))(_root_.scalapb.LiteParser.readMessage(_input__, _))) case tag => if (_unknownFields__ == null) { _unknownFields__ = new _root_.scalapb.UnknownFieldSet.Builder() @@ -1334,6 +1355,7 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r enumeratedEdgeDataBqTable = __enumeratedEdgeDataBqTable, featureDim = __featureDim, transformFnAssetsUri = __transformFnAssetsUri, + quantizedFeatureMetadata = __quantizedFeatureMetadata, unknownFields = if (_unknownFields__ == null) _root_.scalapb.UnknownFieldSet.empty else _unknownFields__.result() ) } @@ -1347,13 +1369,20 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r schemaUri = __fieldsMap.get(scalaDescriptor.findFieldByNumber(4).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), enumeratedEdgeDataBqTable = __fieldsMap.get(scalaDescriptor.findFieldByNumber(5).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), featureDim = __fieldsMap.get(scalaDescriptor.findFieldByNumber(6).get).flatMap(_.as[_root_.scala.Option[_root_.scala.Int]]), - transformFnAssetsUri = __fieldsMap.get(scalaDescriptor.findFieldByNumber(7).get).map(_.as[_root_.scala.Predef.String]).getOrElse("") + transformFnAssetsUri = __fieldsMap.get(scalaDescriptor.findFieldByNumber(7).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), + quantizedFeatureMetadata = __fieldsMap.get(scalaDescriptor.findFieldByNumber(8).get).flatMap(_.as[_root_.scala.Option[snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata]]) ) case _ => throw new RuntimeException("Expected PMessage") } def javaDescriptor: _root_.com.google.protobuf.Descriptors.Descriptor = snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.javaDescriptor.getNestedTypes().get(4) def scalaDescriptor: _root_.scalapb.descriptors.Descriptor = snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.scalaDescriptor.nestedMessages(4) - def messageCompanionForFieldNumber(__number: _root_.scala.Int): _root_.scalapb.GeneratedMessageCompanion[_] = throw new MatchError(__number) + def messageCompanionForFieldNumber(__number: _root_.scala.Int): _root_.scalapb.GeneratedMessageCompanion[_] = { + var __out: _root_.scalapb.GeneratedMessageCompanion[_] = null + (__number: @_root_.scala.unchecked) match { + case 8 => __out = snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata + } + __out + } lazy val nestedMessagesCompanions: Seq[_root_.scalapb.GeneratedMessageCompanion[_ <: _root_.scalapb.GeneratedMessage]] = Seq.empty def enumCompanionForFieldNumber(__fieldNumber: _root_.scala.Int): _root_.scalapb.GeneratedEnumCompanion[_] = throw new MatchError(__fieldNumber) lazy val defaultInstance = snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.EdgeMetadataInfo( @@ -1363,7 +1392,8 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r schemaUri = "", enumeratedEdgeDataBqTable = "", featureDim = _root_.scala.None, - transformFnAssetsUri = "" + transformFnAssetsUri = "", + quantizedFeatureMetadata = _root_.scala.None ) implicit class EdgeMetadataInfoLens[UpperPB](_l: _root_.scalapb.lenses.Lens[UpperPB, snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.EdgeMetadataInfo]) extends _root_.scalapb.lenses.ObjectLens[UpperPB, snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.EdgeMetadataInfo](_l) { def featureKeys: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Seq[_root_.scala.Predef.String]] = field(_.featureKeys)((c_, f_) => c_.copy(featureKeys = f_)) @@ -1374,6 +1404,8 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r def featureDim: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Int] = field(_.getFeatureDim)((c_, f_) => c_.copy(featureDim = Option(f_))) def optionalFeatureDim: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Option[_root_.scala.Int]] = field(_.featureDim)((c_, f_) => c_.copy(featureDim = f_)) def transformFnAssetsUri: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.transformFnAssetsUri)((c_, f_) => c_.copy(transformFnAssetsUri = f_)) + def quantizedFeatureMetadata: _root_.scalapb.lenses.Lens[UpperPB, snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata] = field(_.getQuantizedFeatureMetadata)((c_, f_) => c_.copy(quantizedFeatureMetadata = Option(f_))) + def optionalQuantizedFeatureMetadata: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Option[snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata]] = field(_.quantizedFeatureMetadata)((c_, f_) => c_.copy(quantizedFeatureMetadata = f_)) } final val FEATURE_KEYS_FIELD_NUMBER = 1 final val LABEL_KEYS_FIELD_NUMBER = 2 @@ -1382,6 +1414,7 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r final val ENUMERATED_EDGE_DATA_BQ_TABLE_FIELD_NUMBER = 5 final val FEATURE_DIM_FIELD_NUMBER = 6 final val TRANSFORM_FN_ASSETS_URI_FIELD_NUMBER = 7 + final val QUANTIZED_FEATURE_METADATA_FIELD_NUMBER = 8 def of( featureKeys: _root_.scala.Seq[_root_.scala.Predef.String], labelKeys: _root_.scala.Seq[_root_.scala.Predef.String], @@ -1389,7 +1422,8 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r schemaUri: _root_.scala.Predef.String, enumeratedEdgeDataBqTable: _root_.scala.Predef.String, featureDim: _root_.scala.Option[_root_.scala.Int], - transformFnAssetsUri: _root_.scala.Predef.String + transformFnAssetsUri: _root_.scala.Predef.String, + quantizedFeatureMetadata: _root_.scala.Option[snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata] ): _root_.snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.EdgeMetadataInfo = _root_.snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.EdgeMetadataInfo( featureKeys, labelKeys, @@ -1397,7 +1431,8 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r schemaUri, enumeratedEdgeDataBqTable, featureDim, - transformFnAssetsUri + transformFnAssetsUri, + quantizedFeatureMetadata ) // @@protoc_insertion_point(GeneratedMessageCompanion[snapchat.research.gbml.PreprocessedMetadata.EdgeMetadataInfo]) } diff --git a/scala/common/src/main/scala/snapchat/research/gbml/preprocessed_metadata/PreprocessedMetadataProto.scala b/scala/common/src/main/scala/snapchat/research/gbml/preprocessed_metadata/PreprocessedMetadataProto.scala index ad80de0ad..998cadd75 100644 --- a/scala/common/src/main/scala/snapchat/research/gbml/preprocessed_metadata/PreprocessedMetadataProto.scala +++ b/scala/common/src/main/scala/snapchat/research/gbml/preprocessed_metadata/PreprocessedMetadataProto.scala @@ -14,7 +14,7 @@ object PreprocessedMetadataProto extends _root_.scalapb.GeneratedFileObject { private lazy val ProtoBytes: _root_.scala.Array[Byte] = scalapb.Encoding.fromBase64(scala.collection.immutable.Seq( """CjJzbmFwY2hhdC9yZXNlYXJjaC9nYm1sL3ByZXByb2Nlc3NlZF9tZXRhZGF0YS5wcm90bxIWc25hcGNoYXQucmVzZWFyY2guZ - 2JtbCL9GgoUUHJlcHJvY2Vzc2VkTWV0YWRhdGES5gEKLGNvbmRlbnNlZF9ub2RlX3R5cGVfdG9fcHJlcHJvY2Vzc2VkX21ldGFkY + 2JtbCKlHAoUUHJlcHJvY2Vzc2VkTWV0YWRhdGES5gEKLGNvbmRlbnNlZF9ub2RlX3R5cGVfdG9fcHJlcHJvY2Vzc2VkX21ldGFkY XRhGAEgAygLMlkuc25hcGNoYXQucmVzZWFyY2guZ2JtbC5QcmVwcm9jZXNzZWRNZXRhZGF0YS5Db25kZW5zZWROb2RlVHlwZVRvU HJlcHJvY2Vzc2VkTWV0YWRhdGFFbnRyeUIs4j8pEidjb25kZW5zZWROb2RlVHlwZVRvUHJlcHJvY2Vzc2VkTWV0YWRhdGFSJ2Nvb mRlbnNlZE5vZGVUeXBlVG9QcmVwcm9jZXNzZWRNZXRhZGF0YRLmAQosY29uZGVuc2VkX2VkZ2VfdHlwZV90b19wcmVwcm9jZXNzZ @@ -41,26 +41,28 @@ object PreprocessedMetadataProto extends _root_.scalapb.GeneratedFileObject { hR0cmFuc2Zvcm1GbkFzc2V0c1VyaVIUdHJhbnNmb3JtRm5Bc3NldHNVcmkSpQEKGnF1YW50aXplZF9mZWF0dXJlX21ldGFkYXRhG AogASgLMkguc25hcGNoYXQucmVzZWFyY2guZ2JtbC5QcmVwcm9jZXNzZWRNZXRhZGF0YS5GZWF0dXJlUXVhbnRpemF0aW9uTWV0Y WRhdGFCHeI/GhIYcXVhbnRpemVkRmVhdHVyZU1ldGFkYXRhUhhxdWFudGl6ZWRGZWF0dXJlTWV0YWRhdGFCDgoMX2ZlYXR1cmVfZ - GltGugDChBFZGdlTWV0YWRhdGFJbmZvEjMKDGZlYXR1cmVfa2V5cxgBIAMoCUIQ4j8NEgtmZWF0dXJlS2V5c1ILZmVhdHVyZUtle + GltGpAFChBFZGdlTWV0YWRhdGFJbmZvEjMKDGZlYXR1cmVfa2V5cxgBIAMoCUIQ4j8NEgtmZWF0dXJlS2V5c1ILZmVhdHVyZUtle XMSLQoKbGFiZWxfa2V5cxgCIAMoCUIO4j8LEglsYWJlbEtleXNSCWxhYmVsS2V5cxJGChN0ZnJlY29yZF91cmlfcHJlZml4GAMgA SgJQhbiPxMSEXRmcmVjb3JkVXJpUHJlZml4UhF0ZnJlY29yZFVyaVByZWZpeBItCgpzY2hlbWFfdXJpGAQgASgJQg7iPwsSCXNja GVtYVVyaVIJc2NoZW1hVXJpEmAKHWVudW1lcmF0ZWRfZWRnZV9kYXRhX2JxX3RhYmxlGAUgASgJQh7iPxsSGWVudW1lcmF0ZWRFZ GdlRGF0YUJxVGFibGVSGWVudW1lcmF0ZWRFZGdlRGF0YUJxVGFibGUSNQoLZmVhdHVyZV9kaW0YBiABKA1CD+I/DBIKZmVhdHVyZ URpbUgAUgpmZWF0dXJlRGltiAEBElAKF3RyYW5zZm9ybV9mbl9hc3NldHNfdXJpGAcgASgJQhniPxYSFHRyYW5zZm9ybUZuQXNzZ - XRzVXJpUhR0cmFuc2Zvcm1GbkFzc2V0c1VyaUIOCgxfZmVhdHVyZV9kaW0awgQKEkVkZ2VNZXRhZGF0YU91dHB1dBI4Cg9zcmNfb - m9kZV9pZF9rZXkYASABKAlCEeI/DhIMc3JjTm9kZUlkS2V5UgxzcmNOb2RlSWRLZXkSOAoPZHN0X25vZGVfaWRfa2V5GAIgASgJQ - hHiPw4SDGRzdE5vZGVJZEtleVIMZHN0Tm9kZUlkS2V5EnYKDm1haW5fZWRnZV9pbmZvGAMgASgLMj0uc25hcGNoYXQucmVzZWFyY - 2guZ2JtbC5QcmVwcm9jZXNzZWRNZXRhZGF0YS5FZGdlTWV0YWRhdGFJbmZvQhHiPw4SDG1haW5FZGdlSW5mb1IMbWFpbkVkZ2VJb - mZvEocBChJwb3NpdGl2ZV9lZGdlX2luZm8YBCABKAsyPS5zbmFwY2hhdC5yZXNlYXJjaC5nYm1sLlByZXByb2Nlc3NlZE1ldGFkY - XRhLkVkZ2VNZXRhZGF0YUluZm9CFeI/EhIQcG9zaXRpdmVFZGdlSW5mb0gAUhBwb3NpdGl2ZUVkZ2VJbmZviAEBEocBChJuZWdhd - Gl2ZV9lZGdlX2luZm8YBSABKAsyPS5zbmFwY2hhdC5yZXNlYXJjaC5nYm1sLlByZXByb2Nlc3NlZE1ldGFkYXRhLkVkZ2VNZXRhZ - GF0YUluZm9CFeI/EhIQbmVnYXRpdmVFZGdlSW5mb0gBUhBuZWdhdGl2ZUVkZ2VJbmZviAEBQhUKE19wb3NpdGl2ZV9lZGdlX2luZ - m9CFQoTX25lZ2F0aXZlX2VkZ2VfaW5mbxqxAQosQ29uZGVuc2VkTm9kZVR5cGVUb1ByZXByb2Nlc3NlZE1ldGFkYXRhRW50cnkSG - goDa2V5GAEgASgNQgjiPwUSA2tleVIDa2V5EmEKBXZhbHVlGAIgASgLMj8uc25hcGNoYXQucmVzZWFyY2guZ2JtbC5QcmVwcm9jZ - XNzZWRNZXRhZGF0YS5Ob2RlTWV0YWRhdGFPdXRwdXRCCuI/BxIFdmFsdWVSBXZhbHVlOgI4ARqxAQosQ29uZGVuc2VkRWRnZVR5c - GVUb1ByZXByb2Nlc3NlZE1ldGFkYXRhRW50cnkSGgoDa2V5GAEgASgNQgjiPwUSA2tleVIDa2V5EmEKBXZhbHVlGAIgASgLMj8uc - 25hcGNoYXQucmVzZWFyY2guZ2JtbC5QcmVwcm9jZXNzZWRNZXRhZGF0YS5FZGdlTWV0YWRhdGFPdXRwdXRCCuI/BxIFdmFsdWVSB - XZhbHVlOgI4AWIGcHJvdG8z""" + XRzVXJpUhR0cmFuc2Zvcm1GbkFzc2V0c1VyaRKlAQoacXVhbnRpemVkX2ZlYXR1cmVfbWV0YWRhdGEYCCABKAsySC5zbmFwY2hhd + C5yZXNlYXJjaC5nYm1sLlByZXByb2Nlc3NlZE1ldGFkYXRhLkZlYXR1cmVRdWFudGl6YXRpb25NZXRhZGF0YUId4j8aEhhxdWFud + Gl6ZWRGZWF0dXJlTWV0YWRhdGFSGHF1YW50aXplZEZlYXR1cmVNZXRhZGF0YUIOCgxfZmVhdHVyZV9kaW0awgQKEkVkZ2VNZXRhZ + GF0YU91dHB1dBI4Cg9zcmNfbm9kZV9pZF9rZXkYASABKAlCEeI/DhIMc3JjTm9kZUlkS2V5UgxzcmNOb2RlSWRLZXkSOAoPZHN0X + 25vZGVfaWRfa2V5GAIgASgJQhHiPw4SDGRzdE5vZGVJZEtleVIMZHN0Tm9kZUlkS2V5EnYKDm1haW5fZWRnZV9pbmZvGAMgASgLM + j0uc25hcGNoYXQucmVzZWFyY2guZ2JtbC5QcmVwcm9jZXNzZWRNZXRhZGF0YS5FZGdlTWV0YWRhdGFJbmZvQhHiPw4SDG1haW5FZ + GdlSW5mb1IMbWFpbkVkZ2VJbmZvEocBChJwb3NpdGl2ZV9lZGdlX2luZm8YBCABKAsyPS5zbmFwY2hhdC5yZXNlYXJjaC5nYm1sL + lByZXByb2Nlc3NlZE1ldGFkYXRhLkVkZ2VNZXRhZGF0YUluZm9CFeI/EhIQcG9zaXRpdmVFZGdlSW5mb0gAUhBwb3NpdGl2ZUVkZ + 2VJbmZviAEBEocBChJuZWdhdGl2ZV9lZGdlX2luZm8YBSABKAsyPS5zbmFwY2hhdC5yZXNlYXJjaC5nYm1sLlByZXByb2Nlc3NlZ + E1ldGFkYXRhLkVkZ2VNZXRhZGF0YUluZm9CFeI/EhIQbmVnYXRpdmVFZGdlSW5mb0gBUhBuZWdhdGl2ZUVkZ2VJbmZviAEBQhUKE + 19wb3NpdGl2ZV9lZGdlX2luZm9CFQoTX25lZ2F0aXZlX2VkZ2VfaW5mbxqxAQosQ29uZGVuc2VkTm9kZVR5cGVUb1ByZXByb2Nlc + 3NlZE1ldGFkYXRhRW50cnkSGgoDa2V5GAEgASgNQgjiPwUSA2tleVIDa2V5EmEKBXZhbHVlGAIgASgLMj8uc25hcGNoYXQucmVzZ + WFyY2guZ2JtbC5QcmVwcm9jZXNzZWRNZXRhZGF0YS5Ob2RlTWV0YWRhdGFPdXRwdXRCCuI/BxIFdmFsdWVSBXZhbHVlOgI4ARqxA + QosQ29uZGVuc2VkRWRnZVR5cGVUb1ByZXByb2Nlc3NlZE1ldGFkYXRhRW50cnkSGgoDa2V5GAEgASgNQgjiPwUSA2tleVIDa2V5E + mEKBXZhbHVlGAIgASgLMj8uc25hcGNoYXQucmVzZWFyY2guZ2JtbC5QcmVwcm9jZXNzZWRNZXRhZGF0YS5FZGdlTWV0YWRhdGFPd + XRwdXRCCuI/BxIFdmFsdWVSBXZhbHVlOgI4AWIGcHJvdG8z""" ).mkString) lazy val scalaDescriptor: _root_.scalapb.descriptors.FileDescriptor = { val scalaProto = com.google.protobuf.descriptor.FileDescriptorProto.parseFrom(ProtoBytes) diff --git a/scala_spark35/common/src/main/scala/snapchat/research/gbml/preprocessed_metadata/PreprocessedMetadata.scala b/scala_spark35/common/src/main/scala/snapchat/research/gbml/preprocessed_metadata/PreprocessedMetadata.scala index 7a9012ffa..9ddc933a7 100644 --- a/scala_spark35/common/src/main/scala/snapchat/research/gbml/preprocessed_metadata/PreprocessedMetadata.scala +++ b/scala_spark35/common/src/main/scala/snapchat/research/gbml/preprocessed_metadata/PreprocessedMetadata.scala @@ -1125,6 +1125,8 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r * Feature dimension after preprocessing * @param transformFnAssetsUri * Contains categorical feature vocabularies + * @param quantizedFeatureMetadata + * Optional quantized main-edge feature metadata. */ @SerialVersionUID(0L) final case class EdgeMetadataInfo( @@ -1135,6 +1137,7 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r enumeratedEdgeDataBqTable: _root_.scala.Predef.String = "", featureDim: _root_.scala.Option[_root_.scala.Int] = _root_.scala.None, transformFnAssetsUri: _root_.scala.Predef.String = "", + quantizedFeatureMetadata: _root_.scala.Option[snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata] = _root_.scala.None, unknownFields: _root_.scalapb.UnknownFieldSet = _root_.scalapb.UnknownFieldSet.empty ) extends scalapb.GeneratedMessage with scalapb.lenses.Updatable[EdgeMetadataInfo] { @transient @@ -1181,6 +1184,10 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r __size += _root_.com.google.protobuf.CodedOutputStream.computeStringSize(7, __value) } }; + if (quantizedFeatureMetadata.isDefined) { + val __value = quantizedFeatureMetadata.get + __size += 1 + _root_.com.google.protobuf.CodedOutputStream.computeUInt32SizeNoTag(__value.serializedSize) + __value.serializedSize + }; __size += unknownFields.serializedSize __size } @@ -1230,6 +1237,12 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r _output__.writeString(7, __v) } }; + quantizedFeatureMetadata.foreach { __v => + val __m = __v + _output__.writeTag(8, 2) + _output__.writeUInt32NoTag(__m.serializedSize) + __m.writeTo(_output__) + }; unknownFields.writeTo(_output__) } def clearFeatureKeys = copy(featureKeys = _root_.scala.Seq.empty) @@ -1247,6 +1260,9 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r def clearFeatureDim: EdgeMetadataInfo = copy(featureDim = _root_.scala.None) def withFeatureDim(__v: _root_.scala.Int): EdgeMetadataInfo = copy(featureDim = Option(__v)) def withTransformFnAssetsUri(__v: _root_.scala.Predef.String): EdgeMetadataInfo = copy(transformFnAssetsUri = __v) + def getQuantizedFeatureMetadata: snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata = quantizedFeatureMetadata.getOrElse(snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata.defaultInstance) + def clearQuantizedFeatureMetadata: EdgeMetadataInfo = copy(quantizedFeatureMetadata = _root_.scala.None) + def withQuantizedFeatureMetadata(__v: snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata): EdgeMetadataInfo = copy(quantizedFeatureMetadata = Option(__v)) def withUnknownFields(__v: _root_.scalapb.UnknownFieldSet) = copy(unknownFields = __v) def discardUnknownFields = copy(unknownFields = _root_.scalapb.UnknownFieldSet.empty) def getFieldByNumber(__fieldNumber: _root_.scala.Int): _root_.scala.Any = { @@ -1270,6 +1286,7 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r val __t = transformFnAssetsUri if (__t != "") __t else null } + case 8 => quantizedFeatureMetadata.orNull } } def getField(__field: _root_.scalapb.descriptors.FieldDescriptor): _root_.scalapb.descriptors.PValue = { @@ -1282,6 +1299,7 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r case 5 => _root_.scalapb.descriptors.PString(enumeratedEdgeDataBqTable) case 6 => featureDim.map(_root_.scalapb.descriptors.PInt(_)).getOrElse(_root_.scalapb.descriptors.PEmpty) case 7 => _root_.scalapb.descriptors.PString(transformFnAssetsUri) + case 8 => quantizedFeatureMetadata.map(_.toPMessage).getOrElse(_root_.scalapb.descriptors.PEmpty) } } def toProtoString: _root_.scala.Predef.String = _root_.scalapb.TextFormat.printToUnicodeString(this) @@ -1299,6 +1317,7 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r var __enumeratedEdgeDataBqTable: _root_.scala.Predef.String = "" var __featureDim: _root_.scala.Option[_root_.scala.Int] = _root_.scala.None var __transformFnAssetsUri: _root_.scala.Predef.String = "" + var __quantizedFeatureMetadata: _root_.scala.Option[snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata] = _root_.scala.None var `_unknownFields__`: _root_.scalapb.UnknownFieldSet.Builder = null var _done__ = false while (!_done__) { @@ -1319,6 +1338,8 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r __featureDim = Option(_input__.readUInt32()) case 58 => __transformFnAssetsUri = _input__.readStringRequireUtf8() + case 66 => + __quantizedFeatureMetadata = Option(__quantizedFeatureMetadata.fold(_root_.scalapb.LiteParser.readMessage[snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata](_input__))(_root_.scalapb.LiteParser.readMessage(_input__, _))) case tag => if (_unknownFields__ == null) { _unknownFields__ = new _root_.scalapb.UnknownFieldSet.Builder() @@ -1334,6 +1355,7 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r enumeratedEdgeDataBqTable = __enumeratedEdgeDataBqTable, featureDim = __featureDim, transformFnAssetsUri = __transformFnAssetsUri, + quantizedFeatureMetadata = __quantizedFeatureMetadata, unknownFields = if (_unknownFields__ == null) _root_.scalapb.UnknownFieldSet.empty else _unknownFields__.result() ) } @@ -1347,13 +1369,20 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r schemaUri = __fieldsMap.get(scalaDescriptor.findFieldByNumber(4).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), enumeratedEdgeDataBqTable = __fieldsMap.get(scalaDescriptor.findFieldByNumber(5).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), featureDim = __fieldsMap.get(scalaDescriptor.findFieldByNumber(6).get).flatMap(_.as[_root_.scala.Option[_root_.scala.Int]]), - transformFnAssetsUri = __fieldsMap.get(scalaDescriptor.findFieldByNumber(7).get).map(_.as[_root_.scala.Predef.String]).getOrElse("") + transformFnAssetsUri = __fieldsMap.get(scalaDescriptor.findFieldByNumber(7).get).map(_.as[_root_.scala.Predef.String]).getOrElse(""), + quantizedFeatureMetadata = __fieldsMap.get(scalaDescriptor.findFieldByNumber(8).get).flatMap(_.as[_root_.scala.Option[snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata]]) ) case _ => throw new RuntimeException("Expected PMessage") } def javaDescriptor: _root_.com.google.protobuf.Descriptors.Descriptor = snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.javaDescriptor.getNestedTypes().get(4) def scalaDescriptor: _root_.scalapb.descriptors.Descriptor = snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.scalaDescriptor.nestedMessages(4) - def messageCompanionForFieldNumber(__number: _root_.scala.Int): _root_.scalapb.GeneratedMessageCompanion[_] = throw new MatchError(__number) + def messageCompanionForFieldNumber(__number: _root_.scala.Int): _root_.scalapb.GeneratedMessageCompanion[_] = { + var __out: _root_.scalapb.GeneratedMessageCompanion[_] = null + (__number: @_root_.scala.unchecked) match { + case 8 => __out = snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata + } + __out + } lazy val nestedMessagesCompanions: Seq[_root_.scalapb.GeneratedMessageCompanion[_ <: _root_.scalapb.GeneratedMessage]] = Seq.empty def enumCompanionForFieldNumber(__fieldNumber: _root_.scala.Int): _root_.scalapb.GeneratedEnumCompanion[_] = throw new MatchError(__fieldNumber) lazy val defaultInstance = snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.EdgeMetadataInfo( @@ -1363,7 +1392,8 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r schemaUri = "", enumeratedEdgeDataBqTable = "", featureDim = _root_.scala.None, - transformFnAssetsUri = "" + transformFnAssetsUri = "", + quantizedFeatureMetadata = _root_.scala.None ) implicit class EdgeMetadataInfoLens[UpperPB](_l: _root_.scalapb.lenses.Lens[UpperPB, snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.EdgeMetadataInfo]) extends _root_.scalapb.lenses.ObjectLens[UpperPB, snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.EdgeMetadataInfo](_l) { def featureKeys: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Seq[_root_.scala.Predef.String]] = field(_.featureKeys)((c_, f_) => c_.copy(featureKeys = f_)) @@ -1374,6 +1404,8 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r def featureDim: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Int] = field(_.getFeatureDim)((c_, f_) => c_.copy(featureDim = Option(f_))) def optionalFeatureDim: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Option[_root_.scala.Int]] = field(_.featureDim)((c_, f_) => c_.copy(featureDim = f_)) def transformFnAssetsUri: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Predef.String] = field(_.transformFnAssetsUri)((c_, f_) => c_.copy(transformFnAssetsUri = f_)) + def quantizedFeatureMetadata: _root_.scalapb.lenses.Lens[UpperPB, snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata] = field(_.getQuantizedFeatureMetadata)((c_, f_) => c_.copy(quantizedFeatureMetadata = Option(f_))) + def optionalQuantizedFeatureMetadata: _root_.scalapb.lenses.Lens[UpperPB, _root_.scala.Option[snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata]] = field(_.quantizedFeatureMetadata)((c_, f_) => c_.copy(quantizedFeatureMetadata = f_)) } final val FEATURE_KEYS_FIELD_NUMBER = 1 final val LABEL_KEYS_FIELD_NUMBER = 2 @@ -1382,6 +1414,7 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r final val ENUMERATED_EDGE_DATA_BQ_TABLE_FIELD_NUMBER = 5 final val FEATURE_DIM_FIELD_NUMBER = 6 final val TRANSFORM_FN_ASSETS_URI_FIELD_NUMBER = 7 + final val QUANTIZED_FEATURE_METADATA_FIELD_NUMBER = 8 def of( featureKeys: _root_.scala.Seq[_root_.scala.Predef.String], labelKeys: _root_.scala.Seq[_root_.scala.Predef.String], @@ -1389,7 +1422,8 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r schemaUri: _root_.scala.Predef.String, enumeratedEdgeDataBqTable: _root_.scala.Predef.String, featureDim: _root_.scala.Option[_root_.scala.Int], - transformFnAssetsUri: _root_.scala.Predef.String + transformFnAssetsUri: _root_.scala.Predef.String, + quantizedFeatureMetadata: _root_.scala.Option[snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.FeatureQuantizationMetadata] ): _root_.snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.EdgeMetadataInfo = _root_.snapchat.research.gbml.preprocessed_metadata.PreprocessedMetadata.EdgeMetadataInfo( featureKeys, labelKeys, @@ -1397,7 +1431,8 @@ object PreprocessedMetadata extends scalapb.GeneratedMessageCompanion[snapchat.r schemaUri, enumeratedEdgeDataBqTable, featureDim, - transformFnAssetsUri + transformFnAssetsUri, + quantizedFeatureMetadata ) // @@protoc_insertion_point(GeneratedMessageCompanion[snapchat.research.gbml.PreprocessedMetadata.EdgeMetadataInfo]) } diff --git a/scala_spark35/common/src/main/scala/snapchat/research/gbml/preprocessed_metadata/PreprocessedMetadataProto.scala b/scala_spark35/common/src/main/scala/snapchat/research/gbml/preprocessed_metadata/PreprocessedMetadataProto.scala index ad80de0ad..998cadd75 100644 --- a/scala_spark35/common/src/main/scala/snapchat/research/gbml/preprocessed_metadata/PreprocessedMetadataProto.scala +++ b/scala_spark35/common/src/main/scala/snapchat/research/gbml/preprocessed_metadata/PreprocessedMetadataProto.scala @@ -14,7 +14,7 @@ object PreprocessedMetadataProto extends _root_.scalapb.GeneratedFileObject { private lazy val ProtoBytes: _root_.scala.Array[Byte] = scalapb.Encoding.fromBase64(scala.collection.immutable.Seq( """CjJzbmFwY2hhdC9yZXNlYXJjaC9nYm1sL3ByZXByb2Nlc3NlZF9tZXRhZGF0YS5wcm90bxIWc25hcGNoYXQucmVzZWFyY2guZ - 2JtbCL9GgoUUHJlcHJvY2Vzc2VkTWV0YWRhdGES5gEKLGNvbmRlbnNlZF9ub2RlX3R5cGVfdG9fcHJlcHJvY2Vzc2VkX21ldGFkY + 2JtbCKlHAoUUHJlcHJvY2Vzc2VkTWV0YWRhdGES5gEKLGNvbmRlbnNlZF9ub2RlX3R5cGVfdG9fcHJlcHJvY2Vzc2VkX21ldGFkY XRhGAEgAygLMlkuc25hcGNoYXQucmVzZWFyY2guZ2JtbC5QcmVwcm9jZXNzZWRNZXRhZGF0YS5Db25kZW5zZWROb2RlVHlwZVRvU HJlcHJvY2Vzc2VkTWV0YWRhdGFFbnRyeUIs4j8pEidjb25kZW5zZWROb2RlVHlwZVRvUHJlcHJvY2Vzc2VkTWV0YWRhdGFSJ2Nvb mRlbnNlZE5vZGVUeXBlVG9QcmVwcm9jZXNzZWRNZXRhZGF0YRLmAQosY29uZGVuc2VkX2VkZ2VfdHlwZV90b19wcmVwcm9jZXNzZ @@ -41,26 +41,28 @@ object PreprocessedMetadataProto extends _root_.scalapb.GeneratedFileObject { hR0cmFuc2Zvcm1GbkFzc2V0c1VyaVIUdHJhbnNmb3JtRm5Bc3NldHNVcmkSpQEKGnF1YW50aXplZF9mZWF0dXJlX21ldGFkYXRhG AogASgLMkguc25hcGNoYXQucmVzZWFyY2guZ2JtbC5QcmVwcm9jZXNzZWRNZXRhZGF0YS5GZWF0dXJlUXVhbnRpemF0aW9uTWV0Y WRhdGFCHeI/GhIYcXVhbnRpemVkRmVhdHVyZU1ldGFkYXRhUhhxdWFudGl6ZWRGZWF0dXJlTWV0YWRhdGFCDgoMX2ZlYXR1cmVfZ - GltGugDChBFZGdlTWV0YWRhdGFJbmZvEjMKDGZlYXR1cmVfa2V5cxgBIAMoCUIQ4j8NEgtmZWF0dXJlS2V5c1ILZmVhdHVyZUtle + GltGpAFChBFZGdlTWV0YWRhdGFJbmZvEjMKDGZlYXR1cmVfa2V5cxgBIAMoCUIQ4j8NEgtmZWF0dXJlS2V5c1ILZmVhdHVyZUtle XMSLQoKbGFiZWxfa2V5cxgCIAMoCUIO4j8LEglsYWJlbEtleXNSCWxhYmVsS2V5cxJGChN0ZnJlY29yZF91cmlfcHJlZml4GAMgA SgJQhbiPxMSEXRmcmVjb3JkVXJpUHJlZml4UhF0ZnJlY29yZFVyaVByZWZpeBItCgpzY2hlbWFfdXJpGAQgASgJQg7iPwsSCXNja GVtYVVyaVIJc2NoZW1hVXJpEmAKHWVudW1lcmF0ZWRfZWRnZV9kYXRhX2JxX3RhYmxlGAUgASgJQh7iPxsSGWVudW1lcmF0ZWRFZ GdlRGF0YUJxVGFibGVSGWVudW1lcmF0ZWRFZGdlRGF0YUJxVGFibGUSNQoLZmVhdHVyZV9kaW0YBiABKA1CD+I/DBIKZmVhdHVyZ URpbUgAUgpmZWF0dXJlRGltiAEBElAKF3RyYW5zZm9ybV9mbl9hc3NldHNfdXJpGAcgASgJQhniPxYSFHRyYW5zZm9ybUZuQXNzZ - XRzVXJpUhR0cmFuc2Zvcm1GbkFzc2V0c1VyaUIOCgxfZmVhdHVyZV9kaW0awgQKEkVkZ2VNZXRhZGF0YU91dHB1dBI4Cg9zcmNfb - m9kZV9pZF9rZXkYASABKAlCEeI/DhIMc3JjTm9kZUlkS2V5UgxzcmNOb2RlSWRLZXkSOAoPZHN0X25vZGVfaWRfa2V5GAIgASgJQ - hHiPw4SDGRzdE5vZGVJZEtleVIMZHN0Tm9kZUlkS2V5EnYKDm1haW5fZWRnZV9pbmZvGAMgASgLMj0uc25hcGNoYXQucmVzZWFyY - 2guZ2JtbC5QcmVwcm9jZXNzZWRNZXRhZGF0YS5FZGdlTWV0YWRhdGFJbmZvQhHiPw4SDG1haW5FZGdlSW5mb1IMbWFpbkVkZ2VJb - mZvEocBChJwb3NpdGl2ZV9lZGdlX2luZm8YBCABKAsyPS5zbmFwY2hhdC5yZXNlYXJjaC5nYm1sLlByZXByb2Nlc3NlZE1ldGFkY - XRhLkVkZ2VNZXRhZGF0YUluZm9CFeI/EhIQcG9zaXRpdmVFZGdlSW5mb0gAUhBwb3NpdGl2ZUVkZ2VJbmZviAEBEocBChJuZWdhd - Gl2ZV9lZGdlX2luZm8YBSABKAsyPS5zbmFwY2hhdC5yZXNlYXJjaC5nYm1sLlByZXByb2Nlc3NlZE1ldGFkYXRhLkVkZ2VNZXRhZ - GF0YUluZm9CFeI/EhIQbmVnYXRpdmVFZGdlSW5mb0gBUhBuZWdhdGl2ZUVkZ2VJbmZviAEBQhUKE19wb3NpdGl2ZV9lZGdlX2luZ - m9CFQoTX25lZ2F0aXZlX2VkZ2VfaW5mbxqxAQosQ29uZGVuc2VkTm9kZVR5cGVUb1ByZXByb2Nlc3NlZE1ldGFkYXRhRW50cnkSG - goDa2V5GAEgASgNQgjiPwUSA2tleVIDa2V5EmEKBXZhbHVlGAIgASgLMj8uc25hcGNoYXQucmVzZWFyY2guZ2JtbC5QcmVwcm9jZ - XNzZWRNZXRhZGF0YS5Ob2RlTWV0YWRhdGFPdXRwdXRCCuI/BxIFdmFsdWVSBXZhbHVlOgI4ARqxAQosQ29uZGVuc2VkRWRnZVR5c - GVUb1ByZXByb2Nlc3NlZE1ldGFkYXRhRW50cnkSGgoDa2V5GAEgASgNQgjiPwUSA2tleVIDa2V5EmEKBXZhbHVlGAIgASgLMj8uc - 25hcGNoYXQucmVzZWFyY2guZ2JtbC5QcmVwcm9jZXNzZWRNZXRhZGF0YS5FZGdlTWV0YWRhdGFPdXRwdXRCCuI/BxIFdmFsdWVSB - XZhbHVlOgI4AWIGcHJvdG8z""" + XRzVXJpUhR0cmFuc2Zvcm1GbkFzc2V0c1VyaRKlAQoacXVhbnRpemVkX2ZlYXR1cmVfbWV0YWRhdGEYCCABKAsySC5zbmFwY2hhd + C5yZXNlYXJjaC5nYm1sLlByZXByb2Nlc3NlZE1ldGFkYXRhLkZlYXR1cmVRdWFudGl6YXRpb25NZXRhZGF0YUId4j8aEhhxdWFud + Gl6ZWRGZWF0dXJlTWV0YWRhdGFSGHF1YW50aXplZEZlYXR1cmVNZXRhZGF0YUIOCgxfZmVhdHVyZV9kaW0awgQKEkVkZ2VNZXRhZ + GF0YU91dHB1dBI4Cg9zcmNfbm9kZV9pZF9rZXkYASABKAlCEeI/DhIMc3JjTm9kZUlkS2V5UgxzcmNOb2RlSWRLZXkSOAoPZHN0X + 25vZGVfaWRfa2V5GAIgASgJQhHiPw4SDGRzdE5vZGVJZEtleVIMZHN0Tm9kZUlkS2V5EnYKDm1haW5fZWRnZV9pbmZvGAMgASgLM + j0uc25hcGNoYXQucmVzZWFyY2guZ2JtbC5QcmVwcm9jZXNzZWRNZXRhZGF0YS5FZGdlTWV0YWRhdGFJbmZvQhHiPw4SDG1haW5FZ + GdlSW5mb1IMbWFpbkVkZ2VJbmZvEocBChJwb3NpdGl2ZV9lZGdlX2luZm8YBCABKAsyPS5zbmFwY2hhdC5yZXNlYXJjaC5nYm1sL + lByZXByb2Nlc3NlZE1ldGFkYXRhLkVkZ2VNZXRhZGF0YUluZm9CFeI/EhIQcG9zaXRpdmVFZGdlSW5mb0gAUhBwb3NpdGl2ZUVkZ + 2VJbmZviAEBEocBChJuZWdhdGl2ZV9lZGdlX2luZm8YBSABKAsyPS5zbmFwY2hhdC5yZXNlYXJjaC5nYm1sLlByZXByb2Nlc3NlZ + E1ldGFkYXRhLkVkZ2VNZXRhZGF0YUluZm9CFeI/EhIQbmVnYXRpdmVFZGdlSW5mb0gBUhBuZWdhdGl2ZUVkZ2VJbmZviAEBQhUKE + 19wb3NpdGl2ZV9lZGdlX2luZm9CFQoTX25lZ2F0aXZlX2VkZ2VfaW5mbxqxAQosQ29uZGVuc2VkTm9kZVR5cGVUb1ByZXByb2Nlc + 3NlZE1ldGFkYXRhRW50cnkSGgoDa2V5GAEgASgNQgjiPwUSA2tleVIDa2V5EmEKBXZhbHVlGAIgASgLMj8uc25hcGNoYXQucmVzZ + WFyY2guZ2JtbC5QcmVwcm9jZXNzZWRNZXRhZGF0YS5Ob2RlTWV0YWRhdGFPdXRwdXRCCuI/BxIFdmFsdWVSBXZhbHVlOgI4ARqxA + QosQ29uZGVuc2VkRWRnZVR5cGVUb1ByZXByb2Nlc3NlZE1ldGFkYXRhRW50cnkSGgoDa2V5GAEgASgNQgjiPwUSA2tleVIDa2V5E + mEKBXZhbHVlGAIgASgLMj8uc25hcGNoYXQucmVzZWFyY2guZ2JtbC5QcmVwcm9jZXNzZWRNZXRhZGF0YS5FZGdlTWV0YWRhdGFPd + XRwdXRCCuI/BxIFdmFsdWVSBXZhbHVlOgI4AWIGcHJvdG8z""" ).mkString) lazy val scalaDescriptor: _root_.scalapb.descriptors.FileDescriptor = { val scalaProto = com.google.protobuf.descriptor.FileDescriptorProto.parseFrom(ProtoBytes) diff --git a/snapchat/research/gbml/preprocessed_metadata_pb2.py b/snapchat/research/gbml/preprocessed_metadata_pb2.py index 2fac76d2f..5a1c46a5d 100644 --- a/snapchat/research/gbml/preprocessed_metadata_pb2.py +++ b/snapchat/research/gbml/preprocessed_metadata_pb2.py @@ -14,7 +14,7 @@ -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n2snapchat/research/gbml/preprocessed_metadata.proto\x12\x16snapchat.research.gbml\"\x9c\x10\n\x14PreprocessedMetadata\x12\x8f\x01\n,condensed_node_type_to_preprocessed_metadata\x18\x01 \x03(\x0b\x32Y.snapchat.research.gbml.PreprocessedMetadata.CondensedNodeTypeToPreprocessedMetadataEntry\x12\x8f\x01\n,condensed_edge_type_to_preprocessed_metadata\x18\x02 \x03(\x0b\x32Y.snapchat.research.gbml.PreprocessedMetadata.CondensedEdgeTypeToPreprocessedMetadataEntry\x1aM\n\x19MultiBitQuantizationState\x12\x10\n\x08\x63lip_min\x18\x01 \x01(\x02\x12\x10\n\x08\x63lip_max\x18\x02 \x01(\x02\x12\x0c\n\x04\x62its\x18\x03 \x01(\r\x1a@\n\x1aSingleBitQuantizationState\x12\x10\n\x08neg_mean\x18\x01 \x01(\x02\x12\x10\n\x08pos_mean\x18\x02 \x01(\x02\x1a\xad\x02\n\x1b\x46\x65\x61tureQuantizationMetadata\x12\x1a\n\x12packed_feature_key\x18\x01 \x01(\t\x12!\n\x19quantized_feature_indices\x18\x02 \x03(\r\x12\x61\n\x0fmulti_bit_state\x18\x04 \x01(\x0b\x32\x46.snapchat.research.gbml.PreprocessedMetadata.MultiBitQuantizationStateH\x00\x12\x63\n\x10single_bit_state\x18\x05 \x01(\x0b\x32G.snapchat.research.gbml.PreprocessedMetadata.SingleBitQuantizationStateH\x00\x42\x07\n\x05state\x1a\x8a\x03\n\x12NodeMetadataOutput\x12\x13\n\x0bnode_id_key\x18\x01 \x01(\t\x12\x14\n\x0c\x66\x65\x61ture_keys\x18\x02 \x03(\t\x12\x12\n\nlabel_keys\x18\x03 \x03(\t\x12\x1b\n\x13tfrecord_uri_prefix\x18\x04 \x01(\t\x12\x12\n\nschema_uri\x18\x05 \x01(\t\x12$\n\x1c\x65numerated_node_ids_bq_table\x18\x06 \x01(\t\x12%\n\x1d\x65numerated_node_data_bq_table\x18\x07 \x01(\t\x12\x18\n\x0b\x66\x65\x61ture_dim\x18\x08 \x01(\rH\x00\x88\x01\x01\x12\x1f\n\x17transform_fn_assets_uri\x18\t \x01(\t\x12l\n\x1aquantized_feature_metadata\x18\n \x01(\x0b\x32H.snapchat.research.gbml.PreprocessedMetadata.FeatureQuantizationMetadataB\x0e\n\x0c_feature_dim\x1a\xdf\x01\n\x10\x45\x64geMetadataInfo\x12\x14\n\x0c\x66\x65\x61ture_keys\x18\x01 \x03(\t\x12\x12\n\nlabel_keys\x18\x02 \x03(\t\x12\x1b\n\x13tfrecord_uri_prefix\x18\x03 \x01(\t\x12\x12\n\nschema_uri\x18\x04 \x01(\t\x12%\n\x1d\x65numerated_edge_data_bq_table\x18\x05 \x01(\t\x12\x18\n\x0b\x66\x65\x61ture_dim\x18\x06 \x01(\rH\x00\x88\x01\x01\x12\x1f\n\x17transform_fn_assets_uri\x18\x07 \x01(\tB\x0e\n\x0c_feature_dim\x1a\x8b\x03\n\x12\x45\x64geMetadataOutput\x12\x17\n\x0fsrc_node_id_key\x18\x01 \x01(\t\x12\x17\n\x0f\x64st_node_id_key\x18\x02 \x01(\t\x12U\n\x0emain_edge_info\x18\x03 \x01(\x0b\x32=.snapchat.research.gbml.PreprocessedMetadata.EdgeMetadataInfo\x12^\n\x12positive_edge_info\x18\x04 \x01(\x0b\x32=.snapchat.research.gbml.PreprocessedMetadata.EdgeMetadataInfoH\x00\x88\x01\x01\x12^\n\x12negative_edge_info\x18\x05 \x01(\x0b\x32=.snapchat.research.gbml.PreprocessedMetadata.EdgeMetadataInfoH\x01\x88\x01\x01\x42\x15\n\x13_positive_edge_infoB\x15\n\x13_negative_edge_info\x1a\x8f\x01\n,CondensedNodeTypeToPreprocessedMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\r\x12N\n\x05value\x18\x02 \x01(\x0b\x32?.snapchat.research.gbml.PreprocessedMetadata.NodeMetadataOutput:\x02\x38\x01\x1a\x8f\x01\n,CondensedEdgeTypeToPreprocessedMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\r\x12N\n\x05value\x18\x02 \x01(\x0b\x32?.snapchat.research.gbml.PreprocessedMetadata.EdgeMetadataOutput:\x02\x38\x01\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n2snapchat/research/gbml/preprocessed_metadata.proto\x12\x16snapchat.research.gbml\"\x8a\x11\n\x14PreprocessedMetadata\x12\x8f\x01\n,condensed_node_type_to_preprocessed_metadata\x18\x01 \x03(\x0b\x32Y.snapchat.research.gbml.PreprocessedMetadata.CondensedNodeTypeToPreprocessedMetadataEntry\x12\x8f\x01\n,condensed_edge_type_to_preprocessed_metadata\x18\x02 \x03(\x0b\x32Y.snapchat.research.gbml.PreprocessedMetadata.CondensedEdgeTypeToPreprocessedMetadataEntry\x1aM\n\x19MultiBitQuantizationState\x12\x10\n\x08\x63lip_min\x18\x01 \x01(\x02\x12\x10\n\x08\x63lip_max\x18\x02 \x01(\x02\x12\x0c\n\x04\x62its\x18\x03 \x01(\r\x1a@\n\x1aSingleBitQuantizationState\x12\x10\n\x08neg_mean\x18\x01 \x01(\x02\x12\x10\n\x08pos_mean\x18\x02 \x01(\x02\x1a\xad\x02\n\x1b\x46\x65\x61tureQuantizationMetadata\x12\x1a\n\x12packed_feature_key\x18\x01 \x01(\t\x12!\n\x19quantized_feature_indices\x18\x02 \x03(\r\x12\x61\n\x0fmulti_bit_state\x18\x04 \x01(\x0b\x32\x46.snapchat.research.gbml.PreprocessedMetadata.MultiBitQuantizationStateH\x00\x12\x63\n\x10single_bit_state\x18\x05 \x01(\x0b\x32G.snapchat.research.gbml.PreprocessedMetadata.SingleBitQuantizationStateH\x00\x42\x07\n\x05state\x1a\x8a\x03\n\x12NodeMetadataOutput\x12\x13\n\x0bnode_id_key\x18\x01 \x01(\t\x12\x14\n\x0c\x66\x65\x61ture_keys\x18\x02 \x03(\t\x12\x12\n\nlabel_keys\x18\x03 \x03(\t\x12\x1b\n\x13tfrecord_uri_prefix\x18\x04 \x01(\t\x12\x12\n\nschema_uri\x18\x05 \x01(\t\x12$\n\x1c\x65numerated_node_ids_bq_table\x18\x06 \x01(\t\x12%\n\x1d\x65numerated_node_data_bq_table\x18\x07 \x01(\t\x12\x18\n\x0b\x66\x65\x61ture_dim\x18\x08 \x01(\rH\x00\x88\x01\x01\x12\x1f\n\x17transform_fn_assets_uri\x18\t \x01(\t\x12l\n\x1aquantized_feature_metadata\x18\n \x01(\x0b\x32H.snapchat.research.gbml.PreprocessedMetadata.FeatureQuantizationMetadataB\x0e\n\x0c_feature_dim\x1a\xcd\x02\n\x10\x45\x64geMetadataInfo\x12\x14\n\x0c\x66\x65\x61ture_keys\x18\x01 \x03(\t\x12\x12\n\nlabel_keys\x18\x02 \x03(\t\x12\x1b\n\x13tfrecord_uri_prefix\x18\x03 \x01(\t\x12\x12\n\nschema_uri\x18\x04 \x01(\t\x12%\n\x1d\x65numerated_edge_data_bq_table\x18\x05 \x01(\t\x12\x18\n\x0b\x66\x65\x61ture_dim\x18\x06 \x01(\rH\x00\x88\x01\x01\x12\x1f\n\x17transform_fn_assets_uri\x18\x07 \x01(\t\x12l\n\x1aquantized_feature_metadata\x18\x08 \x01(\x0b\x32H.snapchat.research.gbml.PreprocessedMetadata.FeatureQuantizationMetadataB\x0e\n\x0c_feature_dim\x1a\x8b\x03\n\x12\x45\x64geMetadataOutput\x12\x17\n\x0fsrc_node_id_key\x18\x01 \x01(\t\x12\x17\n\x0f\x64st_node_id_key\x18\x02 \x01(\t\x12U\n\x0emain_edge_info\x18\x03 \x01(\x0b\x32=.snapchat.research.gbml.PreprocessedMetadata.EdgeMetadataInfo\x12^\n\x12positive_edge_info\x18\x04 \x01(\x0b\x32=.snapchat.research.gbml.PreprocessedMetadata.EdgeMetadataInfoH\x00\x88\x01\x01\x12^\n\x12negative_edge_info\x18\x05 \x01(\x0b\x32=.snapchat.research.gbml.PreprocessedMetadata.EdgeMetadataInfoH\x01\x88\x01\x01\x42\x15\n\x13_positive_edge_infoB\x15\n\x13_negative_edge_info\x1a\x8f\x01\n,CondensedNodeTypeToPreprocessedMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\r\x12N\n\x05value\x18\x02 \x01(\x0b\x32?.snapchat.research.gbml.PreprocessedMetadata.NodeMetadataOutput:\x02\x38\x01\x1a\x8f\x01\n,CondensedEdgeTypeToPreprocessedMetadataEntry\x12\x0b\n\x03key\x18\x01 \x01(\r\x12N\n\x05value\x18\x02 \x01(\x0b\x32?.snapchat.research.gbml.PreprocessedMetadata.EdgeMetadataOutput:\x02\x38\x01\x62\x06proto3') @@ -106,7 +106,7 @@ _PREPROCESSEDMETADATA_CONDENSEDEDGETYPETOPREPROCESSEDMETADATAENTRY._options = None _PREPROCESSEDMETADATA_CONDENSEDEDGETYPETOPREPROCESSEDMETADATAENTRY._serialized_options = b'8\001' _PREPROCESSEDMETADATA._serialized_start=79 - _PREPROCESSEDMETADATA._serialized_end=2155 + _PREPROCESSEDMETADATA._serialized_end=2265 _PREPROCESSEDMETADATA_MULTIBITQUANTIZATIONSTATE._serialized_start=395 _PREPROCESSEDMETADATA_MULTIBITQUANTIZATIONSTATE._serialized_end=472 _PREPROCESSEDMETADATA_SINGLEBITQUANTIZATIONSTATE._serialized_start=474 @@ -116,11 +116,11 @@ _PREPROCESSEDMETADATA_NODEMETADATAOUTPUT._serialized_start=845 _PREPROCESSEDMETADATA_NODEMETADATAOUTPUT._serialized_end=1239 _PREPROCESSEDMETADATA_EDGEMETADATAINFO._serialized_start=1242 - _PREPROCESSEDMETADATA_EDGEMETADATAINFO._serialized_end=1465 - _PREPROCESSEDMETADATA_EDGEMETADATAOUTPUT._serialized_start=1468 - _PREPROCESSEDMETADATA_EDGEMETADATAOUTPUT._serialized_end=1863 - _PREPROCESSEDMETADATA_CONDENSEDNODETYPETOPREPROCESSEDMETADATAENTRY._serialized_start=1866 - _PREPROCESSEDMETADATA_CONDENSEDNODETYPETOPREPROCESSEDMETADATAENTRY._serialized_end=2009 - _PREPROCESSEDMETADATA_CONDENSEDEDGETYPETOPREPROCESSEDMETADATAENTRY._serialized_start=2012 - _PREPROCESSEDMETADATA_CONDENSEDEDGETYPETOPREPROCESSEDMETADATAENTRY._serialized_end=2155 + _PREPROCESSEDMETADATA_EDGEMETADATAINFO._serialized_end=1575 + _PREPROCESSEDMETADATA_EDGEMETADATAOUTPUT._serialized_start=1578 + _PREPROCESSEDMETADATA_EDGEMETADATAOUTPUT._serialized_end=1973 + _PREPROCESSEDMETADATA_CONDENSEDNODETYPETOPREPROCESSEDMETADATAENTRY._serialized_start=1976 + _PREPROCESSEDMETADATA_CONDENSEDNODETYPETOPREPROCESSEDMETADATAENTRY._serialized_end=2119 + _PREPROCESSEDMETADATA_CONDENSEDEDGETYPETOPREPROCESSEDMETADATAENTRY._serialized_start=2122 + _PREPROCESSEDMETADATA_CONDENSEDEDGETYPETOPREPROCESSEDMETADATAENTRY._serialized_end=2265 # @@protoc_insertion_point(module_scope) diff --git a/snapchat/research/gbml/preprocessed_metadata_pb2.pyi b/snapchat/research/gbml/preprocessed_metadata_pb2.pyi index 46b80c7bb..8ae9c79a5 100644 --- a/snapchat/research/gbml/preprocessed_metadata_pb2.pyi +++ b/snapchat/research/gbml/preprocessed_metadata_pb2.pyi @@ -154,6 +154,7 @@ class PreprocessedMetadata(google.protobuf.message.Message): ENUMERATED_EDGE_DATA_BQ_TABLE_FIELD_NUMBER: builtins.int FEATURE_DIM_FIELD_NUMBER: builtins.int TRANSFORM_FN_ASSETS_URI_FIELD_NUMBER: builtins.int + QUANTIZED_FEATURE_METADATA_FIELD_NUMBER: builtins.int @property def feature_keys(self) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: """Fields in output TFRecords which reference features.""" @@ -170,6 +171,9 @@ class PreprocessedMetadata(google.protobuf.message.Message): """Feature dimension after preprocessing""" transform_fn_assets_uri: builtins.str """Contains categorical feature vocabularies""" + @property + def quantized_feature_metadata(self) -> global___PreprocessedMetadata.FeatureQuantizationMetadata: + """Optional quantized main-edge feature metadata.""" def __init__( self, *, @@ -180,9 +184,10 @@ class PreprocessedMetadata(google.protobuf.message.Message): enumerated_edge_data_bq_table: builtins.str = ..., feature_dim: builtins.int | None = ..., transform_fn_assets_uri: builtins.str = ..., + quantized_feature_metadata: global___PreprocessedMetadata.FeatureQuantizationMetadata | None = ..., ) -> None: ... - def HasField(self, field_name: typing_extensions.Literal["_feature_dim", b"_feature_dim", "feature_dim", b"feature_dim"]) -> builtins.bool: ... - def ClearField(self, field_name: typing_extensions.Literal["_feature_dim", b"_feature_dim", "enumerated_edge_data_bq_table", b"enumerated_edge_data_bq_table", "feature_dim", b"feature_dim", "feature_keys", b"feature_keys", "label_keys", b"label_keys", "schema_uri", b"schema_uri", "tfrecord_uri_prefix", b"tfrecord_uri_prefix", "transform_fn_assets_uri", b"transform_fn_assets_uri"]) -> None: ... + def HasField(self, field_name: typing_extensions.Literal["_feature_dim", b"_feature_dim", "feature_dim", b"feature_dim", "quantized_feature_metadata", b"quantized_feature_metadata"]) -> builtins.bool: ... + def ClearField(self, field_name: typing_extensions.Literal["_feature_dim", b"_feature_dim", "enumerated_edge_data_bq_table", b"enumerated_edge_data_bq_table", "feature_dim", b"feature_dim", "feature_keys", b"feature_keys", "label_keys", b"label_keys", "quantized_feature_metadata", b"quantized_feature_metadata", "schema_uri", b"schema_uri", "tfrecord_uri_prefix", b"tfrecord_uri_prefix", "transform_fn_assets_uri", b"transform_fn_assets_uri"]) -> None: ... def WhichOneof(self, oneof_group: typing_extensions.Literal["_feature_dim", b"_feature_dim"]) -> typing_extensions.Literal["feature_dim"] | None: ... class EdgeMetadataOutput(google.protobuf.message.Message): diff --git a/tests/integration/pipeline/data_preprocessor/feature_quantization_transform_test.py b/tests/integration/pipeline/data_preprocessor/feature_quantization_transform_test.py index 00cfc390c..d2ef8c309 100644 --- a/tests/integration/pipeline/data_preprocessor/feature_quantization_transform_test.py +++ b/tests/integration/pipeline/data_preprocessor/feature_quantization_transform_test.py @@ -5,20 +5,204 @@ import apache_beam as beam import pyarrow as pa import tensorflow as tf +import tensorflow_data_validation as tfdv +import torch from apache_beam.testing.test_pipeline import TestPipeline from apache_beam.testing.util import assert_that, equal_to from parameterized import parameterized from tensorflow_metadata.proto.v0 import schema_pb2 from tensorflow_transform.tf_metadata.dataset_metadata import DatasetMetadata +from torch_geometric.data import Data +from gigl.common.beam.better_tfrecordio import BetterWriteToTFRecord +from gigl.common.data.dataloaders import TFDatasetOptions, TFRecordDataLoader +from gigl.distributed.utils.neighborloader import ( + EDGE_PACKED_FEATURES_METADATA_KEY, + materialize_quantized_edge_features, +) +from gigl.distributed.utils.serialized_graph_metadata_translator import ( + convert_pb_to_serialized_graph_metadata, +) +from gigl.src.common.types.pb_wrappers.graph_metadata import GraphMetadataPbWrapper +from gigl.src.common.types.pb_wrappers.preprocessed_metadata import ( + PreprocessedMetadataPbWrapper, +) from gigl.src.data_preprocessor.lib.transform.feature_quantization import ( + EDGE_PACKED_FEATURE_KEY, + NODE_PACKED_FEATURE_KEY, apply_feature_quantization_transform, ) from gigl.src.data_preprocessor.lib.types import FeatureQuantizationSpec +from snapchat.research.gbml import graph_schema_pb2, preprocessed_metadata_pb2 from tests.test_assets.test_case import TestCase class FeatureQuantizationTransformTest(TestCase): + def test_edge_quantization_round_trips_through_storage_and_loading(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + metadata_path = os.path.join(temp_dir, "feature_quantization_metadata.json") + tfrecord_prefix = os.path.join(temp_dir, "edges") + schema_path = os.path.join(temp_dir, "schema.pbtxt") + logical_metadata = DatasetMetadata.from_feature_spec( + { + "src": tf.io.FixedLenFeature(shape=[], dtype=tf.int64), + "dst": tf.io.FixedLenFeature(shape=[], dtype=tf.int64), + "quantized": tf.io.FixedLenFeature(shape=[], dtype=tf.float32), + "raw": tf.io.FixedLenFeature(shape=[], dtype=tf.float32), + } + ) + logical_batches = [ + pa.RecordBatch.from_arrays( + [ + pa.array([[0], [1]], type=pa.list_(pa.int64())), + pa.array([[1], [0]], type=pa.list_(pa.int64())), + pa.array([[-2.0], [8.0]], type=pa.list_(pa.float32())), + pa.array([[10.0], [20.0]], type=pa.list_(pa.float32())), + ], + names=["src", "dst", "quantized", "raw"], + ) + ] + + with TestPipeline() as pipeline: + transformed_batches, physical_metadata = ( + apply_feature_quantization_transform( + logical_features=pipeline + | "Create edge RecordBatches" >> beam.Create(logical_batches), + logical_metadata=logical_metadata, + logical_feature_keys=["quantized", "raw"], + quantization_spec=FeatureQuantizationSpec( + feature_keys=["quantized"], bits=2 + ), + quantization_metadata_path=metadata_path, + packed_feature_key=EDGE_PACKED_FEATURE_KEY, + ) + ) + transformed_batches | "Write edge TFRecords" >> BetterWriteToTFRecord( + file_path_prefix=tfrecord_prefix, + transformed_metadata=physical_metadata, + num_shards=1, + ) + + tfdv.write_schema_text(logical_metadata.schema, schema_path) + with tf.io.gfile.GFile(metadata_path) as metadata_file: + quantization_metadata = json.loads(metadata_file.read()) + quantization_metadata_pb = preprocessed_metadata_pb2.PreprocessedMetadata.FeatureQuantizationMetadata( + packed_feature_key=quantization_metadata["packed_feature_key"], + quantized_feature_indices=quantization_metadata[ + "quantized_feature_indices" + ], + ) + quantization_metadata_pb.multi_bit_state.bits = quantization_metadata[ + "bits" + ] + quantization_metadata_pb.multi_bit_state.clip_min = quantization_metadata[ + "clip_min" + ] + quantization_metadata_pb.multi_bit_state.clip_max = quantization_metadata[ + "clip_max" + ] + self.assertEqual( + quantization_metadata_pb.packed_feature_key, "edge_packed_features" + ) + self.assertEqual( + list(quantization_metadata_pb.quantized_feature_indices), [0] + ) + + preprocessed_metadata_pb = preprocessed_metadata_pb2.PreprocessedMetadata() + preprocessed_metadata_pb.condensed_node_type_to_preprocessed_metadata[ + 0 + ].node_id_key = "node_id" + edge_metadata = ( + preprocessed_metadata_pb.condensed_edge_type_to_preprocessed_metadata[0] + ) + edge_metadata.src_node_id_key = "src" + edge_metadata.dst_node_id_key = "dst" + edge_metadata.main_edge_info.CopyFrom( + preprocessed_metadata_pb2.PreprocessedMetadata.EdgeMetadataInfo( + feature_keys=["quantized", "raw"], + feature_dim=2, + tfrecord_uri_prefix=temp_dir, + schema_uri=schema_path, + quantized_feature_metadata=quantization_metadata_pb, + ) + ) + graph_metadata_pb = graph_schema_pb2.GraphMetadata( + node_types=["node"], + edge_types=[ + graph_schema_pb2.EdgeType( + src_node_type="node", relation="connects", dst_node_type="node" + ) + ], + condensed_node_type_map={0: "node"}, + condensed_edge_type_map={ + 0: graph_schema_pb2.EdgeType( + src_node_type="node", relation="connects", dst_node_type="node" + ) + }, + ) + serialized_metadata = convert_pb_to_serialized_graph_metadata( + preprocessed_metadata_pb_wrapper=PreprocessedMetadataPbWrapper( + preprocessed_metadata_pb + ), + graph_metadata_pb_wrapper=GraphMetadataPbWrapper(graph_metadata_pb), + tfrecord_uri_pattern="edges.*\\.tfrecord", + ) + loaded = TFRecordDataLoader(rank=0, world_size=1).load_as_torch_tensors( + serialized_tf_record_info=serialized_metadata.edge_entity_info, + tf_dataset_options=TFDatasetOptions(deterministic=True), + ) + + assert loaded.features is not None + assert loaded.quantized_features is not None + self.assert_tensor_equality(loaded.ids, torch.tensor([[0, 1], [1, 0]])) + self.assert_tensor_equality(loaded.features, torch.tensor([[10.0], [20.0]])) + self.assert_tensor_equality( + loaded.quantized_features, torch.tensor([[0], [192]], dtype=torch.uint8) + ) + materialized, remaining_metadata = materialize_quantized_edge_features( + data=Data(edge_index=loaded.ids, edge_attr=loaded.features), + metadata={EDGE_PACKED_FEATURES_METADATA_KEY: loaded.quantized_features}, + edge_quantization_metadata=serialized_metadata.edge_quantization_metadata, + ) + self.assert_tensor_equality( + materialized.edge_attr, torch.tensor([[-2.0, 10.0], [8.0, 20.0]]) + ) + self.assertEqual(remaining_metadata, {}) + + def test_apply_feature_quantization_transform_rejects_reserved_schema_key( + self, + ) -> None: + logical_metadata = DatasetMetadata.from_feature_spec( + { + "f0": tf.io.FixedLenFeature(shape=[], dtype=tf.float32), + "edge_packed_features": tf.io.FixedLenFeature( + shape=[], dtype=tf.string + ), + } + ) + + with ( + self.assertRaisesRegex(ValueError, "Reserved packed feature key"), + TestPipeline() as pipeline, + ): + apply_feature_quantization_transform( + logical_features=pipeline + | "Create collision input" + >> beam.Create( + [ + pa.RecordBatch.from_arrays( + [pa.array([1.0]), pa.array([b"existing"])], + names=["f0", "edge_packed_features"], + ) + ] + ), + logical_metadata=logical_metadata, + logical_feature_keys=["f0"], + quantization_spec=FeatureQuantizationSpec(feature_keys=["f0"], bits=2), + quantization_metadata_path="unused", + packed_feature_key="edge_packed_features", + ) + @parameterized.expand( [ ( @@ -87,6 +271,7 @@ def test_apply_feature_quantization_transform_writes_metadata( feature_keys=logical_feature_keys, bits=bits ), quantization_metadata_path=metadata_path, + packed_feature_key=NODE_PACKED_FEATURE_KEY, ) ) if use_deferred_metadata: diff --git a/tests/test_assets/distributed/run_distributed_partitioner.py b/tests/test_assets/distributed/run_distributed_partitioner.py index 046b8bf49..89c863d07 100644 --- a/tests/test_assets/distributed/run_distributed_partitioner.py +++ b/tests/test_assets/distributed/run_distributed_partitioner.py @@ -22,6 +22,9 @@ class InputDataStrategy(Enum): REGISTER_EDGE_WEIGHTS_WITHOUT_EDGE_FEATURES = ( "REGISTER_EDGE_WEIGHTS_WITHOUT_EDGE_FEATURES" ) + REGISTER_EDGE_QUANTIZED_FEATURES_WITHOUT_EDGE_FEATURES = ( + "REGISTER_EDGE_QUANTIZED_FEATURES_WITHOUT_EDGE_FEATURES" + ) def run_distributed_partitioner( @@ -95,7 +98,29 @@ def run_distributed_partitioner( init_rpc(master_addr=master_addr, master_port=master_port, num_rpc_threads=4) dist_partitioner: DistPartitioner - if input_data_strategy in ( + if ( + input_data_strategy + == InputDataStrategy.REGISTER_EDGE_QUANTIZED_FEATURES_WITHOUT_EDGE_FEATURES + ): + dist_partitioner = partitioner_class( + should_assign_edges_by_src_node=should_assign_edges_by_src_node, + ) + dist_partitioner.register_node_ids(node_ids=node_ids) + dist_partitioner.register_edge_index(edge_index=edge_index) + edge_quantized_features: Union[torch.Tensor, dict[EdgeType, torch.Tensor]] + if isinstance(edge_index, dict): + edge_index_by_type = cast(dict[EdgeType, torch.Tensor], edge_index) + edge_quantized_features = { + edge_type: indices[0].to(torch.uint8).unsqueeze(1) + for edge_type, indices in edge_index_by_type.items() + } + else: + edge_quantized_features = edge_index[0].to(torch.uint8).unsqueeze(1) + dist_partitioner.register_edge_quantized_features( + edge_quantized_features=edge_quantized_features + ) + partition_output = dist_partitioner.partition() + elif input_data_strategy in ( InputDataStrategy.REGISTER_ALL_ENTITIES_SEPARATELY, InputDataStrategy.REGISTER_EDGE_WEIGHTS_WITHOUT_EDGE_FEATURES, ): @@ -119,6 +144,7 @@ def run_distributed_partitioner( ( output_edge_index, output_edge_features, + _, output_edge_partition_book, ) = dist_partitioner.partition_edge_index_and_edge_features( node_partition_book=output_node_partition_book @@ -181,6 +207,7 @@ def run_distributed_partitioner( ( output_graph, output_edge_features, + _, output_edge_partition_book, ) = dist_partitioner.partition_edge_index_and_edge_features( node_partition_book=output_node_partition_book diff --git a/tests/unit/common/data/dataloaders_test.py b/tests/unit/common/data/dataloaders_test.py index 3bfaff851..12b5de2e5 100644 --- a/tests/unit/common/data/dataloaders_test.py +++ b/tests/unit/common/data/dataloaders_test.py @@ -19,6 +19,7 @@ from gigl.common.data.load_torch_tensors import ( SerializedGraphMetadata, load_torch_tensors_from_tf_record, + remove_sampling_weight_from_edge_quantization_metadata, ) from gigl.src.common.types.pb_wrappers.gbml_config import GbmlConfigPbWrapper from gigl.src.data_preprocessor.lib.types import FeatureSpecDict @@ -29,6 +30,7 @@ from gigl.src.mocking.mocking_assets.mocked_datasets_for_pipeline_tests import ( CORA_NODE_CLASSIFICATION_MOCKED_DATASET_INFO, ) +from gigl.types.graph import FeatureQuantizationMetadata from tests.test_assets.test_case import TestCase _FEATURE_SPEC_WITH_ENTITY_KEY: FeatureSpecDict = { @@ -644,6 +646,91 @@ def test_load_edge_weights_from_tf_record(self): torch.tensor(sorted(edge_feature_vals), dtype=torch.float32), ) + def test_load_edge_weights_rejects_non_raw_field_before_loading(self) -> None: + missing_path = UriFactory.create_uri("/does/not/exist") + serialized_graph_metadata = SerializedGraphMetadata( + node_entity_info=SerializedTFRecordInfo( + tfrecord_uri_prefix=missing_path, + feature_spec={"node_id": tf.io.FixedLenFeature([], tf.int64)}, + feature_keys=[], + feature_dim=0, + entity_key="node_id", + ), + edge_entity_info=SerializedTFRecordInfo( + tfrecord_uri_prefix=missing_path, + feature_spec={ + "src_id": tf.io.FixedLenFeature([], tf.int64), + "dst_id": tf.io.FixedLenFeature([], tf.int64), + "edge_packed_features": tf.io.FixedLenFeature([], tf.string), + }, + feature_keys=[], + feature_dim=0, + entity_key=("src_id", "dst_id"), + packed_feature_key="edge_packed_features", + packed_feature_dim=1, + ), + ) + + with self.assertRaises(ValueError): + load_torch_tensors_from_tf_record( + tf_record_dataloader=TFRecordDataLoader(rank=0, world_size=1), + serialized_graph_metadata=serialized_graph_metadata, + should_load_tensors_in_parallel=False, + weight_edge_feat_name="quantized_weight", + ) + + def test_sampling_weight_removal_updates_edge_quantization_metadata( + self, + ) -> None: + missing_path = UriFactory.create_uri("/does/not/exist") + serialized_graph_metadata = SerializedGraphMetadata( + node_entity_info=SerializedTFRecordInfo( + tfrecord_uri_prefix=missing_path, + feature_spec={"node_id": tf.io.FixedLenFeature([], tf.int64)}, + feature_keys=[], + feature_dim=0, + entity_key="node_id", + ), + edge_entity_info=SerializedTFRecordInfo( + tfrecord_uri_prefix=missing_path, + feature_spec={ + "src_id": tf.io.FixedLenFeature([], tf.int64), + "dst_id": tf.io.FixedLenFeature([], tf.int64), + "raw_embedding": tf.io.FixedLenFeature([2], tf.float32), + "weight": tf.io.FixedLenFeature([], tf.float32), + "edge_packed_features": tf.io.FixedLenFeature([], tf.string), + }, + feature_keys=["raw_embedding", "weight"], + feature_dim=3, + entity_key=("src_id", "dst_id"), + packed_feature_key="edge_packed_features", + packed_feature_dim=1, + ), + edge_quantization_metadata=FeatureQuantizationMetadata( + bits=2, + feature_dim=4, + quantized_feature_indices=(3,), + clip_min=0.0, + clip_max=3.0, + ), + ) + + adjusted_metadata = remove_sampling_weight_from_edge_quantization_metadata( + serialized_graph_metadata=serialized_graph_metadata, + weight_edge_feat_name="weight", + ) + + self.assertEqual( + adjusted_metadata, + FeatureQuantizationMetadata( + bits=2, + feature_dim=3, + quantized_feature_indices=(2,), + clip_min=0.0, + clip_max=3.0, + ), + ) + def test_load_edge_weights_multidim_feature(self): """Weight column offset is correct when a preceding feature key is multi-dimensional. diff --git a/tests/unit/distributed/dist_ablp_neighborloader_test.py b/tests/unit/distributed/dist_ablp_neighborloader_test.py index a3d25dfeb..0c6a7d789 100644 --- a/tests/unit/distributed/dist_ablp_neighborloader_test.py +++ b/tests/unit/distributed/dist_ablp_neighborloader_test.py @@ -218,6 +218,16 @@ def _run_quantized_homogeneous_ablp_loader(_: int, dataset: DistDataset) -> None for batch in loader: assert isinstance(batch, Data) assert_tensor_equality(batch.x, expected_features[batch.node]) + expected_edge_features = { + (0, 1): torch.tensor([0.0, 3.0]), + (1, 0): torch.tensor([2.0, 1.0]), + } + for local_edge_index, edge_feature in zip(batch.edge_index.T, batch.edge_attr): + source, destination = batch.node[local_edge_index] + assert_tensor_equality( + edge_feature, + expected_edge_features[(source.item(), destination.item())], + ) assert _global_pair_set(batch.node, batch.node, batch.y_positive) == [(0, 1)] assert _global_pair_set(batch.node, batch.node, batch.y_negative) == [(0, 2)] batch_count += 1 @@ -776,6 +786,7 @@ def test_homogeneous_ablp_materializes_quantized_features(self) -> None: node_labels=None, edge_index=torch.tensor([[0, 1], [1, 0]]), edge_features=None, + edge_quantized_features=torch.tensor([[48], [144]], dtype=torch.uint8), positive_label=torch.tensor([[0], [1]]), negative_label=torch.tensor([[0], [2]]), ) @@ -792,6 +803,7 @@ def test_homogeneous_ablp_materializes_quantized_features(self) -> None: assert isinstance(loaded_graph_tensors.edge_index, dict) assert isinstance(loaded_graph_tensors.node_features, dict) assert isinstance(loaded_graph_tensors.node_quantized_features, dict) + assert isinstance(loaded_graph_tensors.edge_quantized_features, dict) edge_index = cast(dict[EdgeType, torch.Tensor], loaded_graph_tensors.edge_index) node_features = cast( dict[NodeType, torch.Tensor], loaded_graph_tensors.node_features @@ -799,6 +811,9 @@ def test_homogeneous_ablp_materializes_quantized_features(self) -> None: node_quantized_features = cast( dict[NodeType, torch.Tensor], loaded_graph_tensors.node_quantized_features ) + edge_quantized_features = cast( + dict[EdgeType, torch.Tensor], loaded_graph_tensors.edge_quantized_features + ) partition_output = PartitionOutput( node_partition_book={DEFAULT_HOMOGENEOUS_NODE_TYPE: torch.zeros(3)}, edge_partition_book={edge_type: torch.zeros(3) for edge_type in edge_index}, @@ -820,6 +835,12 @@ def test_homogeneous_ablp_materializes_quantized_features(self) -> None: ids=torch.arange(3), ) }, + partitioned_edge_quantized_features={ + DEFAULT_HOMOGENEOUS_EDGE_TYPE: FeaturePartitionData( + feats=edge_quantized_features[DEFAULT_HOMOGENEOUS_EDGE_TYPE], + ids=torch.arange(2), + ) + }, partitioned_edge_features=None, partitioned_negative_labels=None, partitioned_positive_labels=None, @@ -830,6 +851,13 @@ def test_homogeneous_ablp_materializes_quantized_features(self) -> None: world_size=1, edge_dir="out", node_quantization_metadata=quantization_metadata, + edge_quantization_metadata=FeatureQuantizationMetadata( + bits=2, + feature_dim=2, + quantized_feature_indices=(0, 1), + clip_min=0.0, + clip_max=3.0, + ), ) dataset.build(partition_output=partition_output) diff --git a/tests/unit/distributed/dist_server_test.py b/tests/unit/distributed/dist_server_test.py index c876fcef1..1b0f1675e 100644 --- a/tests/unit/distributed/dist_server_test.py +++ b/tests/unit/distributed/dist_server_test.py @@ -1,5 +1,5 @@ import threading -from unittest.mock import MagicMock, patch +from unittest.mock import MagicMock, call, patch import torch from absl.testing import absltest @@ -65,45 +65,72 @@ def test_get_node_feature_info_with_homogeneous_dataset(self) -> None: # Verify it returns the correct feature info self.assertIsNone(node_feature_info) - def test_get_node_quantization_metadata(self) -> None: - metadata = FeatureQuantizationMetadata( + def test_get_quantization_metadata(self) -> None: + node_metadata = FeatureQuantizationMetadata( bits=2, feature_dim=2, quantized_feature_indices=(0, 1), clip_min=0.0, clip_max=3.0, ) + edge_metadata = { + USER_TO_STORY: FeatureQuantizationMetadata( + bits=4, + feature_dim=3, + quantized_feature_indices=(0, 2), + clip_min=-1.0, + clip_max=1.0, + ) + } dataset = DistDataset( rank=0, world_size=1, edge_dir="out", - node_quantization_metadata=metadata, + node_quantization_metadata=node_metadata, + edge_quantization_metadata=edge_metadata, ) server = dist_server.DistServer(dataset) - self.assertEqual(server.get_node_quantization_metadata(), metadata) + self.assertEqual(server.get_node_quantization_metadata(), node_metadata) + self.assertEqual(server.get_edge_quantization_metadata(), edge_metadata) - def test_remote_dataset_fetches_node_quantization_metadata(self) -> None: - metadata = FeatureQuantizationMetadata( + def test_remote_dataset_fetches_quantization_metadata(self) -> None: + node_metadata = FeatureQuantizationMetadata( bits=2, feature_dim=2, quantized_feature_indices=(0, 1), clip_min=0.0, clip_max=3.0, ) + edge_metadata = { + USER_TO_STORY: FeatureQuantizationMetadata( + bits=4, + feature_dim=3, + quantized_feature_indices=(0, 2), + clip_min=-1.0, + clip_max=1.0, + ) + } with patch( "gigl.distributed.graph_store.remote_dist_dataset.request_server", - return_value=metadata, + side_effect=[node_metadata, edge_metadata], ) as request_server: remote_dataset = RemoteDistDataset(cluster_info=MagicMock(), local_rank=0) self.assertEqual( - remote_dataset.fetch_node_quantization_metadata(), metadata + remote_dataset.fetch_node_quantization_metadata(), node_metadata + ) + self.assertEqual( + remote_dataset.fetch_edge_quantization_metadata(), edge_metadata ) - request_server.assert_called_once_with( - 0, dist_server.DistServer.get_node_quantization_metadata + self.assertEqual( + request_server.call_args_list, + [ + call(0, dist_server.DistServer.get_node_quantization_metadata), + call(0, dist_server.DistServer.get_edge_quantization_metadata), + ], ) def test_get_edge_feature_info_with_heterogeneous_dataset(self) -> None: diff --git a/tests/unit/distributed/distributed_neighborloader_test.py b/tests/unit/distributed/distributed_neighborloader_test.py index e51f10ea0..93cbcad2c 100644 --- a/tests/unit/distributed/distributed_neighborloader_test.py +++ b/tests/unit/distributed/distributed_neighborloader_test.py @@ -12,6 +12,7 @@ from gigl.distributed.dataset_factory import build_dataset from gigl.distributed.dist_dataset import DistDataset from gigl.distributed.distributed_neighborloader import DistNeighborLoader +from gigl.distributed.sampler import EDGE_PACKED_FEATURES_METADATA_KEY from gigl.distributed.utils import get_free_port from gigl.distributed.utils.neighborloader import DatasetSchema from gigl.distributed.utils.serialized_graph_metadata_translator import ( @@ -1164,17 +1165,40 @@ def test_independent_calls_produce_equal_configs(self) -> None: ) self.assertEqual(first, second) + def test_packed_edge_metadata_requests_sampled_edge_ids(self) -> None: + schema = self._schema() + schema = DatasetSchema( + is_homogeneous_with_labeled_edge_type=False, + edge_types=schema.edge_types, + node_feature_info=schema.node_feature_info, + edge_feature_info=None, + edge_dir=schema.edge_dir, + edge_quantization_metadata=FeatureQuantizationMetadata( + bits=2, + feature_dim=2, + quantized_feature_indices=(0, 1), + clip_min=0.0, + clip_max=3.0, + ), + ) + + config = BaseDistLoader.create_sampling_config( + num_neighbors=[1], dataset_schema=schema + ) + + self.assertTrue(config.with_edge) + # NOTE on the test strategy: GiGL loaders always sample via the multiprocess # producer, which spawns worker subprocesses with a *fresh* interpreter # (`mp.get_context("spawn")`, dist_sampling_producer.py). A `mock.patch` applied in the # loader process therefore never reaches the sampler running in that subprocess, so we # cannot inject a synthetic failure by mocking the sampler. Instead we reproduce a real -# sampler failure end-to-end: a heterogeneous dataset with edge features on only a -# subset of its message-passing edge types. When the featureless type is reached during -# sampling, its feature lookup raises `KeyError` inside the sampling coroutine — the exact -# swallowed-exception case this change surfaces. Without the change this hangs forever, so -# the test uses a bounded join. +# sampler failure end-to-end: a heterogeneous dataset with an incomplete feature store +# for one message-passing edge type. When the missing edge ID is reached during sampling, +# its feature lookup raises inside the sampling coroutine - the exact swallowed-exception +# case this change surfaces. Without the change this hangs forever, so the test uses a +# bounded join. def _run_partial_edge_feature_coverage_raises( @@ -1201,11 +1225,11 @@ def _run_partial_edge_feature_coverage_raises( class TestSamplingErrorPropagation(TestCase): def _build_partial_edge_feature_dataset(self) -> DistDataset: - """Build a hetero dataset with edge features on only one message-passing type. + """Build a hetero dataset with an incomplete edge feature store. - ``user-to-story`` has edge features; ``story-to-user`` does not. Both are - reachable from ``user`` seeds within a 2-hop fanout, so the featureless type is - actually sampled and its edge-feature lookup raises inside the coroutine. + Both edge types are reachable from ``user`` seeds within a 2-hop fanout. + ``story-to-user`` omits the last edge ID, so its feature lookup raises inside + the sampling coroutine. """ n = 5 edge_index = torch.tensor([[0, 1, 2, 3, 4], [0, 1, 2, 3, 4]]) @@ -1235,6 +1259,9 @@ def _build_partial_edge_feature_dataset(self) -> DistDataset: _USER_TO_STORY: FeaturePartitionData( feats=torch.ones(n, 3), ids=torch.arange(n) ), + _STORY_TO_USER: FeaturePartitionData( + feats=torch.ones(n - 1, 3), ids=torch.arange(n - 1) + ), }, partitioned_positive_labels=None, partitioned_negative_labels=None, @@ -1264,7 +1291,166 @@ def test_reachable_sampler_failure_raises_not_hangs(self) -> None: message = error_holder.get("msg", "") # The training process raised with the worker's real traceback embedded. self.assertIn("sampling worker failed", message.lower()) - self.assertIn("story-to-user", message) + self.assertIn("IndexError", message) + self.assertIn("index 4 is out of bounds", message) + + +def _run_heterogeneous_partially_quantized_edge_feature_neighbor_loader( + _, + dataset: DistDataset, + expected_edge_features: dict[EdgeType, torch.Tensor], +) -> None: + create_test_process_group() + loader = DistNeighborLoader( + dataset=dataset, + input_nodes=(_USER, torch.tensor([0])), + num_neighbors=[1, 1], + batch_size=1, + pin_memory_device=torch.device("cpu"), + ) + + batch = next(iter(loader)) + assert isinstance(batch, HeteroData) + for edge_type, expected_features in expected_edge_features.items(): + assert_tensor_equality(batch[edge_type].edge_attr, expected_features) + assert not hasattr(batch[_STORY_TO_USER], "edge_attr") + shutdown_rpc() + + +def _run_incoming_heterogeneous_quantized_edge_feature_neighbor_loader( + _, + dataset: DistDataset, + expected_edge_features: torch.Tensor, +) -> None: + create_test_process_group() + loader = DistNeighborLoader( + dataset=dataset, + input_nodes=(_STORY, torch.tensor([0])), + num_neighbors=[1], + batch_size=1, + pin_memory_device=torch.device("cpu"), + ) + + edge_feature_info = loader._edge_feature_info + edge_quantization_metadata = loader._edge_quantization_metadata + assert isinstance(edge_feature_info, dict) + assert isinstance(edge_quantization_metadata, dict) + assert set(edge_feature_info) == {_USER_TO_STORY} + assert set(edge_quantization_metadata) == {_USER_TO_STORY} + assert edge_feature_info[_USER_TO_STORY].dim == 2 + assert edge_quantization_metadata[_USER_TO_STORY].feature_dim == 4 + + batch = next(iter(loader)) + assert isinstance(batch, HeteroData) + assert_tensor_equality(batch[_USER_TO_STORY].edge_attr, expected_edge_features) + assert EDGE_PACKED_FEATURES_METADATA_KEY not in batch + assert f"{EDGE_PACKED_FEATURES_METADATA_KEY}.{_USER_TO_STORY}" not in batch + shutdown_rpc() + + +class HeterogeneousEdgeFeatureLookupTest(TestCase): + def test_heterogeneous_loader_supports_partially_quantized_edge_types( + self, + ) -> None: + # Sampling user reaches both edge types. Only user-to-story has raw and + # packed features, so story-to-user must not be looked up in either store. + expected_edge_features = {_USER_TO_STORY: torch.tensor([[0.0, 10.0]])} + partition_output = PartitionOutput( + node_partition_book={_USER: torch.zeros(1), _STORY: torch.zeros(1)}, + edge_partition_book={_USER_TO_STORY: torch.zeros(1)}, + partitioned_edge_index={ + _USER_TO_STORY: GraphPartitionData( + edge_index=torch.tensor([[0], [0]]), edge_ids=None + ) + }, + partitioned_node_features=None, + partitioned_edge_features={ + _USER_TO_STORY: FeaturePartitionData( + feats=torch.tensor([[10.0]]), ids=torch.tensor([0]) + ), + }, + partitioned_edge_quantized_features={ + _USER_TO_STORY: FeaturePartitionData( + feats=torch.tensor([[0]], dtype=torch.uint8), + ids=torch.tensor([0]), + ) + }, + partitioned_positive_labels=None, + partitioned_negative_labels=None, + partitioned_node_labels=None, + ) + dataset = DistDataset( + rank=0, + world_size=1, + edge_dir="out", + edge_quantization_metadata={ + _USER_TO_STORY: FeatureQuantizationMetadata( + bits=2, + feature_dim=2, + quantized_feature_indices=(0,), + clip_min=0.0, + clip_max=3.0, + ) + }, + ) + dataset.build(partition_output=partition_output) + + mp.spawn( + fn=_run_heterogeneous_partially_quantized_edge_feature_neighbor_loader, + args=(dataset, expected_edge_features), + ) + + def test_incoming_edges_reverse_feature_metadata_and_output_stores(self) -> None: + partition_output = PartitionOutput( + node_partition_book={_USER: torch.zeros(1), _STORY: torch.zeros(1)}, + edge_partition_book={ + _USER_TO_STORY: torch.zeros(1), + _STORY_TO_USER: torch.zeros(1), + }, + partitioned_edge_index={ + _USER_TO_STORY: GraphPartitionData( + edge_index=torch.tensor([[0], [0]]), edge_ids=None + ), + _STORY_TO_USER: GraphPartitionData( + edge_index=torch.tensor([[0], [0]]), edge_ids=None + ), + }, + partitioned_node_features=None, + partitioned_edge_features={ + _USER_TO_STORY: FeaturePartitionData( + feats=torch.tensor([[10.0, 20.0]]), ids=torch.tensor([0]) + ) + }, + partitioned_edge_quantized_features={ + _USER_TO_STORY: FeaturePartitionData( + feats=torch.tensor([[48]], dtype=torch.uint8), + ids=torch.tensor([0]), + ) + }, + partitioned_positive_labels=None, + partitioned_negative_labels=None, + partitioned_node_labels=None, + ) + dataset = DistDataset( + rank=0, + world_size=1, + edge_dir="in", + edge_quantization_metadata={ + _USER_TO_STORY: FeatureQuantizationMetadata( + bits=2, + feature_dim=4, + quantized_feature_indices=(0, 2), + clip_min=0.0, + clip_max=3.0, + ) + }, + ) + dataset.build(partition_output=partition_output) + + mp.spawn( + fn=_run_incoming_heterogeneous_quantized_edge_feature_neighbor_loader, + args=(dataset, torch.tensor([[0.0, 10.0, 3.0, 20.0]])), + ) if __name__ == "__main__": diff --git a/tests/unit/distributed/distributed_partitioner_test.py b/tests/unit/distributed/distributed_partitioner_test.py index 0f817bafa..f3aaea091 100644 --- a/tests/unit/distributed/distributed_partitioner_test.py +++ b/tests/unit/distributed/distributed_partitioner_test.py @@ -686,6 +686,22 @@ def _assert_label_outputs( partitioner_class=DistRangePartitioner, expected_pb_dtype=torch.int64, ), + param( + "Homogeneous packed-edge-only tensor partitioning", + is_heterogeneous=False, + input_data_strategy=InputDataStrategy.REGISTER_EDGE_QUANTIZED_FEATURES_WITHOUT_EDGE_FEATURES, + should_assign_edges_by_src_node=True, + partitioner_class=DistPartitioner, + expected_pb_dtype=torch.uint8, + ), + param( + "Homogeneous packed-edge-only range partitioning", + is_heterogeneous=False, + input_data_strategy=InputDataStrategy.REGISTER_EDGE_QUANTIZED_FEATURES_WITHOUT_EDGE_FEATURES, + should_assign_edges_by_src_node=True, + partitioner_class=DistRangePartitioner, + expected_pb_dtype=torch.int64, + ), ] ) def test_partitioning_correctness( @@ -756,6 +772,11 @@ def test_partitioning_correctness( else: expected_edge_feat_types = [USER_TO_USER_EDGE_TYPE] + is_packed_edge_only = ( + input_data_strategy + == InputDataStrategy.REGISTER_EDGE_QUANTIZED_FEATURES_WITHOUT_EDGE_FEATURES + ) + for rank, partition_output in output_dict.items(): partitioned_edge_index = partition_output.partitioned_edge_index assert partitioned_edge_index is not None @@ -780,7 +801,32 @@ def test_partitioning_correctness( graph.edge_index ) - if ( + if is_packed_edge_only: + self.assertIsNotNone(partition_output.edge_partition_book) + self.assertIsNone(partition_output.partitioned_edge_features) + self.assertIsNotNone( + partition_output.partitioned_edge_quantized_features + ) + packed_features = partition_output.partitioned_edge_quantized_features + assert isinstance(packed_features, FeaturePartitionData) + assert isinstance(partitioned_edge_index, GraphPartitionData) + self.assertEqual(packed_features.feats.dtype, torch.uint8) + self.assertEqual( + packed_features.feats.size(0), + partitioned_edge_index.edge_index.size(1), + ) + assert partitioned_edge_index.edge_ids is not None + if packed_features.ids is not None: + self.assert_tensor_equality( + tensor_a=packed_features.ids, + tensor_b=partitioned_edge_index.edge_ids, + ) + for index, edge_id in enumerate(partitioned_edge_index.edge_ids): + self.assert_tensor_equality( + tensor_a=packed_features.feats[index], + tensor_b=edge_id.to(torch.uint8).unsqueeze(0), + ) + elif ( input_data_strategy == InputDataStrategy.REGISTER_MINIMAL_ENTITIES_SEPARATELY ): diff --git a/tests/unit/distributed/distributed_weighted_sampling_test.py b/tests/unit/distributed/distributed_weighted_sampling_test.py index 9b30acdd8..cb9386b6e 100644 --- a/tests/unit/distributed/distributed_weighted_sampling_test.py +++ b/tests/unit/distributed/distributed_weighted_sampling_test.py @@ -553,6 +553,11 @@ def test_weights_only_no_features_partitioned_correctly(self) -> None: ) assert edge_ids is not None + self.assertIsNotNone( + partition_output.edge_partition_book, + msg=f"Rank {rank}: edge partition book must be retained for weights", + ) + self.assertEqual(weights.shape, edge_ids.shape) expected_weights = edge_ids.float() * 0.1 torch.testing.assert_close( @@ -732,7 +737,7 @@ def test_range_partitioner_homogeneous_weights_partitioned_correctly(self) -> No True, # should_assign_edges_by_src_node self._master_ip_address, master_port, - InputDataStrategy.REGISTER_ALL_ENTITIES_SEPARATELY, + InputDataStrategy.REGISTER_EDGE_WEIGHTS_WITHOUT_EDGE_FEATURES, DistRangePartitioner, rank_to_edge_weights, ), @@ -759,6 +764,11 @@ def test_range_partitioner_homogeneous_weights_partitioned_correctly(self) -> No ) assert edge_ids is not None + self.assertIsNotNone( + partition_output.edge_partition_book, + msg=f"Rank {rank}: edge partition book must be retained for weights", + ) + self.assertEqual( weights.shape, edge_ids.shape, diff --git a/tests/unit/distributed/utils/neighborloader_test.py b/tests/unit/distributed/utils/neighborloader_test.py index 20ad8b710..43884b495 100644 --- a/tests/unit/distributed/utils/neighborloader_test.py +++ b/tests/unit/distributed/utils/neighborloader_test.py @@ -7,6 +7,7 @@ from torch_geometric.typing import EdgeType from gigl.distributed.sampler import ( + EDGE_PACKED_FEATURES_METADATA_KEY, NEGATIVE_LABEL_METADATA_KEY, NODE_PACKED_FEATURES_METADATA_KEY, POSITIVE_LABEL_METADATA_KEY, @@ -16,6 +17,7 @@ extract_edge_type_metadata, extract_metadata, labeled_to_homogeneous, + materialize_quantized_edge_features, materialize_quantized_node_features, patch_fanout_for_sampling, set_missing_features, @@ -24,6 +26,7 @@ strip_non_ppr_edge_types, ) from gigl.types.graph import ( + DEFAULT_HOMOGENEOUS_EDGE_TYPE, FeatureInfo, FeatureQuantizationMetadata, message_passing_to_positive_label, @@ -101,6 +104,113 @@ def test_materialize_quantized_node_features_reconstructs_feature_order( self.assertEqual(set(remaining_metadata), {"request_id"}) self.assert_tensor_equality(remaining_metadata["request_id"], torch.tensor([7])) + def test_materialize_quantized_edge_features_reconstructs_feature_order( + self, + ) -> None: + data = Data(edge_attr=torch.tensor([[10.0, 20.0], [30.0, 40.0]])) + metadata = { + "edge_packed_features": torch.tensor([[48], [144]], dtype=torch.uint8), + "request_id": torch.tensor([7]), + } + + materialized_data, remaining_metadata = materialize_quantized_edge_features( + data=data, + metadata=metadata, + edge_quantization_metadata=FeatureQuantizationMetadata( + bits=2, + feature_dim=4, + quantized_feature_indices=(0, 2), + clip_min=0.0, + clip_max=3.0, + ), + ) + + self.assert_tensor_equality( + materialized_data.edge_attr, + torch.tensor([[0.0, 10.0, 3.0, 20.0], [2.0, 30.0, 1.0, 40.0]]), + ) + self.assertEqual(set(remaining_metadata), {"request_id"}) + + def test_materialize_quantized_edge_features_uses_labeled_homogeneous_key( + self, + ) -> None: + data = Data(edge_index=torch.tensor([[0], [1]])) + typed_key = ( + f"{EDGE_PACKED_FEATURES_METADATA_KEY}.{DEFAULT_HOMOGENEOUS_EDGE_TYPE}" + ) + + materialized_data, remaining_metadata = materialize_quantized_edge_features( + data=data, + metadata={typed_key: torch.tensor([[48]], dtype=torch.uint8)}, + edge_quantization_metadata=FeatureQuantizationMetadata( + bits=2, + feature_dim=2, + quantized_feature_indices=(0, 1), + clip_min=0.0, + clip_max=3.0, + ), + ) + + self.assert_tensor_equality( + materialized_data.edge_attr, torch.tensor([[0.0, 3.0]]) + ) + self.assertEqual(remaining_metadata, {}) + + def test_materialize_quantized_edge_features_uses_effective_edge_type( + self, + ) -> None: + edge_type = ("item", "rev_to", "user") + data = HeteroData() + data[edge_type].edge_attr = torch.tensor([[10.0]]) + metadata = { + f"{EDGE_PACKED_FEATURES_METADATA_KEY}.{edge_type}": torch.tensor( + [[48]], dtype=torch.uint8 + ) + } + quantization_metadata = FeatureQuantizationMetadata( + bits=2, + feature_dim=3, + quantized_feature_indices=(0, 2), + clip_min=0.0, + clip_max=3.0, + ) + + materialized_data, remaining_metadata = materialize_quantized_edge_features( + data=data, + metadata=metadata, + edge_quantization_metadata={edge_type: quantization_metadata}, + ) + + self.assert_tensor_equality( + materialized_data[edge_type].edge_attr, + torch.tensor([[0.0, 10.0, 3.0]]), + ) + self.assertEqual(remaining_metadata, {}) + + def test_materialize_quantized_edge_features_rejects_missing_packed_features_for_sampled_edge_type( + self, + ) -> None: + data = HeteroData() + data[_U2I_EDGE_TYPE].edge_index = torch.tensor([[0], [1]]) + data[_U2I_EDGE_TYPE].edge_attr = torch.tensor([[10.0]]) + + with self.assertRaisesRegex( + ValueError, "Missing packed quantized edge features" + ): + materialize_quantized_edge_features( + data=data, + metadata={}, + edge_quantization_metadata={ + _U2I_EDGE_TYPE: FeatureQuantizationMetadata( + bits=2, + feature_dim=3, + quantized_feature_indices=(0, 2), + clip_min=0.0, + clip_max=3.0, + ) + }, + ) + def test_materialize_quantized_node_features_uses_per_node_type_metadata( self, ) -> None: