fix: only filter currently under maintenance nodes

distributor-node/src/services/networking/NetworkingService.ts

@@ -402,7 +402,7 @@ export class NetworkingService {
 
   async checkActiveStorageNodeEndpoints(): Promise<void> {
     try {
-      const activeStorageOperators = await this.queryNodeApi.getActiveStorageBucketOperatorsData()
+      const activeStorageOperators = await this.queryNodeApi.getOperationallyActiveStorageBucketOperatorsData()
       const endpoints = this.filterStorageNodeEndpoints(
         activeStorageOperators.map(({ id, operatorMetadata }) => ({
           bucketId: id,

distributor-node/src/services/networking/query-node/api.ts

@@ -232,11 +232,30 @@ export class QueryNodeApi {
     return this.getDataObjectsByBagIds(bagIds)
   }
 
-  public getActiveStorageBucketOperatorsData(): Promise<StorageBucketOperatorFieldsFragment[]> {
-    return this.multipleEntitiesQuery<
+  public async getOperationallyActiveStorageBucketOperatorsData(): Promise<StorageBucketOperatorFieldsFragment[]> {
+    const buckets = await this.multipleEntitiesQuery<
       GetActiveStorageBucketOperatorsDataQuery,
       GetActiveStorageBucketOperatorsDataQueryVariables
     >(GetActiveStorageBucketOperatorsData, {}, 'storageBuckets')
+
+    // Filter out nodes/operators under maintenance
+    return buckets.filter(({ operatorMetadata }) => {
+      const status = operatorMetadata?.nodeOperationalStatus
+      const date = new Date()
+      if (
+        !operatorMetadata ||
+        !status ||
+        status.__typename === 'NodeOperationalStatusNormal' ||
+        (status.__typename === 'NodeOperationalStatusNoServiceFrom' && new Date(status.from) > date) || // planned future maintenance (which has not started yet)
+        (status.__typename === 'NodeOperationalStatusNoServiceUntil' &&
+          new Date(status.from) > date &&
+          new Date(status.until) < date) // planned future maintenance with end time (which has not started yet)
+      ) {
+        return true
+      }
+
+      return false
+    })
   }
 
   public async getPackageVersion(): Promise<string> {

distributor-node/src/services/networking/query-node/queries/queries.graphql

@@ -103,21 +103,31 @@ fragment StorageBucketOperatorFields on StorageBucket {
   id
   operatorMetadata {
     nodeEndpoint
+    nodeOperationalStatus {
+      ... on NodeOperationalStatusNormal {
+        __typename
+      }
+      ... on NodeOperationalStatusNoService {
+        __typename
+        forced
+      }
+      ... on NodeOperationalStatusNoServiceFrom {
+        __typename
+        forced
+        from
+      }
+      ... on NodeOperationalStatusNoServiceUntil {
+        __typename
+        forced
+        from
+        until
+      }
+    }
   }
 }
 
 query getActiveStorageBucketOperatorsData {
-  storageBuckets(
-    where: {
-      operatorStatus: { isTypeOf_eq: "StorageBucketOperatorStatusActive" }
-      operatorMetadata: {
-        OR: [
-          { nodeOperationalStatus_isNull: true }
-          { nodeOperationalStatus: { isTypeOf_eq: "NodeOperationalStatusNormal" } }
-        ]
-      }
-    }
-  ) {
+  storageBuckets(where: { operatorStatus: { isTypeOf_eq: "StorageBucketOperatorStatusActive" } }) {
     ...StorageBucketOperatorFields
   }
 }

distributor-node/src/services/networking/query-node/schema.graphql

@@ -432,10 +432,10 @@ enum DistributionBucketOperatorMetadataOrderByInput {
   nodeOperationalStatus_from_DESC
   nodeOperationalStatus_from_ASC_NULLS_FIRST
   nodeOperationalStatus_from_DESC_NULLS_LAST
-  nodeOperationalStatus_to_ASC
-  nodeOperationalStatus_to_DESC
-  nodeOperationalStatus_to_ASC_NULLS_FIRST
-  nodeOperationalStatus_to_DESC_NULLS_LAST
+  nodeOperationalStatus_until_ASC
+  nodeOperationalStatus_until_DESC
+  nodeOperationalStatus_until_ASC_NULLS_FIRST
+  nodeOperationalStatus_until_DESC_NULLS_LAST
   nodeOperationalStatus_isTypeOf_ASC
   nodeOperationalStatus_isTypeOf_DESC
   nodeOperationalStatus_isTypeOf_ASC_NULLS_FIRST
@@ -678,7 +678,7 @@ input DistributionBucketWhereInput {
 
 type DistributionNodeOperationalStatusSetEvent {
   """Distribution bucket operator"""
-  bucketOperator: DistributionBucketOperator
+  bucketOperator: DistributionBucketOperator!
 
   """Operational status that was set"""
   operationalStatus: NodeOperationalStatus!
@@ -1014,7 +1014,7 @@ input NodeLocationMetadataWhereInput {
   coordinates: GeoCoordinatesWhereInput
 }
 
-union NodeOperationalStatus = NodeOperationalStatusNormal | NodeOperationalStatusNoService | NodeOperationalStatusNoServiceFrom | NodeOperationalStatusNoServiceDuring
+union NodeOperationalStatus = NodeOperationalStatusNormal | NodeOperationalStatusNoService | NodeOperationalStatusNoServiceFrom | NodeOperationalStatusNoServiceUntil
 
 type NodeOperationalStatusNormal {
   """Reason why node was set to this state"""
@@ -1032,7 +1032,7 @@ type NodeOperationalStatusNoService {
   rationale: String
 }
 
-type NodeOperationalStatusNoServiceDuring {
+type NodeOperationalStatusNoServiceFrom {
   """
   Whether the state was set by lead (true) or by the operator (false), it is
   meant to prevent worker from unilaterally reversing.
@@ -1042,14 +1042,11 @@ type NodeOperationalStatusNoServiceDuring {
   """The time from which the bucket would have to no service"""
   from: DateTime!
 
-  """The time until which the bucket would have to no service"""
-  to: DateTime!
-
   """Reason why node was set to this state"""
   rationale: String
 }
 
-type NodeOperationalStatusNoServiceFrom {
+type NodeOperationalStatusNoServiceUntil {
   """
   Whether the state was set by lead (true) or by the operator (false), it is
   meant to prevent worker from unilaterally reversing.
@@ -1059,6 +1056,9 @@ type NodeOperationalStatusNoServiceFrom {
   """The time from which the bucket would have to no service"""
   from: DateTime!
 
+  """The time until which the bucket would have to no service"""
+  until: DateTime!
+
   """Reason why node was set to this state"""
   rationale: String
 }
@@ -1093,15 +1093,15 @@ input NodeOperationalStatusWhereInput {
   from_lte: DateTime
   from_in: [DateTime!]
   from_not_in: [DateTime!]
-  to_isNull: Boolean
-  to_eq: DateTime
-  to_not_eq: DateTime
-  to_gt: DateTime
-  to_gte: DateTime
-  to_lt: DateTime
-  to_lte: DateTime
-  to_in: [DateTime!]
-  to_not_in: [DateTime!]
+  until_isNull: Boolean
+  until_eq: DateTime
+  until_not_eq: DateTime
+  until_gt: DateTime
+  until_gte: DateTime
+  until_lt: DateTime
+  until_lte: DateTime
+  until_in: [DateTime!]
+  until_not_in: [DateTime!]
   isTypeOf_isNull: Boolean
   isTypeOf_eq: String
   isTypeOf_not_eq: String
@@ -1181,10 +1181,6 @@ type Query {
   distributionBucketFamilyById(id: String!): DistributionBucketFamily
   distributionBucketFamilyByUniqueInput(where: WhereIdInput!): DistributionBucketFamily @deprecated(reason: "Use distributionBucketFamilyById")
   distributionBucketFamiliesConnection(orderBy: [DistributionBucketFamilyOrderByInput!]!, after: String, first: Int, where: DistributionBucketFamilyWhereInput): DistributionBucketFamiliesConnection!
-  workers(where: WorkerWhereInput, orderBy: [WorkerOrderByInput!], offset: Int, limit: Int): [Worker!]!
-  workerById(id: String!): Worker
-  workerByUniqueInput(where: WhereIdInput!): Worker @deprecated(reason: "Use workerById")
-  workersConnection(orderBy: [WorkerOrderByInput!]!, after: String, first: Int, where: WorkerWhereInput): WorkersConnection!
   squidStatus: SquidStatus
   squidVersion: SquidVersion!
 }
@@ -1597,10 +1593,10 @@ enum StorageBucketOperatorMetadataOrderByInput {
   nodeOperationalStatus_from_DESC
   nodeOperationalStatus_from_ASC_NULLS_FIRST
   nodeOperationalStatus_from_DESC_NULLS_LAST
-  nodeOperationalStatus_to_ASC
-  nodeOperationalStatus_to_DESC
-  nodeOperationalStatus_to_ASC_NULLS_FIRST
-  nodeOperationalStatus_to_DESC_NULLS_LAST
+  nodeOperationalStatus_until_ASC
+  nodeOperationalStatus_until_DESC
+  nodeOperationalStatus_until_ASC_NULLS_FIRST
+  nodeOperationalStatus_until_DESC_NULLS_LAST
   nodeOperationalStatus_isTypeOf_ASC
   nodeOperationalStatus_isTypeOf_DESC
   nodeOperationalStatus_isTypeOf_ASC_NULLS_FIRST
@@ -2149,63 +2145,3 @@ input WhereIdInput {
   id: String!
 }
 
-type Worker {
-  """Worker id ({workingGroupName}-{workerId})"""
-  id: String!
-
-  """WorkerId in specific working group module"""
-  runtimeId: BigInt!
-}
-
-type WorkerEdge {
-  node: Worker!
-  cursor: String!
-}
-
-enum WorkerOrderByInput {
-  id_ASC
-  id_DESC
-  id_ASC_NULLS_FIRST
-  id_DESC_NULLS_LAST
-  runtimeId_ASC
-  runtimeId_DESC
-  runtimeId_ASC_NULLS_FIRST
-  runtimeId_DESC_NULLS_LAST
-}
-
-type WorkersConnection {
-  edges: [WorkerEdge!]!
-  pageInfo: PageInfo!
-  totalCount: Int!
-}
-
-input WorkerWhereInput {
-  id_isNull: Boolean
-  id_eq: String
-  id_not_eq: String
-  id_gt: String
-  id_gte: String
-  id_lt: String
-  id_lte: String
-  id_in: [String!]
-  id_not_in: [String!]
-  id_contains: String
-  id_not_contains: String
-  id_containsInsensitive: String
-  id_not_containsInsensitive: String
-  id_startsWith: String
-  id_not_startsWith: String
-  id_endsWith: String
-  id_not_endsWith: String
-  runtimeId_isNull: Boolean
-  runtimeId_eq: BigInt
-  runtimeId_not_eq: BigInt
-  runtimeId_gt: BigInt
-  runtimeId_gte: BigInt
-  runtimeId_lt: BigInt
-  runtimeId_lte: BigInt
-  runtimeId_in: [BigInt!]
-  runtimeId_not_in: [BigInt!]
-  AND: [WorkerWhereInput!]
-  OR: [WorkerWhereInput!]
-}

storage-node/src/services/queryNode/api.ts

@@ -182,6 +182,29 @@ export class QueryNodeApi {
     return result.data[resultKey]
   }
 
+  private getOperationallyActiveStorageBuckets(
+    buckets: Array<StorageBucketIdsFragment>
+  ): Array<StorageBucketIdsFragment> {
+    // Filter out nodes/operators under maintenance
+    return buckets.filter(({ operatorMetadata }) => {
+      const status = operatorMetadata?.nodeOperationalStatus
+      const date = new Date()
+      if (
+        !operatorMetadata ||
+        !status ||
+        status.__typename === 'NodeOperationalStatusNormal' ||
+        (status.__typename === 'NodeOperationalStatusNoServiceFrom' && new Date(status.from) > date) || // planned future maintenance (which has not started yet)
+        (status.__typename === 'NodeOperationalStatusNoServiceUntil' &&
+          new Date(status.from) > date &&
+          new Date(status.until) < date) // planned future maintenance with end time (which has not started yet)
+      ) {
+        return true
+      }
+
+      return false
+    })
+  }
+
   /**
    * Returns storage bucket IDs filtered by worker ID.
    *
@@ -197,7 +220,7 @@ export class QueryNodeApi {
       return []
     }
 
-    return result
+    return this.getOperationallyActiveStorageBuckets(result)
   }
 
   /**
@@ -301,7 +324,7 @@ export class QueryNodeApi {
       return []
     }
 
-    return result
+    return this.getOperationallyActiveStorageBuckets(result)
   }
 
   /**

storage-node/src/services/queryNode/queries/queries.graphql

@@ -1,34 +1,43 @@
+fragment NodeOperationalStatusFragment on NodeOperationalStatus {
+  __typename
+  ... on NodeOperationalStatusNormal {
+    __typename
+  }
+  ... on NodeOperationalStatusNoService {
+    __typename
+    forced
+  }
+  ... on NodeOperationalStatusNoServiceFrom {
+    __typename
+    forced
+    from
+  }
+  ... on NodeOperationalStatusNoServiceUntil {
+    __typename
+    forced
+    from
+    until
+  }
+}
+
 fragment StorageBucketIds on StorageBucket {
   id
+  operatorMetadata {
+    nodeOperationalStatus {
+      ...NodeOperationalStatusFragment
+    }
+  }
 }
 
 query getStorageBuckets {
-  storageBuckets(
-    where: {
-      operatorStatus: { isTypeOf_eq: "StorageBucketOperatorStatusActive" }
-      operatorMetadata: {
-        OR: [
-          { nodeOperationalStatus_isNull: true }
-          { nodeOperationalStatus: { isTypeOf_eq: "NodeOperationalStatusNormal" } }
-        ]
-      }
-    }
-  ) {
+  storageBuckets(where: { operatorStatus: { isTypeOf_eq: "StorageBucketOperatorStatusActive" } }) {
     ...StorageBucketIds
   }
 }
 
 query getStorageBucketsByWorkerId($workerId: BigInt!) {
   storageBuckets(
-    where: {
-      operatorStatus: { isTypeOf_eq: "StorageBucketOperatorStatusActive", workerId_eq: $workerId }
-      operatorMetadata: {
-        OR: [
-          { nodeOperationalStatus_isNull: true }
-          { nodeOperationalStatus: { isTypeOf_eq: "NodeOperationalStatusNormal" } }
-        ]
-      }
-    }
+    where: { operatorStatus: { isTypeOf_eq: "StorageBucketOperatorStatusActive", workerId_eq: $workerId } }
   ) {
     ...StorageBucketIds
   }
@@ -49,10 +58,13 @@ query getStorageBucketsOperationalStatus($ids: [String!]) {
         ... on NodeOperationalStatusNoServiceFrom {
           __typename
           forced
+          from
         }
-        ... on NodeOperationalStatusNoServiceDuring {
+        ... on NodeOperationalStatusNoServiceUntil {
           __typename
           forced
+          from
+          until
         }
       }
     }