Match TimeStamp and Etag precision and value (Azure#1661)

edwin-huber · web-flow · commit 4af54f0017b6 · 2022-09-08T03:11:11.000Z
* changed table etag and last mod time handling phase 1

* Updated table API to match timestamp and etag
diff --git a/ChangeLog.md b/ChangeLog.md
@@ -8,6 +8,10 @@ General:
 
 - Make emulator start commands async so that they can be awaited by clients.
 
+Table:
+
+- TimeStamp and Etag use the same high precision value as source.
+
 ## 2022.09 Version 3.19.0
 
 General:
diff --git a/src/common/utils/utils.ts b/src/common/utils/utils.ts
@@ -98,7 +98,7 @@ export function truncatedISO8061Date(
   if (hrtimePrecision) {
     return (
       dateString.substring(0, dateString.length - 1) +
-      process.hrtime()[1].toString().padStart(4, "0").slice(0, 3) +
+      process.hrtime()[1].toString().padStart(4, "0").slice(0, 4) +
       "Z"
     );
   }
diff --git a/src/table/entity/NormalizedEntity.ts b/src/table/entity/NormalizedEntity.ts
@@ -1,6 +1,6 @@
+import { truncatedISO8061Date } from "../../common/utils/utils";
 import { Entity } from "../persistence/ITableMetadataStore";
 import { ODATA_TYPE } from "../utils/constants";
-import { getTimestampString } from "../utils/utils";
 import { EdmString } from "./EdmString";
 import { EntityProperty, parseEntityProperty } from "./EntityProperty";
 // import { EdmType } from "./IEdmType";
@@ -34,11 +34,11 @@ export class NormalizedEntity {
     this.propertiesMap.RowKey = rowKeyProperty;
 
     // Sync Timestamp from entity last modified time
-    entity.properties.Timestamp = getTimestampString(
-      typeof entity.lastModifiedTime === "string"
-        ? new Date(entity.lastModifiedTime)
-        : entity.lastModifiedTime
-    );
+    entity.properties.Timestamp =
+      typeof entity.lastModifiedTime === "string" &&
+      entity.lastModifiedTime === ""
+        ? truncatedISO8061Date(new Date(), true, true)
+        : entity.lastModifiedTime;
     entity.properties["Timestamp@odata.type"] = "Edm.DateTime";
 
     for (const key in entity.properties) {
diff --git a/src/table/handlers/TableHandler.ts b/src/table/handlers/TableHandler.ts
@@ -2,7 +2,7 @@ import toReadableStream from "to-readable-stream";
 
 import BufferStream from "../../common/utils/BufferStream";
 import {
-  checkEtagIsInvalidFormat,
+  isEtagValid,
   getUTF8ByteSize,
   newTableEntityEtag
 } from "../utils/utils";
@@ -40,6 +40,7 @@ import {
 } from "../utils/utils";
 import BaseHandler from "./BaseHandler";
 import { EdmType, getEdmType } from "../entity/IEdmType";
+import { truncatedISO8061Date } from "../../common/utils/utils";
 
 interface IPartialResponsePreferProperties {
   statusCode: 200 | 201 | 204;
@@ -198,18 +199,17 @@ export default class TableHandler extends BaseHandler implements ITableHandler {
       throw StorageErrorFactory.getPropertiesNeedValue(context);
     }
 
-    this.checkProperties(context, options.tableEntityProperties);
-
-    const entity: Entity = {
-      PartitionKey: options.tableEntityProperties.PartitionKey,
-      RowKey: options.tableEntityProperties.RowKey,
-      properties: options.tableEntityProperties,
-      lastModifiedTime: context.startTime!,
-      eTag: newTableEntityEtag(context.startTime!)
-    };
     // check that key properties are valid
-    this.validateKey(context, entity.PartitionKey);
-    this.validateKey(context, entity.RowKey);
+    this.validateKey(context, options.tableEntityProperties.PartitionKey);
+    this.validateKey(context, options.tableEntityProperties.RowKey);
+
+    this.checkProperties(context, options.tableEntityProperties);
+    const entity: Entity = this.createPersistedEntity(
+      context,
+      options,
+      options.tableEntityProperties.PartitionKey,
+      options.tableEntityProperties.RowKey
+    );
     let normalizedEntity;
     try {
       normalizedEntity = new NormalizedEntity(entity);
@@ -284,6 +284,31 @@ export default class TableHandler extends BaseHandler implements ITableHandler {
     return response;
   }
 
+  private createPersistedEntity(
+    context: Context,
+    options:
+      | Models.TableMergeEntityOptionalParams
+      | Models.TableInsertEntityOptionalParams
+      | Models.TableUpdateEntityOptionalParams,
+    partitionKey: string,
+    rowKey: string
+  ) {
+    const modTime = truncatedISO8061Date(context.startTime!, true, true);
+    const eTag = newTableEntityEtag(modTime);
+
+    const entity: Entity = {
+      PartitionKey: partitionKey,
+      RowKey: rowKey,
+      properties:
+        options.tableEntityProperties === undefined
+          ? {}
+          : options.tableEntityProperties,
+      lastModifiedTime: modTime,
+      eTag
+    };
+    return entity;
+  }
+
   private static getAndCheck(
     key: string | undefined,
     getFromContext: () => string,
@@ -367,25 +392,21 @@ export default class TableHandler extends BaseHandler implements ITableHandler {
       throw StorageErrorFactory.getPreconditionFailed(context);
     }
     if (options?.ifMatch && options.ifMatch !== "*") {
-      if (checkEtagIsInvalidFormat(options.ifMatch)) {
+      if (isEtagValid(options.ifMatch)) {
         throw StorageErrorFactory.getInvalidOperation(context);
       }
     }
+    // check that key properties are valid
+    this.validateKey(context, partitionKey);
+    this.validateKey(context, rowKey);
 
-    const eTag = newTableEntityEtag(context.startTime!);
-
-    // Entity, which is used to update an existing entity
-    const entity: Entity = {
-      PartitionKey: partitionKey,
-      RowKey: rowKey,
-      properties: options.tableEntityProperties,
-      lastModifiedTime: context.startTime!,
-      eTag
-    };
+    const entity: Entity = this.createPersistedEntity(
+      context,
+      options,
+      partitionKey,
+      rowKey
+    );
 
-    // check that key properties are valid
-    this.validateKey(context, entity.PartitionKey);
-    this.validateKey(context, entity.RowKey);
     let normalizedEntity;
     try {
       normalizedEntity = new NormalizedEntity(entity);
@@ -413,7 +434,7 @@ export default class TableHandler extends BaseHandler implements ITableHandler {
       requestId: tableContext.contextID,
       version: TABLE_API_VERSION,
       date: context.startTime,
-      eTag,
+      eTag: entity.eTag,
       statusCode: 204
     };
 
@@ -443,7 +464,7 @@ export default class TableHandler extends BaseHandler implements ITableHandler {
       throw StorageErrorFactory.getPropertiesNeedValue(context);
     }
     if (options?.ifMatch && options.ifMatch !== "*" && options.ifMatch !== "") {
-      if (checkEtagIsInvalidFormat(options.ifMatch)) {
+      if (isEtagValid(options.ifMatch)) {
         throw StorageErrorFactory.getInvalidOperation(context);
       }
     }
@@ -456,20 +477,17 @@ export default class TableHandler extends BaseHandler implements ITableHandler {
         `TableHandler:mergeEntity() Incoming PartitionKey:${partitionKey} RowKey:${rowKey} in URL parameters don't align with entity body PartitionKey:${options.tableEntityProperties.PartitionKey} RowKey:${options.tableEntityProperties.RowKey}.`
       );
     }
+    // check that key properties are valid
+    this.validateKey(context, partitionKey);
+    this.validateKey(context, rowKey);
 
     this.checkProperties(context, options.tableEntityProperties);
-    const eTag = newTableEntityEtag(context.startTime!);
-
-    const entity: Entity = {
-      PartitionKey: partitionKey,
-      RowKey: rowKey,
-      properties: options.tableEntityProperties,
-      lastModifiedTime: context.startTime!,
-      eTag
-    };
-    // check that key properties are valid
-    this.validateKey(context, entity.PartitionKey);
-    this.validateKey(context, entity.RowKey);
+    const entity: Entity = this.createPersistedEntity(
+      context,
+      options,
+      partitionKey,
+      rowKey
+    );
     let normalizedEntity;
     try {
       normalizedEntity = new NormalizedEntity(entity);
@@ -497,7 +515,7 @@ export default class TableHandler extends BaseHandler implements ITableHandler {
       version: TABLE_API_VERSION,
       date: context.startTime,
       statusCode: 204,
-      eTag
+      eTag: entity.eTag
     };
 
     return response;
@@ -524,7 +542,7 @@ export default class TableHandler extends BaseHandler implements ITableHandler {
     if (ifMatch === "" || ifMatch === undefined) {
       throw StorageErrorFactory.getPreconditionFailed(context);
     }
-    if (ifMatch !== "*" && checkEtagIsInvalidFormat(ifMatch)) {
+    if (ifMatch !== "*" && isEtagValid(ifMatch)) {
       throw StorageErrorFactory.getInvalidOperation(context);
     }
     // currently the props are not coming through as args, so we take them from the table context
@@ -1009,7 +1027,7 @@ export default class TableHandler extends BaseHandler implements ITableHandler {
     // key is a string value that may be up to 1 KiB in size.
     // although a little arbitrary, for performance and
     // generally a better idea, choosing a shorter length
-    if (key.length > DEFAULT_KEY_MAX_LENGTH) {
+    if (key !== undefined && key.length > DEFAULT_KEY_MAX_LENGTH) {
       throw StorageErrorFactory.getInvalidInput(context);
     }
     const match = key.match(/[\u0000-\u001f\u007f-\u009f\/\\\#\?]+/);
@@ -1037,14 +1055,17 @@ export default class TableHandler extends BaseHandler implements ITableHandler {
   ) {
     for (const prop in properties) {
       if (properties.hasOwnProperty(prop)) {
-        if (null !== properties[prop] && undefined !== properties[prop].length) {
+        if (
+          null !== properties[prop] &&
+          undefined !== properties[prop].length
+        ) {
           const typeKey = `${prop}${ODATA_TYPE}`;
           let type;
           if (properties[typeKey]) {
-            type = getEdmType(properties[typeKey])
+            type = getEdmType(properties[typeKey]);
           }
           if (type === EdmType.Binary) {
-            if (Buffer.from(properties[prop], 'base64').length > 64 * 1024) {
+            if (Buffer.from(properties[prop], "base64").length > 64 * 1024) {
               throw StorageErrorFactory.getPropertyValueTooLargeError(context);
             }
           } else if (properties[prop].length > 32 * 1024) {
diff --git a/src/table/persistence/ITableMetadataStore.ts b/src/table/persistence/ITableMetadataStore.ts
@@ -37,7 +37,7 @@ export interface IEntity {
   PartitionKey: string;
   RowKey: string;
   eTag: string;
-  lastModifiedTime: Date;
+  lastModifiedTime: string;
   properties: {
     [propertyName: string]: string | number | boolean | null;
   };
diff --git a/src/table/persistence/LokiTableMetadataStore.ts b/src/table/persistence/LokiTableMetadataStore.ts
@@ -7,7 +7,6 @@ import * as Models from "../generated/artifacts/models";
 import Context from "../generated/Context";
 import { Entity, Table } from "../persistence/ITableMetadataStore";
 import { ODATA_TYPE, QUERY_RESULT_MAX_NUM } from "../utils/constants";
-import { getTimestampString } from "../utils/utils";
 import ITableMetadataStore, { TableACL } from "./ITableMetadataStore";
 
 /** MODELS FOR SERVICE */
@@ -303,7 +302,7 @@ export default class LokiTableMetadataStore implements ITableMetadataStore {
       throw StorageErrorFactory.getEntityAlreadyExist(context);
     }
 
-    entity.properties.Timestamp = getTimestampString(entity.lastModifiedTime);
+    entity.properties.Timestamp = entity.lastModifiedTime;
     entity.properties["Timestamp@odata.type"] = "Edm.DateTime";
 
     if (batchId !== "" && batchId !== undefined) {
@@ -650,7 +649,7 @@ export default class LokiTableMetadataStore implements ITableMetadataStore {
     ) {
       tableEntityCollection.remove(doc);
 
-      entity.properties.Timestamp = getTimestampString(entity.lastModifiedTime);
+      entity.properties.Timestamp = entity.lastModifiedTime;
       entity.properties["Timestamp@odata.type"] = "Edm.DateTime";
 
       tableEntityCollection.insert(entity);
diff --git a/src/table/utils/utils.ts b/src/table/utils/utils.ts
@@ -272,14 +272,14 @@ export function validateTableName(context: Context, tableName: string) {
   }
 }
 
-export function newTableEntityEtag(startTime: Date): string {
+export function newTableEntityEtag(highPresModTime: string): string {
   // Etag as returned by Table Storage should match W/"datetime'<ISO8601datetime>'"
-  // we use the additional hrtime precsion option
-  return (
-    "W/\"datetime'" +
-    encodeURIComponent(truncatedISO8061Date(startTime, true, true)) +
-    "'\""
-  );
+  // as we need the same value for last Modification time, we now only accept a string here
+  return "W/\"datetime'" + encodeURIComponent(highPresModTime) + "'\"";
+}
+
+export function newHighPrecisionTimeStamp(startTime: Date): string {
+  return truncatedISO8061Date(startTime, true, true);
 }
 
 /**
@@ -289,7 +289,7 @@ export function newTableEntityEtag(startTime: Date): string {
  * @param {string} etag
  * @return {*}  {boolean}
  */
-export function checkEtagIsInvalidFormat(etag: string): boolean {
+export function isEtagValid(etag: string): boolean {
   // Weak etag is required. This is parity with Azure and legacy emulator.
   // Source for regex: https://stackoverflow.com/a/11572348
   const match = etag.match(/^[wW]\/"([^"]|\\")*"$/);
diff --git a/tests/table/apis/table.entity.azure.data-tables.test.ts b/tests/table/apis/table.entity.azure.data-tables.test.ts
@@ -1123,7 +1123,7 @@ describe("table Entity APIs test - using Azure/data-tables", () => {
     await tableClient.deleteTable();
   });
 
-  [2, 1, 0].map(delta => {
+  [2, 1, 0].map((delta) => {
     it(`Should insert entities containing binary properties less than or equal than 64K bytes (delta ${delta}), @loki`, async () => {
       const tableClient = createAzureDataTablesClient(
         testLocalAzuriteInstance,
@@ -1134,20 +1134,16 @@ describe("table Entity APIs test - using Azure/data-tables", () => {
       const testEntity: AzureDataTablesTestEntity =
         createBasicEntityForTest(partitionKey);
 
-      testEntity.binaryField = Buffer.alloc((64 * 1024) - delta);
+      testEntity.binaryField = Buffer.alloc(64 * 1024 - delta);
 
       const result = await tableClient.createEntity(testEntity);
-      assert.notStrictEqual(
-        result.etag,
-        undefined,
-        "Did not create entity!"
-      );
+      assert.notStrictEqual(result.etag, undefined, "Did not create entity!");
 
       await tableClient.deleteTable();
     });
   });
 
-  [1, 2, 3].map(delta => {
+  [1, 2, 3].map((delta) => {
     it(`Should not insert entities containing binary properties greater than 64K bytes (delta ${delta}), @loki`, async () => {
       const tableClient = createAzureDataTablesClient(
         testLocalAzuriteInstance,
@@ -1158,7 +1154,7 @@ describe("table Entity APIs test - using Azure/data-tables", () => {
       const testEntity: AzureDataTablesTestEntity =
         createBasicEntityForTest(partitionKey);
 
-      testEntity.binaryField = Buffer.alloc((64 * 1024) + delta);
+      testEntity.binaryField = Buffer.alloc(64 * 1024 + delta);
       try {
         const result = await tableClient.createEntity(testEntity);
         assert.strictEqual(
@@ -1413,8 +1409,8 @@ describe("table Entity APIs test - using Azure/data-tables", () => {
         a: "a".repeat(32 * 1024),
         b: "b".repeat(32 * 1024),
         c: "c".repeat(32 * 1024),
-        d: "d".repeat(32 * 1024),
-      })
+        d: "d".repeat(32 * 1024)
+      });
     }
 
     try {
@@ -1658,7 +1654,6 @@ describe("table Entity APIs test - using Azure/data-tables", () => {
     await tableClient.deleteTable();
   });
 
-
   it("Should insert entities with null properties, @loki", async () => {
     const tableClient = createAzureDataTablesClient(
       testLocalAzuriteInstance,
diff --git a/tests/table/apis/table.entity.rest.test.ts b/tests/table/apis/table.entity.rest.test.ts
diff --git a/tests/table/apis/table.entity.test.ts b/tests/table/apis/table.entity.test.ts

Original file line number	Diff line number	Diff line change
`@@ -98,7 +98,7 @@ export function truncatedISO8061Date(`
`98`	`98`	`if (hrtimePrecision) {`
`99`	`99`	`return (`
`100`	`100`	`dateString.substring(0, dateString.length - 1) +`
`101`		`- process.hrtime()[1].toString().padStart(4, "0").slice(0, 3) +`
	`101`	`+ process.hrtime()[1].toString().padStart(4, "0").slice(0, 4) +`
`102`	`102`	`"Z"`
`103`	`103`	`);`
`104`	`104`	`}`