fix: track skipped features and vertices to avoid throwing error of missing features or vertices

src/files/shapefile/core/chunkBuilder.ts

@@ -7,11 +7,13 @@ export class ChunkBuilder {
   private features: Feature[];
   private skippedFeatures: Feature[];
   private currentVerticesCount: number;
+  private skippedVerticesCount: number;
 
   public constructor(private readonly maxVertices: number, private chunkIndex: number = 0) {
     this.features = [];
     this.skippedFeatures = [];
     this.currentVerticesCount = 0;
+    this.skippedVerticesCount = 0;
   }
 
   public get chunkId(): number {
@@ -26,6 +28,7 @@ export class ChunkBuilder {
     if (featureVertices > this.maxVertices) {
       const featureWithVertices: Feature = { ...feature, properties: { ...feature.properties, vertices: featureVertices } };
       this.skippedFeatures.push(featureWithVertices);
+      this.skippedVerticesCount += featureVertices;
       return FeatureStatus.SKIPPED;
     }
     const canAdd = this.currentVerticesCount + featureVertices <= this.maxVertices;
@@ -46,8 +49,9 @@ export class ChunkBuilder {
     return {
       id: this.chunkIndex,
       features: this.features,
-      skippedFeatures: this.skippedFeatures,
       verticesCount: this.currentVerticesCount,
+      skippedFeatures: this.skippedFeatures,
+      skippedVerticesCount: this.skippedVerticesCount,
     };
   }
 

src/files/shapefile/core/progressTracker.ts

@@ -120,15 +120,12 @@ export class ProgressTracker implements IProgressTracker {
       estimatedRemainingTimeMs = totalEstimatedTimeMs - elapsedTimeMs;
     }
 
-    const estimatedTotalChunks = this.totalVertices > 0 ? Math.ceil(this.totalVertices / this.maxVerticesPerChunk) : this.processedChunks;
-
     const endTime = this.processedFeatures === this.totalFeatures ? currentTime : undefined;
 
     return {
       processedFeatures: this.processedFeatures,
       totalFeatures: this.totalFeatures,
       processedChunks: this.processedChunks,
-      totalChunks: estimatedTotalChunks,
       processedVertices: this.processedVertices,
       totalVertices: this.totalVertices,
       skippedFeatures: this.skippedFeatures,

src/files/shapefile/core/shapeFileReader.ts

@@ -136,10 +136,8 @@ export class ShapefileChunkReader {
             msg: `Feature exceeds maximum vertices limit: ${vertices} > ${this.options.maxVerticesPerChunk}`,
             featureId: feature.id,
           });
-          continue; // Skip features that exceed the limit
         }
         totalFeatures++;
-
         totalVertices += vertices;
       }
     } catch (error) {
@@ -172,7 +170,9 @@ export class ShapefileChunkReader {
 
     this.metricsManager?.sendChunkMetrics(chunk, readTime, processTime);
 
-    this.progressTracker?.addProcessedFeatures(chunk.features.length, chunk.verticesCount);
+    const chunkTotalFeatures = chunk.features.length + chunk.skippedFeatures.length;
+    const chunkTotalVertices = chunk.verticesCount + chunk.skippedVerticesCount;
+    this.progressTracker?.addProcessedFeatures(chunkTotalFeatures, chunkTotalVertices);
     this.progressTracker?.addSkippedFeatures(chunk.skippedFeatures.length);
     this.progressTracker?.incrementChunks();
     const lastFeatureIndex = (this.progressTracker?.getProcessedFeatures() ?? 0) - 1;

src/files/shapefile/types/index.ts

@@ -9,8 +9,9 @@ export * from './state';
 export interface ShapefileChunk {
   id: number;
   features: Feature[];
-  skippedFeatures: Feature[];
   verticesCount: number;
+  skippedFeatures: Feature[];
+  skippedVerticesCount: number;
 }
 
 export interface ChunkProcessor {

src/files/shapefile/types/state.ts

@@ -9,7 +9,6 @@ export interface ProgressInfo {
   processedFeatures: number;
   totalFeatures: number;
   processedChunks: number;
-  totalChunks: number;
   processedVertices: number;
   totalVertices: number;
   skippedFeatures: number;

tests/unit/files/shapefile/progressTracker.spec.ts

@@ -176,7 +176,6 @@ describe('ProgressTracker', () => {
         processedFeatures: 0,
         totalFeatures: mockTotalFeatures,
         processedChunks: 0,
-        totalChunks: 10, // Math.ceil(1000 / 100)
         processedVertices: 0,
         totalVertices: mockTotalVertices,
         percentage: 0,
@@ -269,12 +268,6 @@ describe('ProgressTracker', () => {
       expect(progress.estimatedRemainingTimeMs).toBe(0);
     });
 
-    it('should calculate total chunks correctly based on vertices and chunk size', () => {
-      const progress = progressTracker.calculateProgress();
-
-      expect(progress.totalChunks).toBe(10); // Math.ceil(1000 / 100)
-    });
-
     it('should handle edge case when total vertices is zero', () => {
       progressTracker = new ProgressTracker({
         totalVertices: 0,
@@ -285,7 +278,6 @@ describe('ProgressTracker', () => {
       const progress = progressTracker.calculateProgress();
 
       expect(progress.percentage).toBe(0);
-      expect(progress.totalChunks).toBe(0);
     });
 
     it('should set endTime when all features are processed', () => {
@@ -305,20 +297,6 @@ describe('ProgressTracker', () => {
 
       expect(progress.endTime).toBeUndefined();
     });
-
-    it('should use processed chunks as total when total vertices is zero', () => {
-      progressTracker = new ProgressTracker({
-        totalVertices: 0,
-        totalFeatures: mockTotalFeatures,
-        maxVerticesPerChunk: mockMaxVerticesPerChunk,
-      });
-      progressTracker.incrementChunks();
-      progressTracker.incrementChunks();
-
-      const progress = progressTracker.calculateProgress();
-
-      expect(progress.totalChunks).toBe(2); // Uses processedChunks when totalVertices is 0
-    });
   });
 
   describe('interface compliance', () => {

tests/unit/files/shapefile/shapeFileReader.spec.ts

@@ -14,7 +14,7 @@ import * as vertices from '../../../../src/geo/vertices';
 const shapefilePath = '/path/to/shapefile.shp';
 const dbfFilePath = shapefilePath.replace(/\.shp$/i, '.dbf');
 
-const mockRandomUUID = jest.fn<string, never>();
+const mockRandomUUID = jest.fn<string, []>();
 
 // Mock all dependencies
 jest.mock('node:crypto', () => {
@@ -130,8 +130,9 @@ describe('ShapefileChunkReader', () => {
       mockChunkBuilder.build.mockReturnValue({
         id: 0,
         features: [mockFeature, mockFeature],
-        skippedFeatures: [],
         verticesCount: 100,
+        skippedFeatures: [],
+        skippedVerticesCount: 0,
       });
 
       await reader.readAndProcess(shapefilePath, { process: mockProcessor });
@@ -154,8 +155,14 @@ describe('ShapefileChunkReader', () => {
         .mockReturnValueOnce(FeatureStatus.FULL)
         .mockReturnValueOnce(FeatureStatus.ADD);
 
-      const chunk1: ShapefileChunk = { id: 0, features: [mockFeature], skippedFeatures: [], verticesCount: 50 };
-      const chunk2: ShapefileChunk = { id: 1, features: [mockFeature, mockFeature], skippedFeatures: [], verticesCount: 100 };
+      const chunk1: ShapefileChunk = { id: 0, features: [mockFeature], skippedFeatures: [], verticesCount: 50, skippedVerticesCount: 0 };
+      const chunk2: ShapefileChunk = {
+        id: 1,
+        features: [mockFeature, mockFeature],
+        skippedFeatures: [],
+        verticesCount: 100,
+        skippedVerticesCount: 0,
+      };
       mockChunkBuilder.build.mockReturnValueOnce(chunk1).mockReturnValueOnce(chunk2);
 
       await reader.readAndProcess(shapefilePath, { process: mockProcessor });
@@ -191,6 +198,7 @@ describe('ShapefileChunkReader', () => {
         features: [mockFeature],
         verticesCount: 50,
         skippedFeatures: [],
+        skippedVerticesCount: 0,
       });
 
       await reader.readAndProcess(shapefilePath, { process: mockProcessor });
@@ -222,8 +230,9 @@ describe('ShapefileChunkReader', () => {
       const chunk: ShapefileChunk = {
         id: 0,
         features: [],
-        skippedFeatures: [largeFeature],
         verticesCount: 0,
+        skippedFeatures: [largeFeature],
+        skippedVerticesCount: 8,
       };
 
       mockSource.read.mockResolvedValueOnce({ done: false, value: largeFeature }).mockResolvedValueOnce({ done: true, value: largeFeature });
@@ -245,6 +254,7 @@ describe('ShapefileChunkReader', () => {
         features: [mockFeature],
         verticesCount: 50,
         skippedFeatures: [],
+        skippedVerticesCount: 0,
       });
 
       Object.defineProperty(mockChunkBuilder, 'chunkId', { value: 0, writable: true });
@@ -265,7 +275,7 @@ describe('ShapefileChunkReader', () => {
       mockSource.read.mockResolvedValueOnce({ done: false, value: mockFeature }).mockResolvedValueOnce({ done: true, value: mockFeature });
 
       mockChunkBuilder.canAddFeature.mockReturnValue(FeatureStatus.ADD);
-      const chunk = { id: 0, features: [mockFeature], skippedFeatures: [], verticesCount: 50 } as ShapefileChunk;
+      const chunk = { id: 0, features: [mockFeature], verticesCount: 50, skippedFeatures: [], skippedVerticesCount: 0 } as ShapefileChunk;
       mockChunkBuilder.build.mockReturnValue(chunk);
 
       await reader.readAndProcess(shapefilePath, { process: mockProcessor });
@@ -299,14 +309,16 @@ describe('ShapefileChunkReader', () => {
       const chunk: ShapefileChunk = {
         id: 0,
         features: [],
-        skippedFeatures: [largeFeature],
         verticesCount: 0,
+        skippedFeatures: [largeFeature],
+        skippedVerticesCount: 8,
       };
       const finalChunk: ShapefileChunk = {
         id: 1,
         features: [],
         skippedFeatures: [],
         verticesCount: 0,
+        skippedVerticesCount: 0,
       };
       mockSource.read.mockResolvedValueOnce({ done: false, value: largeFeature }).mockResolvedValueOnce({ done: true, value: largeFeature });
       mockChunkBuilder.canAddFeature.mockReturnValue(FeatureStatus.SKIPPED);
@@ -362,8 +374,8 @@ describe('ShapefileChunkReader', () => {
       // Second feature: exceeds max vertices, should be SKIPPED
       mockChunkBuilder.canAddFeature.mockReturnValueOnce(FeatureStatus.FULL).mockReturnValueOnce(FeatureStatus.SKIPPED);
 
-      const chunk1: ShapefileChunk = { id: 0, features: [normalFeature], skippedFeatures: [], verticesCount: 50 };
-      const chunk2: ShapefileChunk = { id: 1, features: [], skippedFeatures: [largeFeature], verticesCount: 0 };
+      const chunk1: ShapefileChunk = { id: 0, features: [normalFeature], verticesCount: 50, skippedFeatures: [], skippedVerticesCount: 0 };
+      const chunk2: ShapefileChunk = { id: 1, features: [], verticesCount: 0, skippedFeatures: [largeFeature], skippedVerticesCount: 8 };
       mockChunkBuilder.build.mockReturnValueOnce(chunk1).mockReturnValueOnce(chunk2);
 
       await reader.readAndProcess(shapefilePath, { process: mockProcessor });
@@ -399,7 +411,7 @@ describe('ShapefileChunkReader', () => {
       });
     });
 
-    it('should skip features exceeding vertex limit', async () => {
+    it('should include skip features exceeding vertex limit', async () => {
       jest.spyOn(vertices, 'countVertices').mockReturnValueOnce(100).mockReturnValueOnce(2000).mockReturnValueOnce(100);
 
       mockSource.read
@@ -411,8 +423,8 @@ describe('ShapefileChunkReader', () => {
       const stats = await reader.getShapefileStats(shapefilePath);
 
       expect(stats).toEqual({
-        totalVertices: 200, // Only features 1 and 3
-        totalFeatures: 2,
+        totalVertices: 2200,
+        totalFeatures: 3,
       });
     });
 

tests/unit/files/shapefile/utils.ts

@@ -28,7 +28,8 @@ export function createTestChunk(id: number, featuresCount: number, verticesCount
   return {
     id,
     features,
-    skippedFeatures: [],
     verticesCount,
+    skippedFeatures: [],
+    skippedVerticesCount: 0,
   };
 }