diff --git a/src/files/shapefile/core/chunkBuilder.ts b/src/files/shapefile/core/chunkBuilder.ts index 150e77d..c305881 100644 --- a/src/files/shapefile/core/chunkBuilder.ts +++ b/src/files/shapefile/core/chunkBuilder.ts @@ -7,11 +7,13 @@ export class ChunkBuilder { private features: Feature[]; private skippedFeatures: Feature[]; private currentVerticesCount: number; + private skippedVerticesCount: number; public constructor(private readonly maxVertices: number, private chunkIndex: number = 0) { this.features = []; this.skippedFeatures = []; this.currentVerticesCount = 0; + this.skippedVerticesCount = 0; } public get chunkId(): number { @@ -26,6 +28,7 @@ export class ChunkBuilder { if (featureVertices > this.maxVertices) { const featureWithVertices: Feature = { ...feature, properties: { ...feature.properties, vertices: featureVertices } }; this.skippedFeatures.push(featureWithVertices); + this.skippedVerticesCount += featureVertices; return FeatureStatus.SKIPPED; } const canAdd = this.currentVerticesCount + featureVertices <= this.maxVertices; @@ -46,8 +49,9 @@ export class ChunkBuilder { return { id: this.chunkIndex, features: this.features, - skippedFeatures: this.skippedFeatures, verticesCount: this.currentVerticesCount, + skippedFeatures: this.skippedFeatures, + skippedVerticesCount: this.skippedVerticesCount, }; } diff --git a/src/files/shapefile/core/progressTracker.ts b/src/files/shapefile/core/progressTracker.ts index b9fc08b..5dcc86c 100644 --- a/src/files/shapefile/core/progressTracker.ts +++ b/src/files/shapefile/core/progressTracker.ts @@ -120,15 +120,12 @@ export class ProgressTracker implements IProgressTracker { estimatedRemainingTimeMs = totalEstimatedTimeMs - elapsedTimeMs; } - const estimatedTotalChunks = this.totalVertices > 0 ? Math.ceil(this.totalVertices / this.maxVerticesPerChunk) : this.processedChunks; - const endTime = this.processedFeatures === this.totalFeatures ? currentTime : undefined; return { processedFeatures: this.processedFeatures, totalFeatures: this.totalFeatures, processedChunks: this.processedChunks, - totalChunks: estimatedTotalChunks, processedVertices: this.processedVertices, totalVertices: this.totalVertices, skippedFeatures: this.skippedFeatures, diff --git a/src/files/shapefile/core/shapeFileReader.ts b/src/files/shapefile/core/shapeFileReader.ts index 1d1d335..c657f3f 100644 --- a/src/files/shapefile/core/shapeFileReader.ts +++ b/src/files/shapefile/core/shapeFileReader.ts @@ -136,10 +136,8 @@ export class ShapefileChunkReader { msg: `Feature exceeds maximum vertices limit: ${vertices} > ${this.options.maxVerticesPerChunk}`, featureId: feature.id, }); - continue; // Skip features that exceed the limit } totalFeatures++; - totalVertices += vertices; } } catch (error) { @@ -172,7 +170,9 @@ export class ShapefileChunkReader { this.metricsManager?.sendChunkMetrics(chunk, readTime, processTime); - this.progressTracker?.addProcessedFeatures(chunk.features.length, chunk.verticesCount); + const chunkTotalFeatures = chunk.features.length + chunk.skippedFeatures.length; + const chunkTotalVertices = chunk.verticesCount + chunk.skippedVerticesCount; + this.progressTracker?.addProcessedFeatures(chunkTotalFeatures, chunkTotalVertices); this.progressTracker?.addSkippedFeatures(chunk.skippedFeatures.length); this.progressTracker?.incrementChunks(); const lastFeatureIndex = (this.progressTracker?.getProcessedFeatures() ?? 0) - 1; diff --git a/src/files/shapefile/types/index.ts b/src/files/shapefile/types/index.ts index 59333ee..5a0e169 100644 --- a/src/files/shapefile/types/index.ts +++ b/src/files/shapefile/types/index.ts @@ -9,8 +9,9 @@ export * from './state'; export interface ShapefileChunk { id: number; features: Feature[]; - skippedFeatures: Feature[]; verticesCount: number; + skippedFeatures: Feature[]; + skippedVerticesCount: number; } export interface ChunkProcessor { diff --git a/src/files/shapefile/types/state.ts b/src/files/shapefile/types/state.ts index 201695c..665e7ac 100644 --- a/src/files/shapefile/types/state.ts +++ b/src/files/shapefile/types/state.ts @@ -9,7 +9,6 @@ export interface ProgressInfo { processedFeatures: number; totalFeatures: number; processedChunks: number; - totalChunks: number; processedVertices: number; totalVertices: number; skippedFeatures: number; diff --git a/tests/unit/files/shapefile/progressTracker.spec.ts b/tests/unit/files/shapefile/progressTracker.spec.ts index fc7bb81..e3ba1ef 100644 --- a/tests/unit/files/shapefile/progressTracker.spec.ts +++ b/tests/unit/files/shapefile/progressTracker.spec.ts @@ -176,7 +176,6 @@ describe('ProgressTracker', () => { processedFeatures: 0, totalFeatures: mockTotalFeatures, processedChunks: 0, - totalChunks: 10, // Math.ceil(1000 / 100) processedVertices: 0, totalVertices: mockTotalVertices, percentage: 0, @@ -269,12 +268,6 @@ describe('ProgressTracker', () => { expect(progress.estimatedRemainingTimeMs).toBe(0); }); - it('should calculate total chunks correctly based on vertices and chunk size', () => { - const progress = progressTracker.calculateProgress(); - - expect(progress.totalChunks).toBe(10); // Math.ceil(1000 / 100) - }); - it('should handle edge case when total vertices is zero', () => { progressTracker = new ProgressTracker({ totalVertices: 0, @@ -285,7 +278,6 @@ describe('ProgressTracker', () => { const progress = progressTracker.calculateProgress(); expect(progress.percentage).toBe(0); - expect(progress.totalChunks).toBe(0); }); it('should set endTime when all features are processed', () => { @@ -305,20 +297,6 @@ describe('ProgressTracker', () => { expect(progress.endTime).toBeUndefined(); }); - - it('should use processed chunks as total when total vertices is zero', () => { - progressTracker = new ProgressTracker({ - totalVertices: 0, - totalFeatures: mockTotalFeatures, - maxVerticesPerChunk: mockMaxVerticesPerChunk, - }); - progressTracker.incrementChunks(); - progressTracker.incrementChunks(); - - const progress = progressTracker.calculateProgress(); - - expect(progress.totalChunks).toBe(2); // Uses processedChunks when totalVertices is 0 - }); }); describe('interface compliance', () => { diff --git a/tests/unit/files/shapefile/shapeFileReader.spec.ts b/tests/unit/files/shapefile/shapeFileReader.spec.ts index a9618f7..b78434b 100644 --- a/tests/unit/files/shapefile/shapeFileReader.spec.ts +++ b/tests/unit/files/shapefile/shapeFileReader.spec.ts @@ -14,7 +14,7 @@ import * as vertices from '../../../../src/geo/vertices'; const shapefilePath = '/path/to/shapefile.shp'; const dbfFilePath = shapefilePath.replace(/\.shp$/i, '.dbf'); -const mockRandomUUID = jest.fn(); +const mockRandomUUID = jest.fn(); // Mock all dependencies jest.mock('node:crypto', () => { @@ -130,8 +130,9 @@ describe('ShapefileChunkReader', () => { mockChunkBuilder.build.mockReturnValue({ id: 0, features: [mockFeature, mockFeature], - skippedFeatures: [], verticesCount: 100, + skippedFeatures: [], + skippedVerticesCount: 0, }); await reader.readAndProcess(shapefilePath, { process: mockProcessor }); @@ -154,8 +155,14 @@ describe('ShapefileChunkReader', () => { .mockReturnValueOnce(FeatureStatus.FULL) .mockReturnValueOnce(FeatureStatus.ADD); - const chunk1: ShapefileChunk = { id: 0, features: [mockFeature], skippedFeatures: [], verticesCount: 50 }; - const chunk2: ShapefileChunk = { id: 1, features: [mockFeature, mockFeature], skippedFeatures: [], verticesCount: 100 }; + const chunk1: ShapefileChunk = { id: 0, features: [mockFeature], skippedFeatures: [], verticesCount: 50, skippedVerticesCount: 0 }; + const chunk2: ShapefileChunk = { + id: 1, + features: [mockFeature, mockFeature], + skippedFeatures: [], + verticesCount: 100, + skippedVerticesCount: 0, + }; mockChunkBuilder.build.mockReturnValueOnce(chunk1).mockReturnValueOnce(chunk2); await reader.readAndProcess(shapefilePath, { process: mockProcessor }); @@ -191,6 +198,7 @@ describe('ShapefileChunkReader', () => { features: [mockFeature], verticesCount: 50, skippedFeatures: [], + skippedVerticesCount: 0, }); await reader.readAndProcess(shapefilePath, { process: mockProcessor }); @@ -222,8 +230,9 @@ describe('ShapefileChunkReader', () => { const chunk: ShapefileChunk = { id: 0, features: [], - skippedFeatures: [largeFeature], verticesCount: 0, + skippedFeatures: [largeFeature], + skippedVerticesCount: 8, }; mockSource.read.mockResolvedValueOnce({ done: false, value: largeFeature }).mockResolvedValueOnce({ done: true, value: largeFeature }); @@ -245,6 +254,7 @@ describe('ShapefileChunkReader', () => { features: [mockFeature], verticesCount: 50, skippedFeatures: [], + skippedVerticesCount: 0, }); Object.defineProperty(mockChunkBuilder, 'chunkId', { value: 0, writable: true }); @@ -265,7 +275,7 @@ describe('ShapefileChunkReader', () => { mockSource.read.mockResolvedValueOnce({ done: false, value: mockFeature }).mockResolvedValueOnce({ done: true, value: mockFeature }); mockChunkBuilder.canAddFeature.mockReturnValue(FeatureStatus.ADD); - const chunk = { id: 0, features: [mockFeature], skippedFeatures: [], verticesCount: 50 } as ShapefileChunk; + const chunk = { id: 0, features: [mockFeature], verticesCount: 50, skippedFeatures: [], skippedVerticesCount: 0 } as ShapefileChunk; mockChunkBuilder.build.mockReturnValue(chunk); await reader.readAndProcess(shapefilePath, { process: mockProcessor }); @@ -299,14 +309,16 @@ describe('ShapefileChunkReader', () => { const chunk: ShapefileChunk = { id: 0, features: [], - skippedFeatures: [largeFeature], verticesCount: 0, + skippedFeatures: [largeFeature], + skippedVerticesCount: 8, }; const finalChunk: ShapefileChunk = { id: 1, features: [], skippedFeatures: [], verticesCount: 0, + skippedVerticesCount: 0, }; mockSource.read.mockResolvedValueOnce({ done: false, value: largeFeature }).mockResolvedValueOnce({ done: true, value: largeFeature }); mockChunkBuilder.canAddFeature.mockReturnValue(FeatureStatus.SKIPPED); @@ -362,8 +374,8 @@ describe('ShapefileChunkReader', () => { // Second feature: exceeds max vertices, should be SKIPPED mockChunkBuilder.canAddFeature.mockReturnValueOnce(FeatureStatus.FULL).mockReturnValueOnce(FeatureStatus.SKIPPED); - const chunk1: ShapefileChunk = { id: 0, features: [normalFeature], skippedFeatures: [], verticesCount: 50 }; - const chunk2: ShapefileChunk = { id: 1, features: [], skippedFeatures: [largeFeature], verticesCount: 0 }; + const chunk1: ShapefileChunk = { id: 0, features: [normalFeature], verticesCount: 50, skippedFeatures: [], skippedVerticesCount: 0 }; + const chunk2: ShapefileChunk = { id: 1, features: [], verticesCount: 0, skippedFeatures: [largeFeature], skippedVerticesCount: 8 }; mockChunkBuilder.build.mockReturnValueOnce(chunk1).mockReturnValueOnce(chunk2); await reader.readAndProcess(shapefilePath, { process: mockProcessor }); @@ -399,7 +411,7 @@ describe('ShapefileChunkReader', () => { }); }); - it('should skip features exceeding vertex limit', async () => { + it('should include skip features exceeding vertex limit', async () => { jest.spyOn(vertices, 'countVertices').mockReturnValueOnce(100).mockReturnValueOnce(2000).mockReturnValueOnce(100); mockSource.read @@ -411,8 +423,8 @@ describe('ShapefileChunkReader', () => { const stats = await reader.getShapefileStats(shapefilePath); expect(stats).toEqual({ - totalVertices: 200, // Only features 1 and 3 - totalFeatures: 2, + totalVertices: 2200, + totalFeatures: 3, }); }); diff --git a/tests/unit/files/shapefile/utils.ts b/tests/unit/files/shapefile/utils.ts index 9cf793d..501b817 100644 --- a/tests/unit/files/shapefile/utils.ts +++ b/tests/unit/files/shapefile/utils.ts @@ -28,7 +28,8 @@ export function createTestChunk(id: number, featuresCount: number, verticesCount return { id, features, - skippedFeatures: [], verticesCount, + skippedFeatures: [], + skippedVerticesCount: 0, }; }