From cd014d0c1f4709581c5351761cb5edb99529392e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ng=C3=B4=20Qu=E1=BB=91c=20=C4=90=E1=BA=A1t?= Date: Thu, 4 Jun 2026 12:03:48 +0700 Subject: [PATCH 1/3] feat(plugins): import data from CSV and TSV files into a table (#1568) --- CHANGELOG.md | 1 + .../CSVImportPlugin/CSVImportOptions.swift | 84 +++++++ .../CSVImportOptionsView.swift | 80 ++++++ .../CSVImportPlugin/CSVImportParsing.swift | 145 +++++++++++ Plugins/CSVImportPlugin/CSVImportPlugin.swift | 237 ++++++++++++++++++ Plugins/CSVImportPlugin/Info.plist | 12 + Plugins/CSVInspectorPlugin/CSVWriter.swift | 3 +- .../CSVDialect.swift | 26 +- .../CSVStreamingParser.swift | 17 +- .../CSVTypeInferrer.swift | 11 +- .../ImportFormatPlugin.swift | 4 + TablePro.xcodeproj/project.pbxproj | 141 +++++++++++ ...ypeMapper.swift => ImportTypeMapper.swift} | 4 +- ...ImportSheet.swift => RowImportSheet.swift} | 38 ++- TablePro/Views/Main/MainContentView.swift | 2 +- .../Core/Plugins/ImportTypeMapperTests.swift | 42 ++++ .../Plugins/JSONImportTypeMapperTests.swift | 42 ---- .../PluginTestSources/CSVDialect.swift | 1 - .../CSVStreamingParser.swift | 1 - .../PluginTestSources/CSVTypeInferrer.swift | 1 - .../Plugins/CSVImportPluginTests.swift | 188 ++++++++++++++ docs/features/import-export.mdx | 29 ++- 22 files changed, 1021 insertions(+), 88 deletions(-) create mode 100644 Plugins/CSVImportPlugin/CSVImportOptions.swift create mode 100644 Plugins/CSVImportPlugin/CSVImportOptionsView.swift create mode 100644 Plugins/CSVImportPlugin/CSVImportParsing.swift create mode 100644 Plugins/CSVImportPlugin/CSVImportPlugin.swift create mode 100644 Plugins/CSVImportPlugin/Info.plist rename Plugins/{CSVInspectorPlugin => TableProPluginKit}/CSVDialect.swift (88%) rename Plugins/{CSVInspectorPlugin => TableProPluginKit}/CSVStreamingParser.swift (90%) rename Plugins/{CSVInspectorPlugin => TableProPluginKit}/CSVTypeInferrer.swift (86%) rename TablePro/Core/Plugins/{JSONImportTypeMapper.swift => ImportTypeMapper.swift} (97%) rename TablePro/Views/Import/{JSONImportSheet.swift => RowImportSheet.swift} (94%) create mode 100644 TableProTests/Core/Plugins/ImportTypeMapperTests.swift delete mode 100644 TableProTests/Core/Plugins/JSONImportTypeMapperTests.swift delete mode 120000 TableProTests/PluginTestSources/CSVDialect.swift delete mode 120000 TableProTests/PluginTestSources/CSVStreamingParser.swift delete mode 120000 TableProTests/PluginTestSources/CSVTypeInferrer.swift create mode 100644 TableProTests/Plugins/CSVImportPluginTests.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index 3bd649285..5e6a3e78a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Import data from CSV and TSV files into a table: map columns to an existing table or create a new one, with options for delimiter, quote character, encoding, header row, and empty/NULL handling. (#1568) - Each filter row has a checkbox to turn it on or off and an Apply button to filter by just that row. The main Apply runs every active filter, and disabled filters stay in the panel for later. (#1561) - Importing connections from other apps now detects duplicates by host, port, database, and username, and lets you replace, add a copy, or skip each one before import. - Oracle connections negotiate Native Network Encryption when the server asks for it, so servers with `SQLNET.ENCRYPTION_SERVER` or `SQLNET.CRYPTO_CHECKSUM_SERVER` set to REQUIRED now connect (AES with a SHA crypto-checksum), matching what SQL Developer and DBeaver do. (#483) diff --git a/Plugins/CSVImportPlugin/CSVImportOptions.swift b/Plugins/CSVImportPlugin/CSVImportOptions.swift new file mode 100644 index 000000000..b8fded4ca --- /dev/null +++ b/Plugins/CSVImportPlugin/CSVImportOptions.swift @@ -0,0 +1,84 @@ +// +// CSVImportOptions.swift +// CSVImportPlugin +// + +import Foundation +import TableProPluginKit + +struct CSVImportOptions: Equatable, Codable { + enum Delimiter: String, Codable, CaseIterable, Identifiable { + case auto + case comma + case semicolon + case tab + case pipe + + var id: String { rawValue } + + var byte: UInt8? { + switch self { + case .auto: return nil + case .comma: return 0x2C + case .semicolon: return 0x3B + case .tab: return 0x09 + case .pipe: return 0x7C + } + } + } + + enum QuoteCharacter: String, Codable, CaseIterable, Identifiable { + case doubleQuote + case singleQuote + + var id: String { rawValue } + + var byte: UInt8 { + switch self { + case .doubleQuote: return 0x22 + case .singleQuote: return 0x27 + } + } + } + + enum TextEncoding: String, Codable, CaseIterable, Identifiable { + case auto + case utf8 + case isoLatin1 + case windowsCP1252 + + var id: String { rawValue } + + var stringEncoding: String.Encoding? { + switch self { + case .auto: return nil + case .utf8: return .utf8 + case .isoLatin1: return .isoLatin1 + case .windowsCP1252: return .windowsCP1252 + } + } + } + + var delimiter: Delimiter = .auto + var quoteCharacter: QuoteCharacter = .doubleQuote + var encoding: TextEncoding = .auto + var hasHeaderRow: Bool = true + var trimWhitespace: Bool = false + var emptyAsNull: Bool = true + var nullString: String = "" + var errorHandling: ImportErrorHandling = .stopAndRollback + var wrapInTransaction: Bool = true + var deleteExistingRows: Bool = false + + var detectionSignature: String { + [ + delimiter.rawValue, + quoteCharacter.rawValue, + encoding.rawValue, + hasHeaderRow ? "h1" : "h0", + trimWhitespace ? "t1" : "t0", + emptyAsNull ? "n1" : "n0", + nullString + ].joined(separator: "|") + } +} diff --git a/Plugins/CSVImportPlugin/CSVImportOptionsView.swift b/Plugins/CSVImportPlugin/CSVImportOptionsView.swift new file mode 100644 index 000000000..76d4d7d1f --- /dev/null +++ b/Plugins/CSVImportPlugin/CSVImportOptionsView.swift @@ -0,0 +1,80 @@ +// +// CSVImportOptionsView.swift +// CSVImportPlugin +// + +import SwiftUI +import TableProPluginKit + +struct CSVImportOptionsView: View { + let plugin: CSVImportPlugin + + var body: some View { + VStack(alignment: .leading, spacing: 12) { + Picker("Delimiter:", selection: Bindable(plugin).settings.delimiter) { + Text("Auto-detect").tag(CSVImportOptions.Delimiter.auto) + Text("Comma (,)").tag(CSVImportOptions.Delimiter.comma) + Text("Semicolon (;)").tag(CSVImportOptions.Delimiter.semicolon) + Text("Tab").tag(CSVImportOptions.Delimiter.tab) + Text("Pipe (|)").tag(CSVImportOptions.Delimiter.pipe) + } + .pickerStyle(.menu) + .font(.system(size: 13)) + + Picker("Quote character:", selection: Bindable(plugin).settings.quoteCharacter) { + Text("Double quote (\")").tag(CSVImportOptions.QuoteCharacter.doubleQuote) + Text("Single quote (')").tag(CSVImportOptions.QuoteCharacter.singleQuote) + } + .pickerStyle(.menu) + .font(.system(size: 13)) + + Picker("Encoding:", selection: Bindable(plugin).settings.encoding) { + Text("Auto-detect").tag(CSVImportOptions.TextEncoding.auto) + Text("UTF-8").tag(CSVImportOptions.TextEncoding.utf8) + Text("ISO Latin 1").tag(CSVImportOptions.TextEncoding.isoLatin1) + Text("Windows-1252").tag(CSVImportOptions.TextEncoding.windowsCP1252) + } + .pickerStyle(.menu) + .font(.system(size: 13)) + + Toggle("First row is a header", isOn: Bindable(plugin).settings.hasHeaderRow) + .font(.system(size: 13)) + .help("Use the first row as column names. Turn off to import every row as data.") + + Toggle("Trim leading and trailing spaces", isOn: Bindable(plugin).settings.trimWhitespace) + .font(.system(size: 13)) + + Toggle("Treat empty values as NULL", isOn: Bindable(plugin).settings.emptyAsNull) + .font(.system(size: 13)) + .help("Insert NULL for empty fields instead of an empty string.") + + HStack(spacing: 8) { + Text("NULL text:") + TextField("", text: Bindable(plugin).settings.nullString, prompt: Text(verbatim: "\\N")) + .textFieldStyle(.roundedBorder) + .frame(width: 120) + } + .font(.system(size: 13)) + .help("An extra value that should be imported as NULL, for example \\N.") + + Picker("On error:", selection: Bindable(plugin).settings.errorHandling) { + Text("Stop and Rollback").tag(ImportErrorHandling.stopAndRollback) + Text("Stop and Commit").tag(ImportErrorHandling.stopAndCommit) + Text("Skip and Continue").tag(ImportErrorHandling.skipAndContinue) + } + .pickerStyle(.menu) + .font(.system(size: 13)) + + Toggle("Wrap in transaction (BEGIN/COMMIT)", isOn: Bindable(plugin).settings.wrapInTransaction) + .font(.system(size: 13)) + .disabled(plugin.settings.errorHandling == .skipAndContinue) + .help(plugin.settings.errorHandling == .skipAndContinue + ? String(localized: "Not available in skip-and-continue mode") + : String(localized: "Insert all rows in a single transaction. If any row fails, all changes are rolled back.")) + + Toggle("Delete existing rows before import", isOn: Bindable(plugin).settings.deleteExistingRows) + .font(.system(size: 13)) + .help("Remove every row from the target table before inserting the imported rows.") + } + } +} diff --git a/Plugins/CSVImportPlugin/CSVImportParsing.swift b/Plugins/CSVImportPlugin/CSVImportParsing.swift new file mode 100644 index 000000000..8d93dffb2 --- /dev/null +++ b/Plugins/CSVImportPlugin/CSVImportParsing.swift @@ -0,0 +1,145 @@ +// +// CSVImportParsing.swift +// CSVImportPlugin +// +// Pure CSV row extraction, NULL handling, and field inference. Kept free of the +// plugin's loadable-bundle and SwiftUI surface so it can be compiled into the +// test target directly (a loadable .tableplugin cannot be linked by tests). +// The RFC 4180 tokenizer itself lives in TableProPluginKit (CSVStreamingParser), +// shared with the CSV inspector. +// + +import Foundation +import TableProPluginKit + +enum CSVImportParsing { + static let detectionSampleLimit = 200 + + static func resolveDialect(in data: Data, options: CSVImportOptions) -> CSVDialect { + var dialect = CSVDialect.detect(from: data) + if let byte = options.delimiter.byte { + dialect.delimiter = byte + } + dialect.quoteChar = options.quoteCharacter.byte + if let forced = options.encoding.stringEncoding { + dialect.encoding = forced + } + return dialect + } + + static func defaultColumnName(_ index: Int) -> String { + "Column \(index + 1)" + } + + static func columnNames(header: [String]?, columnCount: Int) -> [String] { + var names: [String] = [] + names.reserveCapacity(columnCount) + var used = Set() + for index in 0.. PluginCellValue { + var value = raw + if options.trimWhitespace { + value = value.trimmingCharacters(in: .whitespaces) + } + if options.emptyAsNull, value.isEmpty { + return .null + } + if !options.nullString.isEmpty, value == options.nullString { + return .null + } + return .text(value) + } + + static func row(fields: [String], columnNames: [String], options: CSVImportOptions) -> [String: PluginCellValue] { + var row: [String: PluginCellValue] = [:] + row.reserveCapacity(columnNames.count) + for (index, name) in columnNames.enumerated() { + let raw = index < fields.count ? fields[index] : "" + row[name] = cellValue(from: raw, options: options) + } + return row + } + + static func importFieldType(for type: CSVTypeInferrer.InferredType) -> PluginImportFieldType { + switch type { + case .integer: return .integer + case .real: return .real + case .boolean: return .boolean + case .date: return .text + case .text: return .text + @unknown default: return .text + } + } + + static func isBlank(_ fields: [String]) -> Bool { + fields.allSatisfy { $0.isEmpty } + } + + static func detectFields( + in data: Data, + options: CSVImportOptions, + limit: Int = detectionSampleLimit + ) -> [PluginImportField] { + let dialect = resolveDialect(in: data, options: options) + let parser = CSVStreamingParser(dialect: dialect) + + return data.withUnsafeBytes { raw -> [PluginImportField] in + guard let base = raw.bindMemory(to: UInt8.self).baseAddress else { return [] } + let buffer = UnsafeBufferPointer(start: base, count: raw.count) + let ranges = parser.indexRows(buffer) + guard !ranges.isEmpty else { return [] } + + var dataRanges = ranges[...] + var header: [String]? + if options.hasHeaderRow { + header = parser.parseRow(buffer, range: ranges[0]) + dataRanges = ranges.dropFirst() + } + + let columnCount = header?.count + ?? dataRanges.first.map { parser.parseRow(buffer, range: $0).count } + ?? 0 + guard columnCount > 0 else { return [] } + + let names = columnNames(header: header, columnCount: columnCount) + var samples: [[String]] = Array(repeating: [], count: columnCount) + var firstValues: [String?] = Array(repeating: nil, count: columnCount) + var sampled = 0 + + for range in dataRanges { + if sampled >= limit { break } + let fields = parser.parseRow(buffer, range: range) + if isBlank(fields) { continue } + for column in 0.. AnyView? { + AnyView(CSVImportOptionsView(plugin: self)) + } + + private static let detectionPrefixBytes = 1_048_576 + private static let batchSize = 500 + private static let maxErrors = 1_000 + + func performImport( + source: any PluginImportSource, + sink: any PluginImportDataSink, + progress: PluginImportProgress + ) async throws -> PluginImportResult { + let startTime = Date() + let url = source.fileURL() + let useTransaction = settings.wrapInTransaction && settings.errorHandling != .skipAndContinue + + let data: Data + do { + data = try Data(contentsOf: url, options: .mappedIfSafe) + } catch { + throw PluginImportError.importFailed(error.localizedDescription) + } + + let dialect = CSVImportParsing.resolveDialect(in: data, options: settings) + let parser = CSVStreamingParser(dialect: dialect) + let hasHeader = settings.hasHeaderRow + + let (dataRanges, columnNames) = indexRowsAndNames(in: data, parser: parser, hasHeader: hasHeader) + guard !columnNames.isEmpty else { + throw PluginImportError.importFailed("No columns found in the file") + } + + progress.setEstimatedTotal(dataRanges.count) + + var inserted = 0 + var skipped = 0 + var errors: [PluginImportResult.ImportStatementError] = [] + + do { + if settings.deleteExistingRows { + try await sink.deleteAllRowsFromTargetTable() + } + if useTransaction { + try await sink.beginTransaction() + } + + let lineOffset = hasHeader ? 2 : 1 + var index = 0 + while index < dataRanges.count { + try progress.checkCancellation() + let end = min(index + Self.batchSize, dataRanges.count) + let batch = parseBatch( + in: data, + parser: parser, + ranges: dataRanges[index.. ([Range], [String]) { + data.withUnsafeBytes { raw -> ([Range], [String]) in + guard let base = raw.bindMemory(to: UInt8.self).baseAddress else { return ([], []) } + let buffer = UnsafeBufferPointer(start: base, count: raw.count) + let ranges = parser.indexRows(buffer) + guard !ranges.isEmpty else { return ([], []) } + if hasHeader { + let header = parser.parseRow(buffer, range: ranges[0]) + let names = CSVImportParsing.columnNames(header: header, columnCount: header.count) + return (Array(ranges.dropFirst()), names) + } + let firstCount = parser.parseRow(buffer, range: ranges[0]).count + let names = CSVImportParsing.columnNames(header: nil, columnCount: firstCount) + return (ranges, names) + } + } + + private func parseBatch( + in data: Data, + parser: CSVStreamingParser, + ranges: ArraySlice>, + startIndex: Int, + lineOffset: Int, + columnNames: [String] + ) -> [(line: Int, row: [String: PluginCellValue])] { + data.withUnsafeBytes { raw -> [(line: Int, row: [String: PluginCellValue])] in + guard let base = raw.bindMemory(to: UInt8.self).baseAddress else { return [] } + let buffer = UnsafeBufferPointer(start: base, count: raw.count) + var out: [(line: Int, row: [String: PluginCellValue])] = [] + out.reserveCapacity(ranges.count) + for (offset, range) in ranges.enumerated() { + let fields = parser.parseRow(buffer, range: range) + if CSVImportParsing.isBlank(fields) { continue } + let line = startIndex + offset + lineOffset + out.append((line, CSVImportParsing.row(fields: fields, columnNames: columnNames, options: settings))) + } + return out + } + } + + private func flush( + _ batch: [(line: Int, row: [String: PluginCellValue])], + into sink: any PluginImportDataSink, + progress: PluginImportProgress, + inserted: inout Int, + skipped: inout Int, + errors: inout [PluginImportResult.ImportStatementError] + ) async throws { + guard !batch.isEmpty else { return } + do { + try await sink.insertRows(batch.map(\.row)) + inserted += batch.count + progress.incrementStatement(by: batch.count) + } catch { + switch settings.errorHandling { + case .stopAndRollback, .stopAndCommit: + let firstLine = batch.first?.line ?? 0 + throw PluginImportError.statementFailed( + statement: "rows \(firstLine)-\(batch.last?.line ?? firstLine)", + line: firstLine, + underlyingError: error + ) + case .skipAndContinue: + for entry in batch { + try await insert(entry.row, into: sink, at: entry.line, progress: progress, + inserted: &inserted, skipped: &skipped, errors: &errors) + } + } + } + } + + private func insert( + _ row: [String: PluginCellValue], + into sink: any PluginImportDataSink, + at line: Int, + progress: PluginImportProgress, + inserted: inout Int, + skipped: inout Int, + errors: inout [PluginImportResult.ImportStatementError] + ) async throws { + do { + try await sink.insertRow(row) + inserted += 1 + progress.incrementStatement() + } catch { + switch settings.errorHandling { + case .stopAndRollback, .stopAndCommit: + throw PluginImportError.statementFailed(statement: "row \(line)", line: line, underlyingError: error) + case .skipAndContinue: + skipped += 1 + if errors.count < Self.maxErrors { + errors.append(.init(statement: "row \(line)", line: line, errorMessage: error.localizedDescription)) + } + progress.incrementStatement() + } + } + } + + func detectSourceFields(at url: URL, targetTable: String?) throws -> [PluginImportField] { + let data = try readDetectionPrefix(of: url) + return CSVImportParsing.detectFields(in: data, options: settings) + } + + private func readDetectionPrefix(of url: URL) throws -> Data { + let handle = try FileHandle(forReadingFrom: url) + defer { try? handle.close() } + return handle.readData(ofLength: Self.detectionPrefixBytes) + } +} diff --git a/Plugins/CSVImportPlugin/Info.plist b/Plugins/CSVImportPlugin/Info.plist new file mode 100644 index 000000000..14e54accd --- /dev/null +++ b/Plugins/CSVImportPlugin/Info.plist @@ -0,0 +1,12 @@ + + + + + TableProPluginKitVersion + 18 + TableProProvidesImportFormatIds + + csv + + + diff --git a/Plugins/CSVInspectorPlugin/CSVWriter.swift b/Plugins/CSVInspectorPlugin/CSVWriter.swift index d781fe46d..06153172a 100644 --- a/Plugins/CSVInspectorPlugin/CSVWriter.swift +++ b/Plugins/CSVInspectorPlugin/CSVWriter.swift @@ -1,4 +1,5 @@ import Foundation +import TableProPluginKit struct CSVWriter { enum WriteError: Error, LocalizedError { @@ -34,7 +35,7 @@ struct CSVWriter { defer { try? handle.close() } var buffer = Data() - buffer.reserveCapacity(Self.flushThreshold + 4096) + buffer.reserveCapacity(Self.flushThreshold + 4_096) buffer.append(contentsOf: dialect.bomBytes) append(store.headerSource, from: store, into: &buffer) diff --git a/Plugins/CSVInspectorPlugin/CSVDialect.swift b/Plugins/TableProPluginKit/CSVDialect.swift similarity index 88% rename from Plugins/CSVInspectorPlugin/CSVDialect.swift rename to Plugins/TableProPluginKit/CSVDialect.swift index c90529b17..bc42c6142 100644 --- a/Plugins/CSVInspectorPlugin/CSVDialect.swift +++ b/Plugins/TableProPluginKit/CSVDialect.swift @@ -1,12 +1,12 @@ import Foundation -struct CSVDialect: Equatable, Sendable { - enum LineEnding: Equatable, Sendable { +public struct CSVDialect: Equatable, Sendable { + public enum LineEnding: Equatable, Sendable { case crlf case lf case cr - var bytes: [UInt8] { + public var bytes: [UInt8] { switch self { case .crlf: return [0x0D, 0x0A] case .lf: return [0x0A] @@ -15,13 +15,13 @@ struct CSVDialect: Equatable, Sendable { } } - var delimiter: UInt8 - var quoteChar: UInt8 - var encoding: String.Encoding - var lineEnding: LineEnding - var hasBom: Bool + public var delimiter: UInt8 + public var quoteChar: UInt8 + public var encoding: String.Encoding + public var lineEnding: LineEnding + public var hasBom: Bool - init( + public init( delimiter: UInt8, quoteChar: UInt8 = 0x22, encoding: String.Encoding = .utf8, @@ -35,12 +35,12 @@ struct CSVDialect: Equatable, Sendable { self.hasBom = hasBom } - static let csv = CSVDialect(delimiter: 0x2C) - static let tsv = CSVDialect(delimiter: 0x09) + public static let csv = CSVDialect(delimiter: 0x2C) + public static let tsv = CSVDialect(delimiter: 0x09) private static let detectionScanLimit = 65_536 - static func detect(from data: Data) -> CSVDialect { + public static func detect(from data: Data) -> CSVDialect { var hasBom = false var encoding: String.Encoding = .utf8 var bomLength = 0 @@ -142,7 +142,7 @@ struct CSVDialect: Equatable, Sendable { return .lf } - var bomBytes: [UInt8] { + public var bomBytes: [UInt8] { guard hasBom else { return [] } switch encoding { case .utf8: return [0xEF, 0xBB, 0xBF] diff --git a/Plugins/CSVInspectorPlugin/CSVStreamingParser.swift b/Plugins/TableProPluginKit/CSVStreamingParser.swift similarity index 90% rename from Plugins/CSVInspectorPlugin/CSVStreamingParser.swift rename to Plugins/TableProPluginKit/CSVStreamingParser.swift index 2cd21053b..0bf9451a1 100644 --- a/Plugins/CSVInspectorPlugin/CSVStreamingParser.swift +++ b/Plugins/TableProPluginKit/CSVStreamingParser.swift @@ -1,9 +1,13 @@ import Foundation -struct CSVStreamingParser: Sendable { - let dialect: CSVDialect +public struct CSVStreamingParser: Sendable { + public let dialect: CSVDialect - func indexRows(_ bytes: UnsafeBufferPointer) -> [Range] { + public init(dialect: CSVDialect) { + self.dialect = dialect + } + + public func indexRows(_ bytes: UnsafeBufferPointer) -> [Range] { var ranges: [Range] = [] let quote = dialect.quoteChar let delimiter = dialect.delimiter @@ -61,7 +65,7 @@ struct CSVStreamingParser: Sendable { return ranges } - func parseRow(_ bytes: UnsafeBufferPointer, range: Range) -> [String] { + public func parseRow(_ bytes: UnsafeBufferPointer, range: Range) -> [String] { var fields: [String] = [] var field: [UInt8] = [] let quote = dialect.quoteChar @@ -108,7 +112,7 @@ struct CSVStreamingParser: Sendable { return fields } - func field(_ bytes: UnsafeBufferPointer, range: Range, column: Int) -> String { + public func field(_ bytes: UnsafeBufferPointer, range: Range, column: Int) -> String { guard column >= 0 else { return "" } let quote = dialect.quoteChar let delimiter = dialect.delimiter @@ -162,7 +166,8 @@ struct CSVStreamingParser: Sendable { private func decode(_ bytes: [UInt8]) -> String { if bytes.isEmpty { return "" } return String(bytes: bytes, encoding: dialect.encoding) - ?? String(decoding: bytes, as: UTF8.self) + ?? String(bytes: bytes, encoding: .isoLatin1) + ?? "" } private func bomSkip(in bytes: UnsafeBufferPointer) -> Int { diff --git a/Plugins/CSVInspectorPlugin/CSVTypeInferrer.swift b/Plugins/TableProPluginKit/CSVTypeInferrer.swift similarity index 86% rename from Plugins/CSVInspectorPlugin/CSVTypeInferrer.swift rename to Plugins/TableProPluginKit/CSVTypeInferrer.swift index 3d963824f..6ebf858c2 100644 --- a/Plugins/CSVInspectorPlugin/CSVTypeInferrer.swift +++ b/Plugins/TableProPluginKit/CSVTypeInferrer.swift @@ -1,10 +1,9 @@ import Foundation -import TableProPluginKit -struct CSVTypeInferrer { - typealias InferredType = InspectorColumnType +public struct CSVTypeInferrer { + public typealias InferredType = InspectorColumnType - static let sampleSize = 200 + public static let sampleSize = 200 private static let booleanLiterals: Set = [ "true", "false", "yes", "no", "t", "f", "y", "n" @@ -22,7 +21,7 @@ struct CSVTypeInferrer { return formatter }() - static func infer(column values: [String]) -> InferredType { + public static func infer(column values: [String]) -> InferredType { var sample: [String] = [] sample.reserveCapacity(min(values.count, sampleSize)) for value in values where !value.isEmpty { @@ -38,7 +37,7 @@ struct CSVTypeInferrer { return .text } - static func inferColumns(rows: [[String]], columnCount: Int) -> [InferredType] { + public static func inferColumns(rows: [[String]], columnCount: Int) -> [InferredType] { var result: [InferredType] = [] result.reserveCapacity(columnCount) for col in 0.. [PluginImportField] { [] } } diff --git a/TablePro.xcodeproj/project.pbxproj b/TablePro.xcodeproj/project.pbxproj index 9988d0646..beadf9b0b 100644 --- a/TablePro.xcodeproj/project.pbxproj +++ b/TablePro.xcodeproj/project.pbxproj @@ -40,6 +40,8 @@ 5A86F000D00000000 /* SQLImport.tableplugin in Copy Plug-Ins (12 items) */ = {isa = PBXBuildFile; fileRef = 5A86F000100000000 /* SQLImport.tableplugin */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; 5A86F001A00000000 /* TableProPluginKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 5A860000100000000 /* TableProPluginKit.framework */; }; 5A86F001D00000000 /* JSONImport.tableplugin in Copy Plug-Ins (12 items) */ = {isa = PBXBuildFile; fileRef = 5A86F001100000000 /* JSONImport.tableplugin */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; + 5A86F003A00000000 /* TableProPluginKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 5A860000100000000 /* TableProPluginKit.framework */; }; + 5A86F003D00000000 /* CSVImport.tableplugin in Copy Plug-Ins (12 items) */ = {isa = PBXBuildFile; fileRef = 5A86F003100000000 /* CSVImport.tableplugin */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; 5A87A000A00000000 /* TableProPluginKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 5A860000100000000 /* TableProPluginKit.framework */; }; 5ABBED742FB55E1400A78382 /* TableProPluginKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 5A860000100000000 /* TableProPluginKit.framework */; }; 5ABBED802FB55E1400A78382 /* CSVInspectorPlugin.tableplugin in Copy Plug-Ins (12 items) */ = {isa = PBXBuildFile; fileRef = 5ABBED792FB55E1400A78382 /* CSVInspectorPlugin.tableplugin */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; @@ -196,6 +198,13 @@ remoteGlobalIDString = 5A86F001000000000; remoteInfo = JSONImport; }; + 5A86F003B00000000 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 5A1091BF2EF17EDC0055EA7C /* Project object */; + proxyType = 1; + remoteGlobalIDString = 5A86F003000000000; + remoteInfo = CSVImport; + }; 5ABBED7E2FB55E1400A78382 /* PBXContainerItemProxy */ = { isa = PBXContainerItemProxy; containerPortal = 5A1091BF2EF17EDC0055EA7C /* Project object */; @@ -272,6 +281,7 @@ 5A86E000D00000000 /* MQLExport.tableplugin in Copy Plug-Ins (12 items) */, 5A86F000D00000000 /* SQLImport.tableplugin in Copy Plug-Ins (12 items) */, 5A86F001D00000000 /* JSONImport.tableplugin in Copy Plug-Ins (12 items) */, + 5A86F003D00000000 /* CSVImport.tableplugin in Copy Plug-Ins (12 items) */, 5ABBED802FB55E1400A78382 /* CSVInspectorPlugin.tableplugin in Copy Plug-Ins (12 items) */, ); name = "Copy Plug-Ins (12 items)"; @@ -311,6 +321,7 @@ 5A86E000100000000 /* MQLExport.tableplugin */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = MQLExport.tableplugin; sourceTree = BUILT_PRODUCTS_DIR; }; 5A86F000100000000 /* SQLImport.tableplugin */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = SQLImport.tableplugin; sourceTree = BUILT_PRODUCTS_DIR; }; 5A86F001100000000 /* JSONImport.tableplugin */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = JSONImport.tableplugin; sourceTree = BUILT_PRODUCTS_DIR; }; + 5A86F003100000000 /* CSVImport.tableplugin */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = CSVImport.tableplugin; sourceTree = BUILT_PRODUCTS_DIR; }; 5A87A000100000000 /* CassandraDriver.tableplugin */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = CassandraDriver.tableplugin; sourceTree = BUILT_PRODUCTS_DIR; }; 5ABBED792FB55E1400A78382 /* CSVInspectorPlugin.tableplugin */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = CSVInspectorPlugin.tableplugin; sourceTree = BUILT_PRODUCTS_DIR; }; 5ABCC5A72F43856700EAF3FC /* TableProTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = TableProTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; @@ -503,6 +514,21 @@ ); target = 5ABCC5A62F43856700EAF3FC /* TableProTests */; }; + 5A86F003900000000 /* Exceptions for "Plugins/CSVImportPlugin" folder in "CSVImport" target */ = { + isa = PBXFileSystemSynchronizedBuildFileExceptionSet; + membershipExceptions = ( + Info.plist, + ); + target = 5A86F003000000000 /* CSVImport */; + }; + 5A86F004900000000 /* Exceptions for "Plugins/CSVImportPlugin" folder in "TableProTests" target */ = { + isa = PBXFileSystemSynchronizedBuildFileExceptionSet; + membershipExceptions = ( + CSVImportOptions.swift, + CSVImportParsing.swift, + ); + target = 5ABCC5A62F43856700EAF3FC /* TableProTests */; + }; 5A87A000900000000 /* Exceptions for "Plugins/CassandraDriverPlugin" folder in "CassandraDriver" target */ = { isa = PBXFileSystemSynchronizedBuildFileExceptionSet; membershipExceptions = ( @@ -696,6 +722,15 @@ path = Plugins/JSONImportPlugin; sourceTree = ""; }; + 5A86F003500000000 /* Plugins/CSVImportPlugin */ = { + isa = PBXFileSystemSynchronizedRootGroup; + exceptions = ( + 5A86F003900000000 /* Exceptions for "Plugins/CSVImportPlugin" folder in "CSVImport" target */, + 5A86F004900000000 /* Exceptions for "Plugins/CSVImportPlugin" folder in "TableProTests" target */, + ); + path = Plugins/CSVImportPlugin; + sourceTree = ""; + }; 5A87A000500000000 /* Plugins/CassandraDriverPlugin */ = { isa = PBXFileSystemSynchronizedRootGroup; exceptions = ( @@ -898,6 +933,14 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + 5A86F003300000000 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + 5A86F003A00000000 /* TableProPluginKit.framework in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; 5A87A000300000000 /* Frameworks */ = { isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; @@ -999,6 +1042,7 @@ 5A86E000500000000 /* Plugins/MQLExportPlugin */, 5A86F000500000000 /* Plugins/SQLImportPlugin */, 5A86F001500000000 /* Plugins/JSONImportPlugin */, + 5A86F003500000000 /* Plugins/CSVImportPlugin */, 5ABCC5A82F43856700EAF3FC /* TableProTests */, 5AF00A122FB9000000000001 /* TableProUITests */, 5A32BC012F9D5F1300BAEB5F /* mcp-server */, @@ -1030,6 +1074,7 @@ 5A86E000100000000 /* MQLExport.tableplugin */, 5A86F000100000000 /* SQLImport.tableplugin */, 5A86F001100000000 /* JSONImport.tableplugin */, + 5A86F003100000000 /* CSVImport.tableplugin */, 5ABCC5A72F43856700EAF3FC /* TableProTests.xctest */, 5AF00A102FB9000000000001 /* TableProUITests.xctest */, 5AEA8B2A2F6808270040461A /* EtcdDriverPlugin.tableplugin */, @@ -1136,6 +1181,7 @@ 5A86E000C00000000 /* PBXTargetDependency */, 5A86F000C00000000 /* PBXTargetDependency */, 5A86F001C00000000 /* PBXTargetDependency */, + 5A86F003C00000000 /* PBXTargetDependency */, 5ABBED7F2FB55E1400A78382 /* PBXTargetDependency */, 5ADDB00000000000000000C1 /* PBXTargetDependency */, 5ABQR00000000000000000C1 /* PBXTargetDependency */, @@ -1547,6 +1593,26 @@ productReference = 5A86F001100000000 /* JSONImport.tableplugin */; productType = "com.apple.product-type.bundle"; }; + 5A86F003000000000 /* CSVImport */ = { + isa = PBXNativeTarget; + buildConfigurationList = 5A86F003800000000 /* Build configuration list for PBXNativeTarget "CSVImport" */; + buildPhases = ( + 5A86F003200000000 /* Sources */, + 5A86F003300000000 /* Frameworks */, + 5A86F003400000000 /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + ); + fileSystemSynchronizedGroups = ( + 5A86F003500000000 /* Plugins/CSVImportPlugin */, + ); + name = CSVImport; + productName = CSVImport; + productReference = 5A86F003100000000 /* CSVImport.tableplugin */; + productType = "com.apple.product-type.bundle"; + }; 5A87A000000000000 /* CassandraDriver */ = { isa = PBXNativeTarget; buildConfigurationList = 5A87A000800000000 /* Build configuration list for PBXNativeTarget "CassandraDriver" */; @@ -1834,6 +1900,7 @@ 5A86E000000000000 /* MQLExport */, 5A86F000000000000 /* SQLImport */, 5A86F001000000000 /* JSONImport */, + 5A86F003000000000 /* CSVImport */, 5ABCC5A62F43856700EAF3FC /* TableProTests */, 5AF00A142FB9000000000001 /* TableProUITests */, 5AEA8B292F6808270040461A /* EtcdDriverPlugin */, @@ -1981,6 +2048,13 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + 5A86F003400000000 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; 5A87A000400000000 /* Resources */ = { isa = PBXResourcesBuildPhase; buildActionMask = 2147483647; @@ -2180,6 +2254,13 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + 5A86F003200000000 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; 5A87A000200000000 /* Sources */ = { isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; @@ -2345,6 +2426,11 @@ target = 5A86F001000000000 /* JSONImport */; targetProxy = 5A86F001B00000000 /* PBXContainerItemProxy */; }; + 5A86F003C00000000 /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + target = 5A86F003000000000 /* CSVImport */; + targetProxy = 5A86F003B00000000 /* PBXContainerItemProxy */; + }; 5ABBED7F2FB55E1400A78382 /* PBXTargetDependency */ = { isa = PBXTargetDependency; target = 5ABBED712FB55E1400A78382 /* CSVInspectorPlugin */; @@ -3725,6 +3811,52 @@ }; name = Release; }; + 5A86F003600000000 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + CODE_SIGN_STYLE = Automatic; + COMBINE_HIDPI_IMAGES = YES; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = ""; + GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_FILE = Plugins/CSVImportPlugin/Info.plist; + INFOPLIST_KEY_NSPrincipalClass = "$(PRODUCT_MODULE_NAME).CSVImportPlugin"; + LD_RUNPATH_SEARCH_PATHS = "@executable_path/../Frameworks"; + MACOSX_DEPLOYMENT_TARGET = 14.0; + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = com.TablePro.CSVImportPlugin; + PRODUCT_NAME = "$(TARGET_NAME)"; + SDKROOT = macosx; + SKIP_INSTALL = YES; + SUPPORTED_PLATFORMS = macosx; + SWIFT_VERSION = 5.9; + WRAPPER_EXTENSION = tableplugin; + }; + name = Debug; + }; + 5A86F003700000000 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + CODE_SIGN_STYLE = Automatic; + COMBINE_HIDPI_IMAGES = YES; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = ""; + GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_FILE = Plugins/CSVImportPlugin/Info.plist; + INFOPLIST_KEY_NSPrincipalClass = "$(PRODUCT_MODULE_NAME).CSVImportPlugin"; + LD_RUNPATH_SEARCH_PATHS = "@executable_path/../Frameworks"; + MACOSX_DEPLOYMENT_TARGET = 14.0; + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = com.TablePro.CSVImportPlugin; + PRODUCT_NAME = "$(TARGET_NAME)"; + SDKROOT = macosx; + SKIP_INSTALL = YES; + SUPPORTED_PLATFORMS = macosx; + SWIFT_VERSION = 5.9; + WRAPPER_EXTENSION = tableplugin; + }; + name = Release; + }; 5A87A000600000000 /* Debug */ = { isa = XCBuildConfiguration; buildSettings = { @@ -4336,6 +4468,15 @@ defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; + 5A86F003800000000 /* Build configuration list for PBXNativeTarget "CSVImport" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 5A86F003600000000 /* Debug */, + 5A86F003700000000 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; 5A87A000800000000 /* Build configuration list for PBXNativeTarget "CassandraDriver" */ = { isa = XCConfigurationList; buildConfigurations = ( diff --git a/TablePro/Core/Plugins/JSONImportTypeMapper.swift b/TablePro/Core/Plugins/ImportTypeMapper.swift similarity index 97% rename from TablePro/Core/Plugins/JSONImportTypeMapper.swift rename to TablePro/Core/Plugins/ImportTypeMapper.swift index 8a221535b..d9da17a9f 100644 --- a/TablePro/Core/Plugins/JSONImportTypeMapper.swift +++ b/TablePro/Core/Plugins/ImportTypeMapper.swift @@ -1,12 +1,12 @@ // -// JSONImportTypeMapper.swift +// ImportTypeMapper.swift // TablePro // import Foundation import TableProPluginKit -enum JSONImportTypeMapper { +enum ImportTypeMapper { static func sqlType(for type: PluginImportFieldType, databaseType: DatabaseType) -> String { switch databaseType { case .postgresql, .redshift, .cockroachdb: diff --git a/TablePro/Views/Import/JSONImportSheet.swift b/TablePro/Views/Import/RowImportSheet.swift similarity index 94% rename from TablePro/Views/Import/JSONImportSheet.swift rename to TablePro/Views/Import/RowImportSheet.swift index 407c10f56..79ea59eaa 100644 --- a/TablePro/Views/Import/JSONImportSheet.swift +++ b/TablePro/Views/Import/RowImportSheet.swift @@ -1,10 +1,11 @@ // -// JSONImportSheet.swift +// RowImportSheet.swift // TablePro // -// Dedicated import sheet for row-based formats (JSON / NDJSON): -// map each source field to a column in an existing table, or create a new -// table with columns inferred from the data. +// Import sheet for row-based formats (JSON, NDJSON, CSV): map each source +// field to a column in an existing table, or create a new table with columns +// inferred from the data. The format plugin supplies the icon, name, and the +// field-detection options shown in this sheet. // import Combine @@ -12,8 +13,8 @@ import os import SwiftUI import TableProPluginKit -struct JSONImportSheet: View { - private static let logger = Logger(subsystem: "com.TablePro", category: "JSONImportSheet") +struct RowImportSheet: View { + private static let logger = Logger(subsystem: "com.TablePro", category: "RowImportSheet") @Binding var isPresented: Bool let connection: DatabaseConnection @@ -96,6 +97,9 @@ struct JSONImportSheet: View { guard destination == .existingTable, let table = newValue else { return } Task { await loadExistingContext(table: table) } } + .onChange(of: currentPlugin?.fieldDetectionSignature) { _, _ in + Task { await redetectFields() } + } .onDisappear { importTask?.cancel() } .sheet(isPresented: $showProgressDialog) { if let service = importService { @@ -118,13 +122,13 @@ struct JSONImportSheet: View { private var headerView: some View { HStack(alignment: .top, spacing: 12) { - Image(systemName: "curlybraces") + Image(systemName: currentPlugin.map { type(of: $0).iconName } ?? "tablecells") .font(.title) .foregroundStyle(.blue) VStack(alignment: .leading, spacing: 2) { Text(fileURL.lastPathComponent) .font(.headline) - Text("Import JSON rows into a table") + Text("Import rows into a table") .font(.subheadline) .foregroundStyle(.secondary) } @@ -243,7 +247,7 @@ struct JSONImportSheet: View { Toggle("", isOn: allMappingsIncluded) .labelsHidden() .help(String(localized: "Import all fields")) - Text("JSON field").font(.caption).foregroundStyle(.secondary) + Text("Field").font(.caption).foregroundStyle(.secondary) Text("Column").font(.caption).foregroundStyle(.secondary) } Divider().gridCellColumns(3) @@ -436,7 +440,7 @@ struct JSONImportSheet: View { field: field, include: true, name: field.name, - type: JSONImportTypeMapper.sqlType(for: field.inferredType, databaseType: connection.type), + type: ImportTypeMapper.sqlType(for: field.inferredType, databaseType: connection.type), isPrimaryKey: false, isNullable: true, defaultValue: "" @@ -470,6 +474,18 @@ struct JSONImportSheet: View { } } + @MainActor + private func redetectFields() async { + switch destination { + case .existingTable: + guard let table = selectedTargetTable else { return } + await loadExistingContext(table: table) + case .newTable: + newColumnsLoaded = false + await loadNewColumns() + } + } + // MARK: - Import private func performImport() { @@ -481,7 +497,7 @@ struct JSONImportSheet: View { let name = newTableName.trimmingCharacters(in: .whitespaces) guard !name.isEmpty, let sql = buildCreateTableSQL(tableName: name) else { importError = NSError( - domain: "JSONImport", code: -1, + domain: "RowImport", code: -1, userInfo: [NSLocalizedDescriptionKey: String(localized: "Could not build the CREATE TABLE statement")] ) showErrorDialog = true diff --git a/TablePro/Views/Main/MainContentView.swift b/TablePro/Views/Main/MainContentView.swift index 00647bdff..89d2b72a7 100644 --- a/TablePro/Views/Main/MainContentView.swift +++ b/TablePro/Views/Main/MainContentView.swift @@ -241,7 +241,7 @@ struct MainContentView: View { } ) if let url = coordinator.importFileURL { - JSONImportSheet( + RowImportSheet( isPresented: rowDismiss, connection: connection, fileURL: url, diff --git a/TableProTests/Core/Plugins/ImportTypeMapperTests.swift b/TableProTests/Core/Plugins/ImportTypeMapperTests.swift new file mode 100644 index 000000000..6c09bba6b --- /dev/null +++ b/TableProTests/Core/Plugins/ImportTypeMapperTests.swift @@ -0,0 +1,42 @@ +// +// ImportTypeMapperTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +@Suite("Import Type Mapper") +struct ImportTypeMapperTests { + @Test("PostgreSQL maps inferred types to native SQL types") + func testPostgres() { + #expect(ImportTypeMapper.sqlType(for: .integer, databaseType: .postgresql) == "BIGINT") + #expect(ImportTypeMapper.sqlType(for: .real, databaseType: .postgresql) == "DOUBLE PRECISION") + #expect(ImportTypeMapper.sqlType(for: .boolean, databaseType: .postgresql) == "BOOLEAN") + #expect(ImportTypeMapper.sqlType(for: .json, databaseType: .postgresql) == "JSONB") + #expect(ImportTypeMapper.sqlType(for: .text, databaseType: .postgresql) == "TEXT") + } + + @Test("MySQL maps inferred types to native SQL types") + func testMySQL() { + #expect(ImportTypeMapper.sqlType(for: .integer, databaseType: .mysql) == "BIGINT") + #expect(ImportTypeMapper.sqlType(for: .boolean, databaseType: .mysql) == "TINYINT(1)") + #expect(ImportTypeMapper.sqlType(for: .json, databaseType: .mysql) == "JSON") + } + + @Test("SQLite uses its storage classes") + func testSQLite() { + #expect(ImportTypeMapper.sqlType(for: .integer, databaseType: .sqlite) == "INTEGER") + #expect(ImportTypeMapper.sqlType(for: .real, databaseType: .sqlite) == "REAL") + #expect(ImportTypeMapper.sqlType(for: .json, databaseType: .sqlite) == "TEXT") + } + + @Test("Unhandled database types fall back to generic SQL types") + func testFallback() { + #expect(ImportTypeMapper.sqlType(for: .text, databaseType: .clickhouse) == "TEXT") + #expect(ImportTypeMapper.sqlType(for: .integer, databaseType: .clickhouse) == "INTEGER") + #expect(ImportTypeMapper.sqlType(for: .boolean, databaseType: .clickhouse) == "BOOLEAN") + } +} diff --git a/TableProTests/Core/Plugins/JSONImportTypeMapperTests.swift b/TableProTests/Core/Plugins/JSONImportTypeMapperTests.swift deleted file mode 100644 index e03d4383a..000000000 --- a/TableProTests/Core/Plugins/JSONImportTypeMapperTests.swift +++ /dev/null @@ -1,42 +0,0 @@ -// -// JSONImportTypeMapperTests.swift -// TableProTests -// - -import Foundation -@testable import TablePro -import TableProPluginKit -import Testing - -@Suite("JSON Import Type Mapper") -struct JSONImportTypeMapperTests { - @Test("PostgreSQL maps inferred types to native SQL types") - func testPostgres() { - #expect(JSONImportTypeMapper.sqlType(for: .integer, databaseType: .postgresql) == "BIGINT") - #expect(JSONImportTypeMapper.sqlType(for: .real, databaseType: .postgresql) == "DOUBLE PRECISION") - #expect(JSONImportTypeMapper.sqlType(for: .boolean, databaseType: .postgresql) == "BOOLEAN") - #expect(JSONImportTypeMapper.sqlType(for: .json, databaseType: .postgresql) == "JSONB") - #expect(JSONImportTypeMapper.sqlType(for: .text, databaseType: .postgresql) == "TEXT") - } - - @Test("MySQL maps inferred types to native SQL types") - func testMySQL() { - #expect(JSONImportTypeMapper.sqlType(for: .integer, databaseType: .mysql) == "BIGINT") - #expect(JSONImportTypeMapper.sqlType(for: .boolean, databaseType: .mysql) == "TINYINT(1)") - #expect(JSONImportTypeMapper.sqlType(for: .json, databaseType: .mysql) == "JSON") - } - - @Test("SQLite uses its storage classes") - func testSQLite() { - #expect(JSONImportTypeMapper.sqlType(for: .integer, databaseType: .sqlite) == "INTEGER") - #expect(JSONImportTypeMapper.sqlType(for: .real, databaseType: .sqlite) == "REAL") - #expect(JSONImportTypeMapper.sqlType(for: .json, databaseType: .sqlite) == "TEXT") - } - - @Test("Unhandled database types fall back to generic SQL types") - func testFallback() { - #expect(JSONImportTypeMapper.sqlType(for: .text, databaseType: .clickhouse) == "TEXT") - #expect(JSONImportTypeMapper.sqlType(for: .integer, databaseType: .clickhouse) == "INTEGER") - #expect(JSONImportTypeMapper.sqlType(for: .boolean, databaseType: .clickhouse) == "BOOLEAN") - } -} diff --git a/TableProTests/PluginTestSources/CSVDialect.swift b/TableProTests/PluginTestSources/CSVDialect.swift deleted file mode 120000 index 86975805d..000000000 --- a/TableProTests/PluginTestSources/CSVDialect.swift +++ /dev/null @@ -1 +0,0 @@ -../../Plugins/CSVInspectorPlugin/CSVDialect.swift \ No newline at end of file diff --git a/TableProTests/PluginTestSources/CSVStreamingParser.swift b/TableProTests/PluginTestSources/CSVStreamingParser.swift deleted file mode 120000 index ade260193..000000000 --- a/TableProTests/PluginTestSources/CSVStreamingParser.swift +++ /dev/null @@ -1 +0,0 @@ -../../Plugins/CSVInspectorPlugin/CSVStreamingParser.swift \ No newline at end of file diff --git a/TableProTests/PluginTestSources/CSVTypeInferrer.swift b/TableProTests/PluginTestSources/CSVTypeInferrer.swift deleted file mode 120000 index 1a0f494d7..000000000 --- a/TableProTests/PluginTestSources/CSVTypeInferrer.swift +++ /dev/null @@ -1 +0,0 @@ -../../Plugins/CSVInspectorPlugin/CSVTypeInferrer.swift \ No newline at end of file diff --git a/TableProTests/Plugins/CSVImportPluginTests.swift b/TableProTests/Plugins/CSVImportPluginTests.swift new file mode 100644 index 000000000..91509703f --- /dev/null +++ b/TableProTests/Plugins/CSVImportPluginTests.swift @@ -0,0 +1,188 @@ +// +// CSVImportPluginTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +@Suite("CSV Import Plugin") +struct CSVImportPluginTests { + private func data(_ text: String) -> Data { + Data(text.utf8) + } + + private func fields(_ name: String, _ list: [PluginImportField]) -> PluginImportField? { + list.first { $0.name == name } + } + + // MARK: - Dialect resolution + + @Test("Auto dialect detects the comma delimiter") + func testAutoDelimiter() { + let dialect = CSVImportParsing.resolveDialect(in: data("a,b,c\n1,2,3\n"), options: CSVImportOptions()) + #expect(dialect.delimiter == 0x2C) + } + + @Test("Explicit delimiter overrides detection") + func testDelimiterOverride() { + var options = CSVImportOptions() + options.delimiter = .semicolon + let dialect = CSVImportParsing.resolveDialect(in: data("a,b\n1,2\n"), options: options) + #expect(dialect.delimiter == 0x3B) + } + + @Test("Quote character and forced encoding are applied") + func testQuoteAndEncodingOverride() { + var options = CSVImportOptions() + options.quoteCharacter = .singleQuote + options.encoding = .isoLatin1 + let dialect = CSVImportParsing.resolveDialect(in: data("a,b\n1,2\n"), options: options) + #expect(dialect.quoteChar == 0x27) + #expect(dialect.encoding == .isoLatin1) + } + + // MARK: - Column names + + @Test("Header names are trimmed and empty headers get a placeholder") + func testColumnNamesFromHeader() { + let names = CSVImportParsing.columnNames(header: [" id ", "", "name"], columnCount: 3) + #expect(names == ["id", "Column 2", "name"]) + } + + @Test("Duplicate header names are made unique") + func testColumnNamesDeduplicated() { + let names = CSVImportParsing.columnNames(header: ["x", "x", "x"], columnCount: 3) + #expect(names == ["x", "x 2", "x 3"]) + } + + @Test("Without a header, names are synthesized positionally") + func testColumnNamesSynthesized() { + let names = CSVImportParsing.columnNames(header: nil, columnCount: 3) + #expect(names == ["Column 1", "Column 2", "Column 3"]) + } + + // MARK: - Cell values + + @Test("Empty fields become NULL by default") + func testEmptyAsNull() { + #expect(CSVImportParsing.cellValue(from: "", options: CSVImportOptions()) == .null) + } + + @Test("Empty fields stay empty text when emptyAsNull is off") + func testEmptyAsText() { + var options = CSVImportOptions() + options.emptyAsNull = false + #expect(CSVImportParsing.cellValue(from: "", options: options) == .text("")) + } + + @Test("A configured NULL token becomes NULL") + func testNullToken() { + var options = CSVImportOptions() + options.nullString = "\\N" + #expect(CSVImportParsing.cellValue(from: "\\N", options: options) == .null) + #expect(CSVImportParsing.cellValue(from: "value", options: options) == .text("value")) + } + + @Test("Whitespace is trimmed only when requested") + func testTrimWhitespace() { + var options = CSVImportOptions() + options.trimWhitespace = true + #expect(CSVImportParsing.cellValue(from: " hi ", options: options) == .text("hi")) + #expect(CSVImportParsing.cellValue(from: " hi ", options: CSVImportOptions()) == .text(" hi ")) + } + + @Test("Trimming an all-space field yields NULL when emptyAsNull is on") + func testTrimToNull() { + var options = CSVImportOptions() + options.trimWhitespace = true + #expect(CSVImportParsing.cellValue(from: " ", options: options) == .null) + } + + // MARK: - Row mapping + + @Test("Fields map to column names by position") + func testRowMapping() { + let row = CSVImportParsing.row(fields: ["1", "Alice"], columnNames: ["id", "name"], options: CSVImportOptions()) + #expect(row["id"] == .text("1")) + #expect(row["name"] == .text("Alice")) + } + + @Test("Missing trailing fields become NULL") + func testRaggedShortRow() { + let row = CSVImportParsing.row(fields: ["1"], columnNames: ["id", "name"], options: CSVImportOptions()) + #expect(row["id"] == .text("1")) + #expect(row["name"] == .null) + } + + @Test("Extra fields beyond the column count are ignored") + func testRaggedLongRow() { + let row = CSVImportParsing.row(fields: ["1", "Alice", "extra"], columnNames: ["id", "name"], options: CSVImportOptions()) + #expect(row.count == 2) + #expect(row["name"] == .text("Alice")) + } + + // MARK: - Type mapping + + @Test("Inspector types map to import field types, date falls back to text") + func testImportFieldTypeMapping() { + #expect(CSVImportParsing.importFieldType(for: .integer) == .integer) + #expect(CSVImportParsing.importFieldType(for: .real) == .real) + #expect(CSVImportParsing.importFieldType(for: .boolean) == .boolean) + #expect(CSVImportParsing.importFieldType(for: .text) == .text) + #expect(CSVImportParsing.importFieldType(for: .date) == .text) + } + + @Test("Blank rows are detected") + func testIsBlank() { + #expect(CSVImportParsing.isBlank([""])) + #expect(CSVImportParsing.isBlank(["", ""])) + #expect(!CSVImportParsing.isBlank(["", "x"])) + } + + // MARK: - Field detection + + @Test("Detects header names, sample values, and inferred types") + func testDetectFields() { + let csv = "id,name,score,active\n1,Alice,1.5,true\n2,Bob,2.0,false\n" + let result = CSVImportParsing.detectFields(in: data(csv), options: CSVImportOptions()) + #expect(result.map(\.name) == ["id", "name", "score", "active"]) + #expect(fields("id", result)?.inferredType == .integer) + #expect(fields("name", result)?.inferredType == .text) + #expect(fields("score", result)?.inferredType == .real) + #expect(fields("active", result)?.inferredType == .boolean) + #expect(fields("name", result)?.sampleValue == "Alice") + } + + @Test("Quoted fields keep embedded delimiters and newlines") + func testDetectQuotedFields() { + let csv = "name,note\n\"a,b\",\"line1\nline2\"\n" + let result = CSVImportParsing.detectFields(in: data(csv), options: CSVImportOptions()) + #expect(result.map(\.name) == ["name", "note"]) + #expect(fields("name", result)?.sampleValue == "a,b") + #expect(fields("note", result)?.sampleValue == "line1\nline2") + } + + @Test("Doubled quotes decode to a single quote") + func testDetectDoubledQuotes() { + let csv = "label\n\"say \"\"hi\"\"\"\n" + let result = CSVImportParsing.detectFields(in: data(csv), options: CSVImportOptions()) + #expect(fields("label", result)?.sampleValue == "say \"hi\"") + } + + @Test("Header-less detection uses positional names") + func testDetectWithoutHeader() { + var options = CSVImportOptions() + options.hasHeaderRow = false + let result = CSVImportParsing.detectFields(in: data("1,Alice\n2,Bob\n"), options: options) + #expect(result.map(\.name) == ["Column 1", "Column 2"]) + #expect(fields("Column 1", result)?.inferredType == .integer) + } + + @Test("Semicolon-delimited files are auto-detected") + func testDetectSemicolon() { + let result = CSVImportParsing.detectFields(in: data("a;b;c\n1;2;3\n"), options: CSVImportOptions()) + #expect(result.map(\.name) == ["a", "b", "c"]) + } +} diff --git a/docs/features/import-export.mdx b/docs/features/import-export.mdx index 0c7cf3076..8a5fbb2a5 100644 --- a/docs/features/import-export.mdx +++ b/docs/features/import-export.mdx @@ -1,11 +1,11 @@ --- title: Import & Export -description: Export to CSV, JSON, SQL, MQL, or XLSX. Import SQL files with transaction safety and progress tracking. +description: Export to CSV, JSON, SQL, MQL, or XLSX. Import SQL, JSON, and CSV files with column mapping, transaction safety, and progress tracking. --- # Import & Export -Export data in five formats (CSV, JSON, SQL, MQL, XLSX), import SQL files with gzip support, and paste tabular data from the clipboard into the grid. +Export data in five formats (CSV, JSON, SQL, MQL, XLSX), import SQL, JSON, and CSV files (with gzip support for SQL), and paste tabular data from the clipboard into the grid. ## Export Data @@ -198,7 +198,7 @@ Paste tabular data directly into the data grid. Press `Cmd+V` after selecting a ## Import Data -Import `.sql` and `.sql.gz` files (statements execute directly against your database: backups, migrations, seed data), or `.json` / `.jsonl` files into a table (see [Import JSON Data](#import-json-data)). +Import `.sql` and `.sql.gz` files (statements execute directly against your database: backups, migrations, seed data), `.json` / `.jsonl` files into a table (see [Import JSON Data](#import-json-data)), or `.csv` / `.tsv` files into a table (see [Import CSV Data](#import-csv-data)). **MongoDB**: SQL import is not available. Use `mongoimport` or the MQL shell. @@ -268,6 +268,29 @@ Choose a destination: Rows insert through parameterized statements, so JSON values are never concatenated into SQL. Nested objects and arrays are stored as JSON text. The on-error, transaction, and "delete existing rows" options work the same as SQL import. +### Import CSV Data + +Choose **File** > **Import** > **From CSV** and pick a `.csv` or `.tsv` file. TablePro opens the same row import sheet used for JSON, with CSV-specific parsing options. The delimiter, quote character, encoding, and line ending are auto-detected; change any of them and the field mapping re-reads the file. + +Choose a destination: + +- **Existing table**: pick the table, then map each CSV column to a table column. Columns are auto-matched by header name; toggle one off to skip it, or remap it. +- **New table**: name the table and review the columns TablePro infers from the data. Each column's name, type, primary key, nullable flag, and default are editable before the table is created. + +CSV parsing options: + +| Option | Description | Default | +|--------|-------------|---------| +| Delimiter | Comma, semicolon, tab, or pipe | Auto-detect | +| Quote character | Double or single quote | Double quote (`"`) | +| Encoding | UTF-8, ISO Latin 1, or Windows-1252 | Auto-detect | +| First row is a header | Use row 1 as column names; off imports every row as data | Yes | +| Trim leading and trailing spaces | Trim each field before import | No | +| Treat empty values as NULL | Insert NULL for empty fields instead of empty text | Yes | +| NULL text | An extra value imported as NULL, for example `\N` | None | + +Quoted fields keep embedded commas and newlines (RFC 4180), and doubled quotes (`""`) decode to a single quote. Rows insert through parameterized statements in batches. The on-error, transaction, and "delete existing rows" options work the same as SQL import. + ## Progress and Errors During import, a progress bar shows statements processed and overall completion. From 006fb241188f8b48749aab4ab61e66a4ffc5f4c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ng=C3=B4=20Qu=E1=BB=91c=20=C4=90=E1=BA=A1t?= Date: Fri, 5 Jun 2026 13:57:27 +0700 Subject: [PATCH 2/3] fix(plugins): pin import sheet column headers and compact CSV options layout --- .../CSVImportOptionsView.swift | 128 ++++++----- TablePro/Views/Import/RowImportSheet.swift | 204 +++++++++++------- 2 files changed, 199 insertions(+), 133 deletions(-) diff --git a/Plugins/CSVImportPlugin/CSVImportOptionsView.swift b/Plugins/CSVImportPlugin/CSVImportOptionsView.swift index 76d4d7d1f..28e9efa34 100644 --- a/Plugins/CSVImportPlugin/CSVImportOptionsView.swift +++ b/Plugins/CSVImportPlugin/CSVImportOptionsView.swift @@ -10,71 +10,87 @@ struct CSVImportOptionsView: View { let plugin: CSVImportPlugin var body: some View { - VStack(alignment: .leading, spacing: 12) { - Picker("Delimiter:", selection: Bindable(plugin).settings.delimiter) { - Text("Auto-detect").tag(CSVImportOptions.Delimiter.auto) - Text("Comma (,)").tag(CSVImportOptions.Delimiter.comma) - Text("Semicolon (;)").tag(CSVImportOptions.Delimiter.semicolon) - Text("Tab").tag(CSVImportOptions.Delimiter.tab) - Text("Pipe (|)").tag(CSVImportOptions.Delimiter.pipe) - } - .pickerStyle(.menu) - .font(.system(size: 13)) + HStack(alignment: .top, spacing: 32) { + Grid(alignment: .leading, horizontalSpacing: 8, verticalSpacing: 10) { + GridRow { + Text("Delimiter:") + .gridColumnAlignment(.trailing) + Picker("", selection: Bindable(plugin).settings.delimiter) { + Text("Auto-detect").tag(CSVImportOptions.Delimiter.auto) + Text("Comma (,)").tag(CSVImportOptions.Delimiter.comma) + Text("Semicolon (;)").tag(CSVImportOptions.Delimiter.semicolon) + Text("Tab").tag(CSVImportOptions.Delimiter.tab) + Text("Pipe (|)").tag(CSVImportOptions.Delimiter.pipe) + } + .pickerStyle(.menu) + .labelsHidden() + .frame(width: 170) + } - Picker("Quote character:", selection: Bindable(plugin).settings.quoteCharacter) { - Text("Double quote (\")").tag(CSVImportOptions.QuoteCharacter.doubleQuote) - Text("Single quote (')").tag(CSVImportOptions.QuoteCharacter.singleQuote) - } - .pickerStyle(.menu) - .font(.system(size: 13)) + GridRow { + Text("Quote character:") + Picker("", selection: Bindable(plugin).settings.quoteCharacter) { + Text("Double quote (\")").tag(CSVImportOptions.QuoteCharacter.doubleQuote) + Text("Single quote (')").tag(CSVImportOptions.QuoteCharacter.singleQuote) + } + .pickerStyle(.menu) + .labelsHidden() + .frame(width: 170) + } - Picker("Encoding:", selection: Bindable(plugin).settings.encoding) { - Text("Auto-detect").tag(CSVImportOptions.TextEncoding.auto) - Text("UTF-8").tag(CSVImportOptions.TextEncoding.utf8) - Text("ISO Latin 1").tag(CSVImportOptions.TextEncoding.isoLatin1) - Text("Windows-1252").tag(CSVImportOptions.TextEncoding.windowsCP1252) - } - .pickerStyle(.menu) - .font(.system(size: 13)) + GridRow { + Text("Encoding:") + Picker("", selection: Bindable(plugin).settings.encoding) { + Text("Auto-detect").tag(CSVImportOptions.TextEncoding.auto) + Text("UTF-8").tag(CSVImportOptions.TextEncoding.utf8) + Text("ISO Latin 1").tag(CSVImportOptions.TextEncoding.isoLatin1) + Text("Windows-1252").tag(CSVImportOptions.TextEncoding.windowsCP1252) + } + .pickerStyle(.menu) + .labelsHidden() + .frame(width: 170) + } - Toggle("First row is a header", isOn: Bindable(plugin).settings.hasHeaderRow) - .font(.system(size: 13)) - .help("Use the first row as column names. Turn off to import every row as data.") + GridRow { + Text("On error:") + Picker("", selection: Bindable(plugin).settings.errorHandling) { + Text("Stop and Rollback").tag(ImportErrorHandling.stopAndRollback) + Text("Stop and Commit").tag(ImportErrorHandling.stopAndCommit) + Text("Skip and Continue").tag(ImportErrorHandling.skipAndContinue) + } + .pickerStyle(.menu) + .labelsHidden() + .frame(width: 170) + } - Toggle("Trim leading and trailing spaces", isOn: Bindable(plugin).settings.trimWhitespace) - .font(.system(size: 13)) + GridRow { + Text("NULL text:") + TextField("", text: Bindable(plugin).settings.nullString, prompt: Text(verbatim: "\\N")) + .textFieldStyle(.roundedBorder) + .frame(width: 170) + .help("An extra value that should be imported as NULL, for example \\N.") + } + } - Toggle("Treat empty values as NULL", isOn: Bindable(plugin).settings.emptyAsNull) - .font(.system(size: 13)) - .help("Insert NULL for empty fields instead of an empty string.") + VStack(alignment: .leading, spacing: 10) { + Toggle("First row is a header", isOn: Bindable(plugin).settings.hasHeaderRow) + .help("Use the first row as column names. Turn off to import every row as data.") - HStack(spacing: 8) { - Text("NULL text:") - TextField("", text: Bindable(plugin).settings.nullString, prompt: Text(verbatim: "\\N")) - .textFieldStyle(.roundedBorder) - .frame(width: 120) - } - .font(.system(size: 13)) - .help("An extra value that should be imported as NULL, for example \\N.") + Toggle("Trim leading and trailing spaces", isOn: Bindable(plugin).settings.trimWhitespace) - Picker("On error:", selection: Bindable(plugin).settings.errorHandling) { - Text("Stop and Rollback").tag(ImportErrorHandling.stopAndRollback) - Text("Stop and Commit").tag(ImportErrorHandling.stopAndCommit) - Text("Skip and Continue").tag(ImportErrorHandling.skipAndContinue) - } - .pickerStyle(.menu) - .font(.system(size: 13)) + Toggle("Treat empty values as NULL", isOn: Bindable(plugin).settings.emptyAsNull) + .help("Insert NULL for empty fields instead of an empty string.") - Toggle("Wrap in transaction (BEGIN/COMMIT)", isOn: Bindable(plugin).settings.wrapInTransaction) - .font(.system(size: 13)) - .disabled(plugin.settings.errorHandling == .skipAndContinue) - .help(plugin.settings.errorHandling == .skipAndContinue - ? String(localized: "Not available in skip-and-continue mode") - : String(localized: "Insert all rows in a single transaction. If any row fails, all changes are rolled back.")) + Toggle("Wrap in transaction (BEGIN/COMMIT)", isOn: Bindable(plugin).settings.wrapInTransaction) + .disabled(plugin.settings.errorHandling == .skipAndContinue) + .help(plugin.settings.errorHandling == .skipAndContinue + ? String(localized: "Not available in skip-and-continue mode") + : String(localized: "Insert all rows in a single transaction. If any row fails, all changes are rolled back.")) - Toggle("Delete existing rows before import", isOn: Bindable(plugin).settings.deleteExistingRows) - .font(.system(size: 13)) - .help("Remove every row from the target table before inserting the imported rows.") + Toggle("Delete existing rows before import", isOn: Bindable(plugin).settings.deleteExistingRows) + .help("Remove every row from the target table before inserting the imported rows.") + } } + .font(.system(size: 13)) } } diff --git a/TablePro/Views/Import/RowImportSheet.swift b/TablePro/Views/Import/RowImportSheet.swift index 79ea59eaa..d7f562f73 100644 --- a/TablePro/Views/Import/RowImportSheet.swift +++ b/TablePro/Views/Import/RowImportSheet.swift @@ -86,7 +86,7 @@ struct RowImportSheet: View { footerView .padding() } - .frame(width: 720, height: 600) + .frame(width: 720, height: 640) .task { await loadTables() await loadNewColumns() @@ -241,94 +241,144 @@ struct RowImportSheet: View { } private var mappingTable: some View { - ScrollView { - Grid(alignment: .leading, horizontalSpacing: 12, verticalSpacing: 6) { - GridRow { - Toggle("", isOn: allMappingsIncluded) - .labelsHidden() - .help(String(localized: "Import all fields")) - Text("Field").font(.caption).foregroundStyle(.secondary) - Text("Column").font(.caption).foregroundStyle(.secondary) - } - Divider().gridCellColumns(3) - - ForEach(mappings) { row in - GridRow { - Toggle("", isOn: mappingBinding(row).include).labelsHidden() - VStack(alignment: .leading, spacing: 1) { - Text(row.field.name).lineLimit(1) - if let sample = row.field.sampleValue, !sample.isEmpty { - Text(sample).font(.caption).foregroundStyle(.secondary).lineLimit(1) - } - } - Picker("", selection: mappingBinding(row).targetColumn) { - Text("Skip").tag(String?.none) - ForEach(targetColumns, id: \.self) { column in - Text(column).tag(String?.some(column)) - } - } - .labelsHidden() - .frame(maxWidth: 240, alignment: .leading) - .disabled(!row.include) + VStack(spacing: 0) { + HStack(spacing: 12) { + Toggle("", isOn: allMappingsIncluded) + .labelsHidden() + .help(String(localized: "Import all fields")) + .frame(width: 16) + Text("Field") + .font(.caption) + .foregroundStyle(.secondary) + .frame(maxWidth: .infinity, alignment: .leading) + Text("Column") + .font(.caption) + .foregroundStyle(.secondary) + .frame(width: 240, alignment: .leading) + } + .padding(.horizontal) + .padding(.vertical, 6) + Divider() + + ScrollView { + VStack(spacing: 6) { + ForEach(mappings) { row in + mappingRow(row) } } + .padding(.horizontal) + .padding(.vertical, 8) } - .padding(.horizontal) - .padding(.vertical, 8) + } + } + + private func mappingRow(_ row: FieldMapping) -> some View { + HStack(spacing: 12) { + Toggle("", isOn: mappingBinding(row).include) + .labelsHidden() + .frame(width: 16) + VStack(alignment: .leading, spacing: 1) { + Text(row.field.name).lineLimit(1) + if let sample = row.field.sampleValue, !sample.isEmpty { + Text(sample).font(.caption).foregroundStyle(.secondary).lineLimit(1) + } + } + .frame(maxWidth: .infinity, alignment: .leading) + Picker("", selection: mappingBinding(row).targetColumn) { + Text("Skip").tag(String?.none) + ForEach(targetColumns, id: \.self) { column in + Text(column).tag(String?.some(column)) + } + } + .labelsHidden() + .frame(width: 240, alignment: .leading) + .disabled(!row.include) } } private var newColumnsTable: some View { - ScrollView { - Grid(alignment: .leading, horizontalSpacing: 10, verticalSpacing: 6) { - GridRow { - Toggle("", isOn: allColumnsIncluded) - .labelsHidden() - .help(String(localized: "Create all columns")) - Text("Column").font(.caption).foregroundStyle(.secondary) - Text("Type").font(.caption).foregroundStyle(.secondary) - Text("Key").font(.caption).foregroundStyle(.secondary) - Text("Null").font(.caption).foregroundStyle(.secondary) - Text("Default").font(.caption).foregroundStyle(.secondary) + VStack(spacing: 0) { + HStack(spacing: 10) { + Toggle("", isOn: allColumnsIncluded) + .labelsHidden() + .help(String(localized: "Create all columns")) + .frame(width: 16) + Text("Column") + .font(.caption) + .foregroundStyle(.secondary) + .frame(width: 150, alignment: .leading) + Text("Type") + .font(.caption) + .foregroundStyle(.secondary) + .frame(width: 150, alignment: .leading) + Text("Key") + .font(.caption) + .foregroundStyle(.secondary) + .frame(width: 30) + Text("Null") + .font(.caption) + .foregroundStyle(.secondary) + .frame(width: 30) + Text("Default") + .font(.caption) + .foregroundStyle(.secondary) + .frame(maxWidth: .infinity, alignment: .leading) + } + .padding(.horizontal) + .padding(.vertical, 6) + Divider() + + ScrollView { + VStack(spacing: 6) { + ForEach(newColumns) { row in + newColumnRow(row) + } } - Divider().gridCellColumns(6) - - ForEach(newColumns) { row in - GridRow { - Toggle("", isOn: columnBinding(row).include).labelsHidden() - TextField("name", text: columnBinding(row).name) - .textFieldStyle(.roundedBorder) - .frame(width: 150) - .disabled(!row.include) - Menu { - ForEach(typeOptions(including: row.type), id: \.self) { type in - Button { - columnBinding(row).type.wrappedValue = type - } label: { - if type.caseInsensitiveCompare(row.type) == .orderedSame { - Label(type, systemImage: "checkmark") - } else { - Text(type) - } - } - } - } label: { - Text(row.type) - .frame(maxWidth: .infinity, alignment: .leading) + .padding(.horizontal) + .padding(.vertical, 8) + } + } + } + + private func newColumnRow(_ row: NewColumn) -> some View { + HStack(spacing: 10) { + Toggle("", isOn: columnBinding(row).include) + .labelsHidden() + .frame(width: 16) + TextField("name", text: columnBinding(row).name) + .textFieldStyle(.roundedBorder) + .frame(width: 150) + .disabled(!row.include) + Menu { + ForEach(typeOptions(including: row.type), id: \.self) { type in + Button { + columnBinding(row).type.wrappedValue = type + } label: { + if type.caseInsensitiveCompare(row.type) == .orderedSame { + Label(type, systemImage: "checkmark") + } else { + Text(type) } - .frame(width: 150) - .disabled(!row.include) - Toggle("", isOn: columnBinding(row).isPrimaryKey).labelsHidden().disabled(!row.include) - Toggle("", isOn: columnBinding(row).isNullable).labelsHidden().disabled(!row.include) - TextField("", text: columnBinding(row).defaultValue) - .textFieldStyle(.roundedBorder) - .frame(minWidth: 120) - .disabled(!row.include) } } + } label: { + Text(row.type) + .frame(maxWidth: .infinity, alignment: .leading) } - .padding(.horizontal) - .padding(.vertical, 8) + .frame(width: 150) + .disabled(!row.include) + Toggle("", isOn: columnBinding(row).isPrimaryKey) + .labelsHidden() + .frame(width: 30) + .disabled(!row.include) + Toggle("", isOn: columnBinding(row).isNullable) + .labelsHidden() + .frame(width: 30) + .disabled(!row.include) + TextField("", text: columnBinding(row).defaultValue) + .textFieldStyle(.roundedBorder) + .frame(maxWidth: .infinity) + .disabled(!row.include) } } From 8578ce16b7a489a1ed8e47b27072c0f62f2df333 Mon Sep 17 00:00:00 2001 From: Ngo Quoc Dat Date: Sat, 6 Jun 2026 02:52:59 +0700 Subject: [PATCH 3/3] fix(plugins): run JSON and CSV imports through a shared runner that fixes rollback, duplicates, and stop-and-commit --- CHANGELOG.md | 3 + .../CSVImportPlugin/CSVImportParsing.swift | 9 +- Plugins/CSVImportPlugin/CSVImportPlugin.swift | 139 +++--------- .../JSONImportPlugin/JSONImportPlugin.swift | 156 +++----------- .../CSVStreamingParser.swift | 7 +- .../TableProPluginKit/RowImportRunner.swift | 160 ++++++++++++++ .../Plugins/CSVImportPluginTests.swift | 22 ++ .../Plugins/RowImportRunnerTests.swift | 200 ++++++++++++++++++ 8 files changed, 459 insertions(+), 237 deletions(-) create mode 100644 Plugins/TableProPluginKit/RowImportRunner.swift create mode 100644 TableProTests/Plugins/RowImportRunnerTests.swift diff --git a/CHANGELOG.md b/CHANGELOG.md index ab3b995d6..3be2ac070 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,6 +35,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- JSON import: "Delete existing rows before import" now runs inside the import transaction, so a failed import restores the deleted rows instead of leaving the table emptied. +- JSON import: skip-and-continue mode no longer inserts duplicate rows when part of a batch had already been written before an error. +- JSON import: "Stop and Commit" now keeps the rows inserted before the error instead of rolling them back. - Opening the connection or database switcher now puts the cursor in its search field even while a filter input is being edited; the filter text is kept. (#1575) - TablePro no longer shows its icon for .sql, .sqlite, and .duckdb files in Finder when it is not the default app for those types. (#1594) - The JSON results view shows row data right away instead of staying blank until you switch between Tree and Text, and it updates when the row selection changes. A spinner shows while large results are being formatted. (#1576) diff --git a/Plugins/CSVImportPlugin/CSVImportParsing.swift b/Plugins/CSVImportPlugin/CSVImportParsing.swift index 8d93dffb2..a5e0f1017 100644 --- a/Plugins/CSVImportPlugin/CSVImportParsing.swift +++ b/Plugins/CSVImportPlugin/CSVImportParsing.swift @@ -64,6 +64,11 @@ enum CSVImportParsing { return .text(value) } + static func sampleText(from raw: String, options: CSVImportOptions) -> String? { + guard case .text(let value) = cellValue(from: raw, options: options), !value.isEmpty else { return nil } + return value + } + static func row(fields: [String], columnNames: [String], options: CSVImportOptions) -> [String: PluginCellValue] { var row: [String: PluginCellValue] = [:] row.reserveCapacity(columnNames.count) @@ -125,8 +130,8 @@ enum CSVImportParsing { let fields = parser.parseRow(buffer, range: range) if isBlank(fields) { continue } for column in 0.. PluginImportResult { let startTime = Date() let url = source.fileURL() - let useTransaction = settings.wrapInTransaction && settings.errorHandling != .skipAndContinue let data: Data do { @@ -67,58 +62,40 @@ final class CSVImportPlugin: ImportFormatPlugin, SettablePlugin { progress.setEstimatedTotal(dataRanges.count) - var inserted = 0 - var skipped = 0 - var errors: [PluginImportResult.ImportStatementError] = [] - - do { - if settings.deleteExistingRows { - try await sink.deleteAllRowsFromTargetTable() - } - if useTransaction { - try await sink.beginTransaction() - } - - let lineOffset = hasHeader ? 2 : 1 - var index = 0 - while index < dataRanges.count { - try progress.checkCancellation() - let end = min(index + Self.batchSize, dataRanges.count) - let batch = parseBatch( - in: data, - parser: parser, - ranges: dataRanges[index.. 0 { + progress.incrementStatement(by: blankRows) } - - if useTransaction { - try await sink.commitTransaction() - } - } catch { - if useTransaction { - do { - try await sink.rollbackTransaction() - } catch { - Self.logger.warning("Rollback after failed import also failed: \(error.localizedDescription)") - } - } - if error is PluginImportCancellationError { throw error } - if error is PluginImportError { throw error } - throw PluginImportError.importFailed(error.localizedDescription) + cursor = end + return batch } - progress.finalize() return PluginImportResult( - executedStatements: inserted, + executedStatements: outcome.inserted, executionTime: Date().timeIntervalSince(startTime), - skippedStatements: skipped, - errors: errors + skippedStatements: outcome.skipped, + errors: outcome.errors ) } @@ -166,64 +143,6 @@ final class CSVImportPlugin: ImportFormatPlugin, SettablePlugin { } } - private func flush( - _ batch: [(line: Int, row: [String: PluginCellValue])], - into sink: any PluginImportDataSink, - progress: PluginImportProgress, - inserted: inout Int, - skipped: inout Int, - errors: inout [PluginImportResult.ImportStatementError] - ) async throws { - guard !batch.isEmpty else { return } - do { - try await sink.insertRows(batch.map(\.row)) - inserted += batch.count - progress.incrementStatement(by: batch.count) - } catch { - switch settings.errorHandling { - case .stopAndRollback, .stopAndCommit: - let firstLine = batch.first?.line ?? 0 - throw PluginImportError.statementFailed( - statement: "rows \(firstLine)-\(batch.last?.line ?? firstLine)", - line: firstLine, - underlyingError: error - ) - case .skipAndContinue: - for entry in batch { - try await insert(entry.row, into: sink, at: entry.line, progress: progress, - inserted: &inserted, skipped: &skipped, errors: &errors) - } - } - } - } - - private func insert( - _ row: [String: PluginCellValue], - into sink: any PluginImportDataSink, - at line: Int, - progress: PluginImportProgress, - inserted: inout Int, - skipped: inout Int, - errors: inout [PluginImportResult.ImportStatementError] - ) async throws { - do { - try await sink.insertRow(row) - inserted += 1 - progress.incrementStatement() - } catch { - switch settings.errorHandling { - case .stopAndRollback, .stopAndCommit: - throw PluginImportError.statementFailed(statement: "row \(line)", line: line, underlyingError: error) - case .skipAndContinue: - skipped += 1 - if errors.count < Self.maxErrors { - errors.append(.init(statement: "row \(line)", line: line, errorMessage: error.localizedDescription)) - } - progress.incrementStatement() - } - } - } - func detectSourceFields(at url: URL, targetTable: String?) throws -> [PluginImportField] { let data = try readDetectionPrefix(of: url) return CSVImportParsing.detectFields(in: data, options: settings) diff --git a/Plugins/JSONImportPlugin/JSONImportPlugin.swift b/Plugins/JSONImportPlugin/JSONImportPlugin.swift index d5a5cb923..bc13faf63 100644 --- a/Plugins/JSONImportPlugin/JSONImportPlugin.swift +++ b/Plugins/JSONImportPlugin/JSONImportPlugin.swift @@ -4,14 +4,11 @@ // import Foundation -import os import SwiftUI import TableProPluginKit @Observable final class JSONImportPlugin: ImportFormatPlugin, SettablePlugin { - private static let logger = Logger(subsystem: "com.TablePro", category: "JSONImportPlugin") - static let pluginName = "JSON Import" static let pluginVersion = "1.0.0" static let pluginDescription = "Import data from JSON files" @@ -38,6 +35,8 @@ final class JSONImportPlugin: ImportFormatPlugin, SettablePlugin { settings = JSONImportOptions() } + private static let batchSize = 500 + func performImport( source: any PluginImportSource, sink: any PluginImportDataSink, @@ -45,141 +44,54 @@ final class JSONImportPlugin: ImportFormatPlugin, SettablePlugin { ) async throws -> PluginImportResult { let startTime = Date() let url = source.fileURL() - let useTransaction = settings.wrapInTransaction && settings.errorHandling != .skipAndContinue - - let batchSize = 500 - var inserted = 0 - var skipped = 0 - var errors: [PluginImportResult.ImportStatementError] = [] - let maxErrors = 1_000 - var batch: [(line: Int, row: [String: PluginCellValue])] = [] - - do { - if settings.deleteExistingRows { - try await sink.deleteAllRowsFromTargetTable() - } - if useTransaction { - try await sink.beginTransaction() - } + let configuration = RowImportRunner.Configuration( + errorHandling: settings.errorHandling, + wrapInTransaction: settings.wrapInTransaction, + deleteExistingRows: settings.deleteExistingRows + ) - if JSONImportParsing.isLineDelimited(url) { - progress.setEstimatedTotal(max(1, Int(source.fileSizeBytes() / 256))) - var lineNumber = 0 - for try await line in url.lines { - try progress.checkCancellation() + let outcome: RowImportRunner.Outcome + if JSONImportParsing.isLineDelimited(url) { + progress.setEstimatedTotal(max(1, Int(source.fileSizeBytes() / 256))) + var lines = url.lines.makeAsyncIterator() + var lineNumber = 0 + outcome = try await RowImportRunner.run( + configuration: configuration, sink: sink, progress: progress + ) { + var batch: [RowImportRunner.Entry] = [] + while batch.count < Self.batchSize, let line = try await lines.next() { lineNumber += 1 let trimmed = line.trimmingCharacters(in: .whitespacesAndNewlines) guard !trimmed.isEmpty else { continue } batch.append((lineNumber, try JSONImportParsing.parseRow(fromLine: trimmed))) - if batch.count >= batchSize { - try await flush(&batch, into: sink, progress: progress, - inserted: &inserted, skipped: &skipped, errors: &errors, maxErrors: maxErrors) - } - } - } else { - let rawRows = try JSONImportParsing.parseRows(at: url, targetTable: sink.targetTable) - progress.setEstimatedTotal(rawRows.count) - for (index, rawRow) in rawRows.enumerated() { - try progress.checkCancellation() - batch.append((index + 1, JSONImportParsing.convertRow(rawRow))) - if batch.count >= batchSize { - try await flush(&batch, into: sink, progress: progress, - inserted: &inserted, skipped: &skipped, errors: &errors, maxErrors: maxErrors) - } } + return batch.isEmpty ? nil : batch } - - try await flush(&batch, into: sink, progress: progress, - inserted: &inserted, skipped: &skipped, errors: &errors, maxErrors: maxErrors) - - if useTransaction { - try await sink.commitTransaction() - } - } catch { - if useTransaction { - do { - try await sink.rollbackTransaction() - } catch { - Self.logger.warning("Rollback after failed import also failed: \(error.localizedDescription)") + } else { + let rawRows = try JSONImportParsing.parseRows(at: url, targetTable: sink.targetTable) + progress.setEstimatedTotal(rawRows.count) + var cursor = 0 + outcome = try await RowImportRunner.run( + configuration: configuration, sink: sink, progress: progress + ) { + guard cursor < rawRows.count else { return nil } + let end = min(cursor + Self.batchSize, rawRows.count) + let batch = (cursor.. [PluginImportField] { diff --git a/Plugins/TableProPluginKit/CSVStreamingParser.swift b/Plugins/TableProPluginKit/CSVStreamingParser.swift index 0bf9451a1..0990148f1 100644 --- a/Plugins/TableProPluginKit/CSVStreamingParser.swift +++ b/Plugins/TableProPluginKit/CSVStreamingParser.swift @@ -165,9 +165,10 @@ public struct CSVStreamingParser: Sendable { private func decode(_ bytes: [UInt8]) -> String { if bytes.isEmpty { return "" } - return String(bytes: bytes, encoding: dialect.encoding) - ?? String(bytes: bytes, encoding: .isoLatin1) - ?? "" + if let decoded = String(bytes: bytes, encoding: dialect.encoding) { + return decoded + } + return String(bytes.map { Character(UnicodeScalar($0)) }) } private func bomSkip(in bytes: UnsafeBufferPointer) -> Int { diff --git a/Plugins/TableProPluginKit/RowImportRunner.swift b/Plugins/TableProPluginKit/RowImportRunner.swift new file mode 100644 index 000000000..5d32e0a9a --- /dev/null +++ b/Plugins/TableProPluginKit/RowImportRunner.swift @@ -0,0 +1,160 @@ +// +// RowImportRunner.swift +// TableProPluginKit +// +// Shared orchestration for row-based importers (JSON, CSV): transaction +// lifecycle, delete-before-import, batching, per-mode error handling, and +// progress. Format plugins supply rows through the nextBatch closure. +// + +import Foundation +import os + +public enum RowImportRunner { + public typealias Entry = (line: Int, row: [String: PluginCellValue]) + + public struct Configuration: Sendable { + public var errorHandling: ImportErrorHandling + public var wrapInTransaction: Bool + public var deleteExistingRows: Bool + public var maxRecordedErrors: Int + + public init( + errorHandling: ImportErrorHandling, + wrapInTransaction: Bool, + deleteExistingRows: Bool, + maxRecordedErrors: Int = 1_000 + ) { + self.errorHandling = errorHandling + self.wrapInTransaction = wrapInTransaction + self.deleteExistingRows = deleteExistingRows + self.maxRecordedErrors = maxRecordedErrors + } + } + + public struct Outcome: Sendable { + public let inserted: Int + public let skipped: Int + public let errors: [PluginImportResult.ImportStatementError] + } + + private static let logger = Logger(subsystem: "com.TablePro", category: "RowImportRunner") + + public static func run( + configuration: Configuration, + sink: any PluginImportDataSink, + progress: PluginImportProgress, + nextBatch: () async throws -> [Entry]? + ) async throws -> Outcome { + let useTransaction = configuration.wrapInTransaction + && configuration.errorHandling != .skipAndContinue + var inserted = 0 + var skipped = 0 + var errors: [PluginImportResult.ImportStatementError] = [] + + do { + if useTransaction { + try await sink.beginTransaction() + } + if configuration.deleteExistingRows { + try await sink.deleteAllRowsFromTargetTable() + } + + while let batch = try await nextBatch() { + try progress.checkCancellation() + guard !batch.isEmpty else { continue } + if configuration.errorHandling == .skipAndContinue { + try await insertSkippingErrors( + batch, into: sink, progress: progress, configuration: configuration, + inserted: &inserted, skipped: &skipped, errors: &errors + ) + } else { + try await insertBatch(batch, into: sink, progress: progress, inserted: &inserted) + } + } + + if useTransaction { + try await sink.commitTransaction() + } + } catch { + try await conclude(after: error, sink: sink, useTransaction: useTransaction, configuration: configuration) + } + + progress.finalize() + return Outcome(inserted: inserted, skipped: skipped, errors: errors) + } + + private static func insertBatch( + _ batch: [Entry], + into sink: any PluginImportDataSink, + progress: PluginImportProgress, + inserted: inout Int + ) async throws { + do { + try await sink.insertRows(batch.map(\.row)) + inserted += batch.count + progress.incrementStatement(by: batch.count) + } catch { + let firstLine = batch.first?.line ?? 0 + throw PluginImportError.statementFailed( + statement: "rows \(firstLine)-\(batch.last?.line ?? firstLine)", + line: firstLine, + underlyingError: error + ) + } + } + + private static func insertSkippingErrors( + _ batch: [Entry], + into sink: any PluginImportDataSink, + progress: PluginImportProgress, + configuration: Configuration, + inserted: inout Int, + skipped: inout Int, + errors: inout [PluginImportResult.ImportStatementError] + ) async throws { + for entry in batch { + try progress.checkCancellation() + do { + try await sink.insertRow(entry.row) + inserted += 1 + } catch { + skipped += 1 + if errors.count < configuration.maxRecordedErrors { + errors.append(.init( + statement: "row \(entry.line)", + line: entry.line, + errorMessage: error.localizedDescription + )) + } + } + progress.incrementStatement() + } + } + + private static func conclude( + after error: Error, + sink: any PluginImportDataSink, + useTransaction: Bool, + configuration: Configuration + ) async throws -> Never { + if useTransaction { + if configuration.errorHandling == .stopAndCommit, !(error is PluginImportCancellationError) { + do { + try await sink.commitTransaction() + } catch { + logger.warning("Commit of partial import failed: \(error.localizedDescription)") + } + } else { + do { + try await sink.rollbackTransaction() + } catch { + logger.warning("Rollback after failed import also failed: \(error.localizedDescription)") + } + } + } + if error is PluginImportCancellationError { throw error } + if error is PluginImportError { throw error } + throw PluginImportError.importFailed(error.localizedDescription) + } +} diff --git a/TableProTests/Plugins/CSVImportPluginTests.swift b/TableProTests/Plugins/CSVImportPluginTests.swift index 91509703f..c01a965be 100644 --- a/TableProTests/Plugins/CSVImportPluginTests.swift +++ b/TableProTests/Plugins/CSVImportPluginTests.swift @@ -185,4 +185,26 @@ struct CSVImportPluginTests { let result = CSVImportParsing.detectFields(in: data("a;b;c\n1;2;3\n"), options: CSVImportOptions()) #expect(result.map(\.name) == ["a", "b", "c"]) } + + @Test("Trim option applies during detection, matching imported values") + func testDetectTrimAffectsInference() { + let csv = "n\n 1 \n 2 \n" + var options = CSVImportOptions() + options.trimWhitespace = true + let trimmed = CSVImportParsing.detectFields(in: data(csv), options: options) + #expect(fields("n", trimmed)?.inferredType == .integer) + #expect(fields("n", trimmed)?.sampleValue == "1") + + let untrimmed = CSVImportParsing.detectFields(in: data(csv), options: CSVImportOptions()) + #expect(fields("n", untrimmed)?.inferredType == .text) + } + + @Test("NULL token values are excluded from detection samples") + func testDetectNullTokenExcluded() { + var options = CSVImportOptions() + options.nullString = "\\N" + let result = CSVImportParsing.detectFields(in: data("n\n\\N\n5\n"), options: options) + #expect(fields("n", result)?.inferredType == .integer) + #expect(fields("n", result)?.sampleValue == "5") + } } diff --git a/TableProTests/Plugins/RowImportRunnerTests.swift b/TableProTests/Plugins/RowImportRunnerTests.swift new file mode 100644 index 000000000..1b93d854a --- /dev/null +++ b/TableProTests/Plugins/RowImportRunnerTests.swift @@ -0,0 +1,200 @@ +// +// RowImportRunnerTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +private struct MockSinkError: Error {} + +private final class MockImportSink: PluginImportDataSink, @unchecked Sendable { + let databaseTypeId = "mock" + let targetTable: String? = "table" + + private(set) var calls: [String] = [] + var onInsertRow: ([String: PluginCellValue]) throws -> Void = { _ in } + var onInsertRows: ([[String: PluginCellValue]]) throws -> Void = { _ in } + + func execute(statement: String) async throws { + calls.append("execute") + } + + func insertRow(_ values: [String: PluginCellValue]) async throws { + calls.append("insertRow") + try onInsertRow(values) + } + + func insertRows(_ rows: [[String: PluginCellValue]]) async throws { + calls.append("insertRows(\(rows.count))") + try onInsertRows(rows) + } + + func deleteAllRowsFromTargetTable() async throws { + calls.append("deleteAll") + } + + func beginTransaction() async throws { + calls.append("begin") + } + + func commitTransaction() async throws { + calls.append("commit") + } + + func rollbackTransaction() async throws { + calls.append("rollback") + } + + func disableForeignKeyChecks() async throws {} + func enableForeignKeyChecks() async throws {} +} + +@Suite("Row Import Runner") +struct RowImportRunnerTests { + private func entry(_ line: Int, _ value: String = "v") -> RowImportRunner.Entry { + (line, ["c": .text(value)]) + } + + private func makeProgress() -> PluginImportProgress { + PluginImportProgress(progress: Progress()) + } + + private func provider(_ groups: [[RowImportRunner.Entry]]) -> () async throws -> [RowImportRunner.Entry]? { + var remaining = groups + return { + remaining.isEmpty ? nil : remaining.removeFirst() + } + } + + private func configuration( + _ errorHandling: ImportErrorHandling, + wrapInTransaction: Bool = true, + deleteExistingRows: Bool = false, + maxRecordedErrors: Int = 1_000 + ) -> RowImportRunner.Configuration { + RowImportRunner.Configuration( + errorHandling: errorHandling, + wrapInTransaction: wrapInTransaction, + deleteExistingRows: deleteExistingRows, + maxRecordedErrors: maxRecordedErrors + ) + } + + @Test("Delete existing rows runs inside the transaction") + func testDeleteRunsInsideTransaction() async throws { + let sink = MockImportSink() + let outcome = try await RowImportRunner.run( + configuration: configuration(.stopAndRollback, deleteExistingRows: true), + sink: sink, + progress: makeProgress(), + nextBatch: provider([[entry(1), entry(2)]]) + ) + #expect(sink.calls == ["begin", "deleteAll", "insertRows(2)", "commit"]) + #expect(outcome.inserted == 2) + } + + @Test("Stop and rollback rolls back and reports the failed row range") + func testStopAndRollbackRollsBack() async throws { + let sink = MockImportSink() + sink.onInsertRows = { _ in throw MockSinkError() } + await #expect(throws: PluginImportError.self) { + _ = try await RowImportRunner.run( + configuration: configuration(.stopAndRollback), + sink: sink, + progress: makeProgress(), + nextBatch: provider([[entry(1), entry(2)]]) + ) + } + #expect(sink.calls.contains("rollback")) + #expect(!sink.calls.contains("commit")) + } + + @Test("Stop and commit keeps rows inserted before the error") + func testStopAndCommitCommitsPartialWork() async throws { + let sink = MockImportSink() + var batchesSeen = 0 + sink.onInsertRows = { _ in + batchesSeen += 1 + if batchesSeen == 2 { throw MockSinkError() } + } + await #expect(throws: PluginImportError.self) { + _ = try await RowImportRunner.run( + configuration: configuration(.stopAndCommit), + sink: sink, + progress: makeProgress(), + nextBatch: provider([[entry(1)], [entry(2)]]) + ) + } + #expect(sink.calls.contains("commit")) + #expect(!sink.calls.contains("rollback")) + } + + @Test("Skip mode inserts row by row, never retries a batch, and skips only failures") + func testSkipModeInsertsRowByRowWithoutBatchRetry() async throws { + let sink = MockImportSink() + sink.onInsertRow = { values in + if values["c"] == .text("bad") { throw MockSinkError() } + } + let outcome = try await RowImportRunner.run( + configuration: configuration(.skipAndContinue), + sink: sink, + progress: makeProgress(), + nextBatch: provider([[entry(1), entry(2, "bad"), entry(3)]]) + ) + #expect(outcome.inserted == 2) + #expect(outcome.skipped == 1) + #expect(outcome.errors.map(\.line) == [2]) + #expect(sink.calls.filter { $0 == "insertRow" }.count == 3) + #expect(!sink.calls.contains { $0.hasPrefix("insertRows") }) + #expect(!sink.calls.contains("begin")) + #expect(!sink.calls.contains("commit")) + #expect(!sink.calls.contains("rollback")) + } + + @Test("Skip mode caps recorded errors but keeps counting skips") + func testErrorCapLimitsRecordedErrors() async throws { + let sink = MockImportSink() + sink.onInsertRow = { _ in throw MockSinkError() } + let outcome = try await RowImportRunner.run( + configuration: configuration(.skipAndContinue, maxRecordedErrors: 2), + sink: sink, + progress: makeProgress(), + nextBatch: provider([[entry(1), entry(2), entry(3), entry(4)]]) + ) + #expect(outcome.inserted == 0) + #expect(outcome.skipped == 4) + #expect(outcome.errors.count == 2) + } + + @Test("Cancellation rolls back even in stop-and-commit mode") + func testCancellationRollsBack() async throws { + let sink = MockImportSink() + let progress = makeProgress() + progress.cancel() + await #expect(throws: PluginImportCancellationError.self) { + _ = try await RowImportRunner.run( + configuration: configuration(.stopAndCommit), + sink: sink, + progress: progress, + nextBatch: provider([[entry(1)]]) + ) + } + #expect(sink.calls.contains("rollback")) + #expect(!sink.calls.contains("commit")) + } + + @Test("Transaction is skipped when wrap is off") + func testNoTransactionWhenWrapOff() async throws { + let sink = MockImportSink() + let outcome = try await RowImportRunner.run( + configuration: configuration(.stopAndRollback, wrapInTransaction: false), + sink: sink, + progress: makeProgress(), + nextBatch: provider([[entry(1)]]) + ) + #expect(outcome.inserted == 1) + #expect(sink.calls == ["insertRows(1)"]) + } +}