diff --git a/CHANGELOG.md b/CHANGELOG.md index 7214be899..3be2ac070 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Added +- Import data from CSV and TSV files into a table: map columns to an existing table or create a new one, with options for delimiter, quote character, encoding, header row, and empty/NULL handling. (#1568) - SQL autocomplete completes database, schema, and table names at each segment of qualified names for schema-organized connections (Snowflake, BigQuery), fetches tables of unopened schemas on demand, resolves alias columns for schema-qualified tables, and suggests the active connection's full dialect function list. - Each filter row has a checkbox to turn it on or off and an Apply button to filter by just that row. The main Apply runs every active filter, and disabled filters stay in the panel for later. (#1561) - Importing connections from other apps now detects duplicates by host, port, database, and username, and lets you replace, add a copy, or skip each one before import. @@ -34,6 +35,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Fixed +- JSON import: "Delete existing rows before import" now runs inside the import transaction, so a failed import restores the deleted rows instead of leaving the table emptied. +- JSON import: skip-and-continue mode no longer inserts duplicate rows when part of a batch had already been written before an error. +- JSON import: "Stop and Commit" now keeps the rows inserted before the error instead of rolling them back. - Opening the connection or database switcher now puts the cursor in its search field even while a filter input is being edited; the filter text is kept. (#1575) - TablePro no longer shows its icon for .sql, .sqlite, and .duckdb files in Finder when it is not the default app for those types. (#1594) - The JSON results view shows row data right away instead of staying blank until you switch between Tree and Text, and it updates when the row selection changes. A spinner shows while large results are being formatted. (#1576) diff --git a/Plugins/CSVImportPlugin/CSVImportOptions.swift b/Plugins/CSVImportPlugin/CSVImportOptions.swift new file mode 100644 index 000000000..b8fded4ca --- /dev/null +++ b/Plugins/CSVImportPlugin/CSVImportOptions.swift @@ -0,0 +1,84 @@ +// +// CSVImportOptions.swift +// CSVImportPlugin +// + +import Foundation +import TableProPluginKit + +struct CSVImportOptions: Equatable, Codable { + enum Delimiter: String, Codable, CaseIterable, Identifiable { + case auto + case comma + case semicolon + case tab + case pipe + + var id: String { rawValue } + + var byte: UInt8? { + switch self { + case .auto: return nil + case .comma: return 0x2C + case .semicolon: return 0x3B + case .tab: return 0x09 + case .pipe: return 0x7C + } + } + } + + enum QuoteCharacter: String, Codable, CaseIterable, Identifiable { + case doubleQuote + case singleQuote + + var id: String { rawValue } + + var byte: UInt8 { + switch self { + case .doubleQuote: return 0x22 + case .singleQuote: return 0x27 + } + } + } + + enum TextEncoding: String, Codable, CaseIterable, Identifiable { + case auto + case utf8 + case isoLatin1 + case windowsCP1252 + + var id: String { rawValue } + + var stringEncoding: String.Encoding? { + switch self { + case .auto: return nil + case .utf8: return .utf8 + case .isoLatin1: return .isoLatin1 + case .windowsCP1252: return .windowsCP1252 + } + } + } + + var delimiter: Delimiter = .auto + var quoteCharacter: QuoteCharacter = .doubleQuote + var encoding: TextEncoding = .auto + var hasHeaderRow: Bool = true + var trimWhitespace: Bool = false + var emptyAsNull: Bool = true + var nullString: String = "" + var errorHandling: ImportErrorHandling = .stopAndRollback + var wrapInTransaction: Bool = true + var deleteExistingRows: Bool = false + + var detectionSignature: String { + [ + delimiter.rawValue, + quoteCharacter.rawValue, + encoding.rawValue, + hasHeaderRow ? "h1" : "h0", + trimWhitespace ? "t1" : "t0", + emptyAsNull ? "n1" : "n0", + nullString + ].joined(separator: "|") + } +} diff --git a/Plugins/CSVImportPlugin/CSVImportOptionsView.swift b/Plugins/CSVImportPlugin/CSVImportOptionsView.swift new file mode 100644 index 000000000..28e9efa34 --- /dev/null +++ b/Plugins/CSVImportPlugin/CSVImportOptionsView.swift @@ -0,0 +1,96 @@ +// +// CSVImportOptionsView.swift +// CSVImportPlugin +// + +import SwiftUI +import TableProPluginKit + +struct CSVImportOptionsView: View { + let plugin: CSVImportPlugin + + var body: some View { + HStack(alignment: .top, spacing: 32) { + Grid(alignment: .leading, horizontalSpacing: 8, verticalSpacing: 10) { + GridRow { + Text("Delimiter:") + .gridColumnAlignment(.trailing) + Picker("", selection: Bindable(plugin).settings.delimiter) { + Text("Auto-detect").tag(CSVImportOptions.Delimiter.auto) + Text("Comma (,)").tag(CSVImportOptions.Delimiter.comma) + Text("Semicolon (;)").tag(CSVImportOptions.Delimiter.semicolon) + Text("Tab").tag(CSVImportOptions.Delimiter.tab) + Text("Pipe (|)").tag(CSVImportOptions.Delimiter.pipe) + } + .pickerStyle(.menu) + .labelsHidden() + .frame(width: 170) + } + + GridRow { + Text("Quote character:") + Picker("", selection: Bindable(plugin).settings.quoteCharacter) { + Text("Double quote (\")").tag(CSVImportOptions.QuoteCharacter.doubleQuote) + Text("Single quote (')").tag(CSVImportOptions.QuoteCharacter.singleQuote) + } + .pickerStyle(.menu) + .labelsHidden() + .frame(width: 170) + } + + GridRow { + Text("Encoding:") + Picker("", selection: Bindable(plugin).settings.encoding) { + Text("Auto-detect").tag(CSVImportOptions.TextEncoding.auto) + Text("UTF-8").tag(CSVImportOptions.TextEncoding.utf8) + Text("ISO Latin 1").tag(CSVImportOptions.TextEncoding.isoLatin1) + Text("Windows-1252").tag(CSVImportOptions.TextEncoding.windowsCP1252) + } + .pickerStyle(.menu) + .labelsHidden() + .frame(width: 170) + } + + GridRow { + Text("On error:") + Picker("", selection: Bindable(plugin).settings.errorHandling) { + Text("Stop and Rollback").tag(ImportErrorHandling.stopAndRollback) + Text("Stop and Commit").tag(ImportErrorHandling.stopAndCommit) + Text("Skip and Continue").tag(ImportErrorHandling.skipAndContinue) + } + .pickerStyle(.menu) + .labelsHidden() + .frame(width: 170) + } + + GridRow { + Text("NULL text:") + TextField("", text: Bindable(plugin).settings.nullString, prompt: Text(verbatim: "\\N")) + .textFieldStyle(.roundedBorder) + .frame(width: 170) + .help("An extra value that should be imported as NULL, for example \\N.") + } + } + + VStack(alignment: .leading, spacing: 10) { + Toggle("First row is a header", isOn: Bindable(plugin).settings.hasHeaderRow) + .help("Use the first row as column names. Turn off to import every row as data.") + + Toggle("Trim leading and trailing spaces", isOn: Bindable(plugin).settings.trimWhitespace) + + Toggle("Treat empty values as NULL", isOn: Bindable(plugin).settings.emptyAsNull) + .help("Insert NULL for empty fields instead of an empty string.") + + Toggle("Wrap in transaction (BEGIN/COMMIT)", isOn: Bindable(plugin).settings.wrapInTransaction) + .disabled(plugin.settings.errorHandling == .skipAndContinue) + .help(plugin.settings.errorHandling == .skipAndContinue + ? String(localized: "Not available in skip-and-continue mode") + : String(localized: "Insert all rows in a single transaction. If any row fails, all changes are rolled back.")) + + Toggle("Delete existing rows before import", isOn: Bindable(plugin).settings.deleteExistingRows) + .help("Remove every row from the target table before inserting the imported rows.") + } + } + .font(.system(size: 13)) + } +} diff --git a/Plugins/CSVImportPlugin/CSVImportParsing.swift b/Plugins/CSVImportPlugin/CSVImportParsing.swift new file mode 100644 index 000000000..a5e0f1017 --- /dev/null +++ b/Plugins/CSVImportPlugin/CSVImportParsing.swift @@ -0,0 +1,150 @@ +// +// CSVImportParsing.swift +// CSVImportPlugin +// +// Pure CSV row extraction, NULL handling, and field inference. Kept free of the +// plugin's loadable-bundle and SwiftUI surface so it can be compiled into the +// test target directly (a loadable .tableplugin cannot be linked by tests). +// The RFC 4180 tokenizer itself lives in TableProPluginKit (CSVStreamingParser), +// shared with the CSV inspector. +// + +import Foundation +import TableProPluginKit + +enum CSVImportParsing { + static let detectionSampleLimit = 200 + + static func resolveDialect(in data: Data, options: CSVImportOptions) -> CSVDialect { + var dialect = CSVDialect.detect(from: data) + if let byte = options.delimiter.byte { + dialect.delimiter = byte + } + dialect.quoteChar = options.quoteCharacter.byte + if let forced = options.encoding.stringEncoding { + dialect.encoding = forced + } + return dialect + } + + static func defaultColumnName(_ index: Int) -> String { + "Column \(index + 1)" + } + + static func columnNames(header: [String]?, columnCount: Int) -> [String] { + var names: [String] = [] + names.reserveCapacity(columnCount) + var used = Set() + for index in 0.. PluginCellValue { + var value = raw + if options.trimWhitespace { + value = value.trimmingCharacters(in: .whitespaces) + } + if options.emptyAsNull, value.isEmpty { + return .null + } + if !options.nullString.isEmpty, value == options.nullString { + return .null + } + return .text(value) + } + + static func sampleText(from raw: String, options: CSVImportOptions) -> String? { + guard case .text(let value) = cellValue(from: raw, options: options), !value.isEmpty else { return nil } + return value + } + + static func row(fields: [String], columnNames: [String], options: CSVImportOptions) -> [String: PluginCellValue] { + var row: [String: PluginCellValue] = [:] + row.reserveCapacity(columnNames.count) + for (index, name) in columnNames.enumerated() { + let raw = index < fields.count ? fields[index] : "" + row[name] = cellValue(from: raw, options: options) + } + return row + } + + static func importFieldType(for type: CSVTypeInferrer.InferredType) -> PluginImportFieldType { + switch type { + case .integer: return .integer + case .real: return .real + case .boolean: return .boolean + case .date: return .text + case .text: return .text + @unknown default: return .text + } + } + + static func isBlank(_ fields: [String]) -> Bool { + fields.allSatisfy { $0.isEmpty } + } + + static func detectFields( + in data: Data, + options: CSVImportOptions, + limit: Int = detectionSampleLimit + ) -> [PluginImportField] { + let dialect = resolveDialect(in: data, options: options) + let parser = CSVStreamingParser(dialect: dialect) + + return data.withUnsafeBytes { raw -> [PluginImportField] in + guard let base = raw.bindMemory(to: UInt8.self).baseAddress else { return [] } + let buffer = UnsafeBufferPointer(start: base, count: raw.count) + let ranges = parser.indexRows(buffer) + guard !ranges.isEmpty else { return [] } + + var dataRanges = ranges[...] + var header: [String]? + if options.hasHeaderRow { + header = parser.parseRow(buffer, range: ranges[0]) + dataRanges = ranges.dropFirst() + } + + let columnCount = header?.count + ?? dataRanges.first.map { parser.parseRow(buffer, range: $0).count } + ?? 0 + guard columnCount > 0 else { return [] } + + let names = columnNames(header: header, columnCount: columnCount) + var samples: [[String]] = Array(repeating: [], count: columnCount) + var firstValues: [String?] = Array(repeating: nil, count: columnCount) + var sampled = 0 + + for range in dataRanges { + if sampled >= limit { break } + let fields = parser.parseRow(buffer, range: range) + if isBlank(fields) { continue } + for column in 0.. AnyView? { + AnyView(CSVImportOptionsView(plugin: self)) + } + + private static let detectionPrefixBytes = 1_048_576 + private static let batchSize = 500 + + func performImport( + source: any PluginImportSource, + sink: any PluginImportDataSink, + progress: PluginImportProgress + ) async throws -> PluginImportResult { + let startTime = Date() + let url = source.fileURL() + + let data: Data + do { + data = try Data(contentsOf: url, options: .mappedIfSafe) + } catch { + throw PluginImportError.importFailed(error.localizedDescription) + } + + let dialect = CSVImportParsing.resolveDialect(in: data, options: settings) + let parser = CSVStreamingParser(dialect: dialect) + let hasHeader = settings.hasHeaderRow + + let (dataRanges, columnNames) = indexRowsAndNames(in: data, parser: parser, hasHeader: hasHeader) + guard !columnNames.isEmpty else { + throw PluginImportError.importFailed("No columns found in the file") + } + + progress.setEstimatedTotal(dataRanges.count) + + let lineOffset = hasHeader ? 2 : 1 + var cursor = 0 + let outcome = try await RowImportRunner.run( + configuration: RowImportRunner.Configuration( + errorHandling: settings.errorHandling, + wrapInTransaction: settings.wrapInTransaction, + deleteExistingRows: settings.deleteExistingRows + ), + sink: sink, + progress: progress + ) { + guard cursor < dataRanges.count else { return nil } + let end = min(cursor + Self.batchSize, dataRanges.count) + let batch = self.parseBatch( + in: data, + parser: parser, + ranges: dataRanges[cursor.. 0 { + progress.incrementStatement(by: blankRows) + } + cursor = end + return batch + } + + return PluginImportResult( + executedStatements: outcome.inserted, + executionTime: Date().timeIntervalSince(startTime), + skippedStatements: outcome.skipped, + errors: outcome.errors + ) + } + + private func indexRowsAndNames( + in data: Data, + parser: CSVStreamingParser, + hasHeader: Bool + ) -> ([Range], [String]) { + data.withUnsafeBytes { raw -> ([Range], [String]) in + guard let base = raw.bindMemory(to: UInt8.self).baseAddress else { return ([], []) } + let buffer = UnsafeBufferPointer(start: base, count: raw.count) + let ranges = parser.indexRows(buffer) + guard !ranges.isEmpty else { return ([], []) } + if hasHeader { + let header = parser.parseRow(buffer, range: ranges[0]) + let names = CSVImportParsing.columnNames(header: header, columnCount: header.count) + return (Array(ranges.dropFirst()), names) + } + let firstCount = parser.parseRow(buffer, range: ranges[0]).count + let names = CSVImportParsing.columnNames(header: nil, columnCount: firstCount) + return (ranges, names) + } + } + + private func parseBatch( + in data: Data, + parser: CSVStreamingParser, + ranges: ArraySlice>, + startIndex: Int, + lineOffset: Int, + columnNames: [String] + ) -> [(line: Int, row: [String: PluginCellValue])] { + data.withUnsafeBytes { raw -> [(line: Int, row: [String: PluginCellValue])] in + guard let base = raw.bindMemory(to: UInt8.self).baseAddress else { return [] } + let buffer = UnsafeBufferPointer(start: base, count: raw.count) + var out: [(line: Int, row: [String: PluginCellValue])] = [] + out.reserveCapacity(ranges.count) + for (offset, range) in ranges.enumerated() { + let fields = parser.parseRow(buffer, range: range) + if CSVImportParsing.isBlank(fields) { continue } + let line = startIndex + offset + lineOffset + out.append((line, CSVImportParsing.row(fields: fields, columnNames: columnNames, options: settings))) + } + return out + } + } + + func detectSourceFields(at url: URL, targetTable: String?) throws -> [PluginImportField] { + let data = try readDetectionPrefix(of: url) + return CSVImportParsing.detectFields(in: data, options: settings) + } + + private func readDetectionPrefix(of url: URL) throws -> Data { + let handle = try FileHandle(forReadingFrom: url) + defer { try? handle.close() } + return handle.readData(ofLength: Self.detectionPrefixBytes) + } +} diff --git a/Plugins/CSVImportPlugin/Info.plist b/Plugins/CSVImportPlugin/Info.plist new file mode 100644 index 000000000..14e54accd --- /dev/null +++ b/Plugins/CSVImportPlugin/Info.plist @@ -0,0 +1,12 @@ + + + + + TableProPluginKitVersion + 18 + TableProProvidesImportFormatIds + + csv + + + diff --git a/Plugins/CSVInspectorPlugin/CSVWriter.swift b/Plugins/CSVInspectorPlugin/CSVWriter.swift index d781fe46d..06153172a 100644 --- a/Plugins/CSVInspectorPlugin/CSVWriter.swift +++ b/Plugins/CSVInspectorPlugin/CSVWriter.swift @@ -1,4 +1,5 @@ import Foundation +import TableProPluginKit struct CSVWriter { enum WriteError: Error, LocalizedError { @@ -34,7 +35,7 @@ struct CSVWriter { defer { try? handle.close() } var buffer = Data() - buffer.reserveCapacity(Self.flushThreshold + 4096) + buffer.reserveCapacity(Self.flushThreshold + 4_096) buffer.append(contentsOf: dialect.bomBytes) append(store.headerSource, from: store, into: &buffer) diff --git a/Plugins/JSONImportPlugin/JSONImportPlugin.swift b/Plugins/JSONImportPlugin/JSONImportPlugin.swift index d5a5cb923..bc13faf63 100644 --- a/Plugins/JSONImportPlugin/JSONImportPlugin.swift +++ b/Plugins/JSONImportPlugin/JSONImportPlugin.swift @@ -4,14 +4,11 @@ // import Foundation -import os import SwiftUI import TableProPluginKit @Observable final class JSONImportPlugin: ImportFormatPlugin, SettablePlugin { - private static let logger = Logger(subsystem: "com.TablePro", category: "JSONImportPlugin") - static let pluginName = "JSON Import" static let pluginVersion = "1.0.0" static let pluginDescription = "Import data from JSON files" @@ -38,6 +35,8 @@ final class JSONImportPlugin: ImportFormatPlugin, SettablePlugin { settings = JSONImportOptions() } + private static let batchSize = 500 + func performImport( source: any PluginImportSource, sink: any PluginImportDataSink, @@ -45,141 +44,54 @@ final class JSONImportPlugin: ImportFormatPlugin, SettablePlugin { ) async throws -> PluginImportResult { let startTime = Date() let url = source.fileURL() - let useTransaction = settings.wrapInTransaction && settings.errorHandling != .skipAndContinue - - let batchSize = 500 - var inserted = 0 - var skipped = 0 - var errors: [PluginImportResult.ImportStatementError] = [] - let maxErrors = 1_000 - var batch: [(line: Int, row: [String: PluginCellValue])] = [] - - do { - if settings.deleteExistingRows { - try await sink.deleteAllRowsFromTargetTable() - } - if useTransaction { - try await sink.beginTransaction() - } + let configuration = RowImportRunner.Configuration( + errorHandling: settings.errorHandling, + wrapInTransaction: settings.wrapInTransaction, + deleteExistingRows: settings.deleteExistingRows + ) - if JSONImportParsing.isLineDelimited(url) { - progress.setEstimatedTotal(max(1, Int(source.fileSizeBytes() / 256))) - var lineNumber = 0 - for try await line in url.lines { - try progress.checkCancellation() + let outcome: RowImportRunner.Outcome + if JSONImportParsing.isLineDelimited(url) { + progress.setEstimatedTotal(max(1, Int(source.fileSizeBytes() / 256))) + var lines = url.lines.makeAsyncIterator() + var lineNumber = 0 + outcome = try await RowImportRunner.run( + configuration: configuration, sink: sink, progress: progress + ) { + var batch: [RowImportRunner.Entry] = [] + while batch.count < Self.batchSize, let line = try await lines.next() { lineNumber += 1 let trimmed = line.trimmingCharacters(in: .whitespacesAndNewlines) guard !trimmed.isEmpty else { continue } batch.append((lineNumber, try JSONImportParsing.parseRow(fromLine: trimmed))) - if batch.count >= batchSize { - try await flush(&batch, into: sink, progress: progress, - inserted: &inserted, skipped: &skipped, errors: &errors, maxErrors: maxErrors) - } - } - } else { - let rawRows = try JSONImportParsing.parseRows(at: url, targetTable: sink.targetTable) - progress.setEstimatedTotal(rawRows.count) - for (index, rawRow) in rawRows.enumerated() { - try progress.checkCancellation() - batch.append((index + 1, JSONImportParsing.convertRow(rawRow))) - if batch.count >= batchSize { - try await flush(&batch, into: sink, progress: progress, - inserted: &inserted, skipped: &skipped, errors: &errors, maxErrors: maxErrors) - } } + return batch.isEmpty ? nil : batch } - - try await flush(&batch, into: sink, progress: progress, - inserted: &inserted, skipped: &skipped, errors: &errors, maxErrors: maxErrors) - - if useTransaction { - try await sink.commitTransaction() - } - } catch { - if useTransaction { - do { - try await sink.rollbackTransaction() - } catch { - Self.logger.warning("Rollback after failed import also failed: \(error.localizedDescription)") + } else { + let rawRows = try JSONImportParsing.parseRows(at: url, targetTable: sink.targetTable) + progress.setEstimatedTotal(rawRows.count) + var cursor = 0 + outcome = try await RowImportRunner.run( + configuration: configuration, sink: sink, progress: progress + ) { + guard cursor < rawRows.count else { return nil } + let end = min(cursor + Self.batchSize, rawRows.count) + let batch = (cursor.. [PluginImportField] { diff --git a/Plugins/CSVInspectorPlugin/CSVDialect.swift b/Plugins/TableProPluginKit/CSVDialect.swift similarity index 88% rename from Plugins/CSVInspectorPlugin/CSVDialect.swift rename to Plugins/TableProPluginKit/CSVDialect.swift index c90529b17..bc42c6142 100644 --- a/Plugins/CSVInspectorPlugin/CSVDialect.swift +++ b/Plugins/TableProPluginKit/CSVDialect.swift @@ -1,12 +1,12 @@ import Foundation -struct CSVDialect: Equatable, Sendable { - enum LineEnding: Equatable, Sendable { +public struct CSVDialect: Equatable, Sendable { + public enum LineEnding: Equatable, Sendable { case crlf case lf case cr - var bytes: [UInt8] { + public var bytes: [UInt8] { switch self { case .crlf: return [0x0D, 0x0A] case .lf: return [0x0A] @@ -15,13 +15,13 @@ struct CSVDialect: Equatable, Sendable { } } - var delimiter: UInt8 - var quoteChar: UInt8 - var encoding: String.Encoding - var lineEnding: LineEnding - var hasBom: Bool + public var delimiter: UInt8 + public var quoteChar: UInt8 + public var encoding: String.Encoding + public var lineEnding: LineEnding + public var hasBom: Bool - init( + public init( delimiter: UInt8, quoteChar: UInt8 = 0x22, encoding: String.Encoding = .utf8, @@ -35,12 +35,12 @@ struct CSVDialect: Equatable, Sendable { self.hasBom = hasBom } - static let csv = CSVDialect(delimiter: 0x2C) - static let tsv = CSVDialect(delimiter: 0x09) + public static let csv = CSVDialect(delimiter: 0x2C) + public static let tsv = CSVDialect(delimiter: 0x09) private static let detectionScanLimit = 65_536 - static func detect(from data: Data) -> CSVDialect { + public static func detect(from data: Data) -> CSVDialect { var hasBom = false var encoding: String.Encoding = .utf8 var bomLength = 0 @@ -142,7 +142,7 @@ struct CSVDialect: Equatable, Sendable { return .lf } - var bomBytes: [UInt8] { + public var bomBytes: [UInt8] { guard hasBom else { return [] } switch encoding { case .utf8: return [0xEF, 0xBB, 0xBF] diff --git a/Plugins/CSVInspectorPlugin/CSVStreamingParser.swift b/Plugins/TableProPluginKit/CSVStreamingParser.swift similarity index 89% rename from Plugins/CSVInspectorPlugin/CSVStreamingParser.swift rename to Plugins/TableProPluginKit/CSVStreamingParser.swift index 2cd21053b..0990148f1 100644 --- a/Plugins/CSVInspectorPlugin/CSVStreamingParser.swift +++ b/Plugins/TableProPluginKit/CSVStreamingParser.swift @@ -1,9 +1,13 @@ import Foundation -struct CSVStreamingParser: Sendable { - let dialect: CSVDialect +public struct CSVStreamingParser: Sendable { + public let dialect: CSVDialect - func indexRows(_ bytes: UnsafeBufferPointer) -> [Range] { + public init(dialect: CSVDialect) { + self.dialect = dialect + } + + public func indexRows(_ bytes: UnsafeBufferPointer) -> [Range] { var ranges: [Range] = [] let quote = dialect.quoteChar let delimiter = dialect.delimiter @@ -61,7 +65,7 @@ struct CSVStreamingParser: Sendable { return ranges } - func parseRow(_ bytes: UnsafeBufferPointer, range: Range) -> [String] { + public func parseRow(_ bytes: UnsafeBufferPointer, range: Range) -> [String] { var fields: [String] = [] var field: [UInt8] = [] let quote = dialect.quoteChar @@ -108,7 +112,7 @@ struct CSVStreamingParser: Sendable { return fields } - func field(_ bytes: UnsafeBufferPointer, range: Range, column: Int) -> String { + public func field(_ bytes: UnsafeBufferPointer, range: Range, column: Int) -> String { guard column >= 0 else { return "" } let quote = dialect.quoteChar let delimiter = dialect.delimiter @@ -161,8 +165,10 @@ struct CSVStreamingParser: Sendable { private func decode(_ bytes: [UInt8]) -> String { if bytes.isEmpty { return "" } - return String(bytes: bytes, encoding: dialect.encoding) - ?? String(decoding: bytes, as: UTF8.self) + if let decoded = String(bytes: bytes, encoding: dialect.encoding) { + return decoded + } + return String(bytes.map { Character(UnicodeScalar($0)) }) } private func bomSkip(in bytes: UnsafeBufferPointer) -> Int { diff --git a/Plugins/CSVInspectorPlugin/CSVTypeInferrer.swift b/Plugins/TableProPluginKit/CSVTypeInferrer.swift similarity index 86% rename from Plugins/CSVInspectorPlugin/CSVTypeInferrer.swift rename to Plugins/TableProPluginKit/CSVTypeInferrer.swift index 3d963824f..6ebf858c2 100644 --- a/Plugins/CSVInspectorPlugin/CSVTypeInferrer.swift +++ b/Plugins/TableProPluginKit/CSVTypeInferrer.swift @@ -1,10 +1,9 @@ import Foundation -import TableProPluginKit -struct CSVTypeInferrer { - typealias InferredType = InspectorColumnType +public struct CSVTypeInferrer { + public typealias InferredType = InspectorColumnType - static let sampleSize = 200 + public static let sampleSize = 200 private static let booleanLiterals: Set = [ "true", "false", "yes", "no", "t", "f", "y", "n" @@ -22,7 +21,7 @@ struct CSVTypeInferrer { return formatter }() - static func infer(column values: [String]) -> InferredType { + public static func infer(column values: [String]) -> InferredType { var sample: [String] = [] sample.reserveCapacity(min(values.count, sampleSize)) for value in values where !value.isEmpty { @@ -38,7 +37,7 @@ struct CSVTypeInferrer { return .text } - static func inferColumns(rows: [[String]], columnCount: Int) -> [InferredType] { + public static func inferColumns(rows: [[String]], columnCount: Int) -> [InferredType] { var result: [InferredType] = [] result.reserveCapacity(columnCount) for col in 0.. [PluginImportField] { [] } } diff --git a/Plugins/TableProPluginKit/RowImportRunner.swift b/Plugins/TableProPluginKit/RowImportRunner.swift new file mode 100644 index 000000000..5d32e0a9a --- /dev/null +++ b/Plugins/TableProPluginKit/RowImportRunner.swift @@ -0,0 +1,160 @@ +// +// RowImportRunner.swift +// TableProPluginKit +// +// Shared orchestration for row-based importers (JSON, CSV): transaction +// lifecycle, delete-before-import, batching, per-mode error handling, and +// progress. Format plugins supply rows through the nextBatch closure. +// + +import Foundation +import os + +public enum RowImportRunner { + public typealias Entry = (line: Int, row: [String: PluginCellValue]) + + public struct Configuration: Sendable { + public var errorHandling: ImportErrorHandling + public var wrapInTransaction: Bool + public var deleteExistingRows: Bool + public var maxRecordedErrors: Int + + public init( + errorHandling: ImportErrorHandling, + wrapInTransaction: Bool, + deleteExistingRows: Bool, + maxRecordedErrors: Int = 1_000 + ) { + self.errorHandling = errorHandling + self.wrapInTransaction = wrapInTransaction + self.deleteExistingRows = deleteExistingRows + self.maxRecordedErrors = maxRecordedErrors + } + } + + public struct Outcome: Sendable { + public let inserted: Int + public let skipped: Int + public let errors: [PluginImportResult.ImportStatementError] + } + + private static let logger = Logger(subsystem: "com.TablePro", category: "RowImportRunner") + + public static func run( + configuration: Configuration, + sink: any PluginImportDataSink, + progress: PluginImportProgress, + nextBatch: () async throws -> [Entry]? + ) async throws -> Outcome { + let useTransaction = configuration.wrapInTransaction + && configuration.errorHandling != .skipAndContinue + var inserted = 0 + var skipped = 0 + var errors: [PluginImportResult.ImportStatementError] = [] + + do { + if useTransaction { + try await sink.beginTransaction() + } + if configuration.deleteExistingRows { + try await sink.deleteAllRowsFromTargetTable() + } + + while let batch = try await nextBatch() { + try progress.checkCancellation() + guard !batch.isEmpty else { continue } + if configuration.errorHandling == .skipAndContinue { + try await insertSkippingErrors( + batch, into: sink, progress: progress, configuration: configuration, + inserted: &inserted, skipped: &skipped, errors: &errors + ) + } else { + try await insertBatch(batch, into: sink, progress: progress, inserted: &inserted) + } + } + + if useTransaction { + try await sink.commitTransaction() + } + } catch { + try await conclude(after: error, sink: sink, useTransaction: useTransaction, configuration: configuration) + } + + progress.finalize() + return Outcome(inserted: inserted, skipped: skipped, errors: errors) + } + + private static func insertBatch( + _ batch: [Entry], + into sink: any PluginImportDataSink, + progress: PluginImportProgress, + inserted: inout Int + ) async throws { + do { + try await sink.insertRows(batch.map(\.row)) + inserted += batch.count + progress.incrementStatement(by: batch.count) + } catch { + let firstLine = batch.first?.line ?? 0 + throw PluginImportError.statementFailed( + statement: "rows \(firstLine)-\(batch.last?.line ?? firstLine)", + line: firstLine, + underlyingError: error + ) + } + } + + private static func insertSkippingErrors( + _ batch: [Entry], + into sink: any PluginImportDataSink, + progress: PluginImportProgress, + configuration: Configuration, + inserted: inout Int, + skipped: inout Int, + errors: inout [PluginImportResult.ImportStatementError] + ) async throws { + for entry in batch { + try progress.checkCancellation() + do { + try await sink.insertRow(entry.row) + inserted += 1 + } catch { + skipped += 1 + if errors.count < configuration.maxRecordedErrors { + errors.append(.init( + statement: "row \(entry.line)", + line: entry.line, + errorMessage: error.localizedDescription + )) + } + } + progress.incrementStatement() + } + } + + private static func conclude( + after error: Error, + sink: any PluginImportDataSink, + useTransaction: Bool, + configuration: Configuration + ) async throws -> Never { + if useTransaction { + if configuration.errorHandling == .stopAndCommit, !(error is PluginImportCancellationError) { + do { + try await sink.commitTransaction() + } catch { + logger.warning("Commit of partial import failed: \(error.localizedDescription)") + } + } else { + do { + try await sink.rollbackTransaction() + } catch { + logger.warning("Rollback after failed import also failed: \(error.localizedDescription)") + } + } + } + if error is PluginImportCancellationError { throw error } + if error is PluginImportError { throw error } + throw PluginImportError.importFailed(error.localizedDescription) + } +} diff --git a/TablePro.xcodeproj/project.pbxproj b/TablePro.xcodeproj/project.pbxproj index 9988d0646..beadf9b0b 100644 --- a/TablePro.xcodeproj/project.pbxproj +++ b/TablePro.xcodeproj/project.pbxproj @@ -40,6 +40,8 @@ 5A86F000D00000000 /* SQLImport.tableplugin in Copy Plug-Ins (12 items) */ = {isa = PBXBuildFile; fileRef = 5A86F000100000000 /* SQLImport.tableplugin */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; 5A86F001A00000000 /* TableProPluginKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 5A860000100000000 /* TableProPluginKit.framework */; }; 5A86F001D00000000 /* JSONImport.tableplugin in Copy Plug-Ins (12 items) */ = {isa = PBXBuildFile; fileRef = 5A86F001100000000 /* JSONImport.tableplugin */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; + 5A86F003A00000000 /* TableProPluginKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 5A860000100000000 /* TableProPluginKit.framework */; }; + 5A86F003D00000000 /* CSVImport.tableplugin in Copy Plug-Ins (12 items) */ = {isa = PBXBuildFile; fileRef = 5A86F003100000000 /* CSVImport.tableplugin */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; 5A87A000A00000000 /* TableProPluginKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 5A860000100000000 /* TableProPluginKit.framework */; }; 5ABBED742FB55E1400A78382 /* TableProPluginKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 5A860000100000000 /* TableProPluginKit.framework */; }; 5ABBED802FB55E1400A78382 /* CSVInspectorPlugin.tableplugin in Copy Plug-Ins (12 items) */ = {isa = PBXBuildFile; fileRef = 5ABBED792FB55E1400A78382 /* CSVInspectorPlugin.tableplugin */; settings = {ATTRIBUTES = (CodeSignOnCopy, RemoveHeadersOnCopy, ); }; }; @@ -196,6 +198,13 @@ remoteGlobalIDString = 5A86F001000000000; remoteInfo = JSONImport; }; + 5A86F003B00000000 /* PBXContainerItemProxy */ = { + isa = PBXContainerItemProxy; + containerPortal = 5A1091BF2EF17EDC0055EA7C /* Project object */; + proxyType = 1; + remoteGlobalIDString = 5A86F003000000000; + remoteInfo = CSVImport; + }; 5ABBED7E2FB55E1400A78382 /* PBXContainerItemProxy */ = { isa = PBXContainerItemProxy; containerPortal = 5A1091BF2EF17EDC0055EA7C /* Project object */; @@ -272,6 +281,7 @@ 5A86E000D00000000 /* MQLExport.tableplugin in Copy Plug-Ins (12 items) */, 5A86F000D00000000 /* SQLImport.tableplugin in Copy Plug-Ins (12 items) */, 5A86F001D00000000 /* JSONImport.tableplugin in Copy Plug-Ins (12 items) */, + 5A86F003D00000000 /* CSVImport.tableplugin in Copy Plug-Ins (12 items) */, 5ABBED802FB55E1400A78382 /* CSVInspectorPlugin.tableplugin in Copy Plug-Ins (12 items) */, ); name = "Copy Plug-Ins (12 items)"; @@ -311,6 +321,7 @@ 5A86E000100000000 /* MQLExport.tableplugin */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = MQLExport.tableplugin; sourceTree = BUILT_PRODUCTS_DIR; }; 5A86F000100000000 /* SQLImport.tableplugin */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = SQLImport.tableplugin; sourceTree = BUILT_PRODUCTS_DIR; }; 5A86F001100000000 /* JSONImport.tableplugin */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = JSONImport.tableplugin; sourceTree = BUILT_PRODUCTS_DIR; }; + 5A86F003100000000 /* CSVImport.tableplugin */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = CSVImport.tableplugin; sourceTree = BUILT_PRODUCTS_DIR; }; 5A87A000100000000 /* CassandraDriver.tableplugin */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = CassandraDriver.tableplugin; sourceTree = BUILT_PRODUCTS_DIR; }; 5ABBED792FB55E1400A78382 /* CSVInspectorPlugin.tableplugin */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = CSVInspectorPlugin.tableplugin; sourceTree = BUILT_PRODUCTS_DIR; }; 5ABCC5A72F43856700EAF3FC /* TableProTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = TableProTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; @@ -503,6 +514,21 @@ ); target = 5ABCC5A62F43856700EAF3FC /* TableProTests */; }; + 5A86F003900000000 /* Exceptions for "Plugins/CSVImportPlugin" folder in "CSVImport" target */ = { + isa = PBXFileSystemSynchronizedBuildFileExceptionSet; + membershipExceptions = ( + Info.plist, + ); + target = 5A86F003000000000 /* CSVImport */; + }; + 5A86F004900000000 /* Exceptions for "Plugins/CSVImportPlugin" folder in "TableProTests" target */ = { + isa = PBXFileSystemSynchronizedBuildFileExceptionSet; + membershipExceptions = ( + CSVImportOptions.swift, + CSVImportParsing.swift, + ); + target = 5ABCC5A62F43856700EAF3FC /* TableProTests */; + }; 5A87A000900000000 /* Exceptions for "Plugins/CassandraDriverPlugin" folder in "CassandraDriver" target */ = { isa = PBXFileSystemSynchronizedBuildFileExceptionSet; membershipExceptions = ( @@ -696,6 +722,15 @@ path = Plugins/JSONImportPlugin; sourceTree = ""; }; + 5A86F003500000000 /* Plugins/CSVImportPlugin */ = { + isa = PBXFileSystemSynchronizedRootGroup; + exceptions = ( + 5A86F003900000000 /* Exceptions for "Plugins/CSVImportPlugin" folder in "CSVImport" target */, + 5A86F004900000000 /* Exceptions for "Plugins/CSVImportPlugin" folder in "TableProTests" target */, + ); + path = Plugins/CSVImportPlugin; + sourceTree = ""; + }; 5A87A000500000000 /* Plugins/CassandraDriverPlugin */ = { isa = PBXFileSystemSynchronizedRootGroup; exceptions = ( @@ -898,6 +933,14 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + 5A86F003300000000 /* Frameworks */ = { + isa = PBXFrameworksBuildPhase; + buildActionMask = 2147483647; + files = ( + 5A86F003A00000000 /* TableProPluginKit.framework in Frameworks */, + ); + runOnlyForDeploymentPostprocessing = 0; + }; 5A87A000300000000 /* Frameworks */ = { isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; @@ -999,6 +1042,7 @@ 5A86E000500000000 /* Plugins/MQLExportPlugin */, 5A86F000500000000 /* Plugins/SQLImportPlugin */, 5A86F001500000000 /* Plugins/JSONImportPlugin */, + 5A86F003500000000 /* Plugins/CSVImportPlugin */, 5ABCC5A82F43856700EAF3FC /* TableProTests */, 5AF00A122FB9000000000001 /* TableProUITests */, 5A32BC012F9D5F1300BAEB5F /* mcp-server */, @@ -1030,6 +1074,7 @@ 5A86E000100000000 /* MQLExport.tableplugin */, 5A86F000100000000 /* SQLImport.tableplugin */, 5A86F001100000000 /* JSONImport.tableplugin */, + 5A86F003100000000 /* CSVImport.tableplugin */, 5ABCC5A72F43856700EAF3FC /* TableProTests.xctest */, 5AF00A102FB9000000000001 /* TableProUITests.xctest */, 5AEA8B2A2F6808270040461A /* EtcdDriverPlugin.tableplugin */, @@ -1136,6 +1181,7 @@ 5A86E000C00000000 /* PBXTargetDependency */, 5A86F000C00000000 /* PBXTargetDependency */, 5A86F001C00000000 /* PBXTargetDependency */, + 5A86F003C00000000 /* PBXTargetDependency */, 5ABBED7F2FB55E1400A78382 /* PBXTargetDependency */, 5ADDB00000000000000000C1 /* PBXTargetDependency */, 5ABQR00000000000000000C1 /* PBXTargetDependency */, @@ -1547,6 +1593,26 @@ productReference = 5A86F001100000000 /* JSONImport.tableplugin */; productType = "com.apple.product-type.bundle"; }; + 5A86F003000000000 /* CSVImport */ = { + isa = PBXNativeTarget; + buildConfigurationList = 5A86F003800000000 /* Build configuration list for PBXNativeTarget "CSVImport" */; + buildPhases = ( + 5A86F003200000000 /* Sources */, + 5A86F003300000000 /* Frameworks */, + 5A86F003400000000 /* Resources */, + ); + buildRules = ( + ); + dependencies = ( + ); + fileSystemSynchronizedGroups = ( + 5A86F003500000000 /* Plugins/CSVImportPlugin */, + ); + name = CSVImport; + productName = CSVImport; + productReference = 5A86F003100000000 /* CSVImport.tableplugin */; + productType = "com.apple.product-type.bundle"; + }; 5A87A000000000000 /* CassandraDriver */ = { isa = PBXNativeTarget; buildConfigurationList = 5A87A000800000000 /* Build configuration list for PBXNativeTarget "CassandraDriver" */; @@ -1834,6 +1900,7 @@ 5A86E000000000000 /* MQLExport */, 5A86F000000000000 /* SQLImport */, 5A86F001000000000 /* JSONImport */, + 5A86F003000000000 /* CSVImport */, 5ABCC5A62F43856700EAF3FC /* TableProTests */, 5AF00A142FB9000000000001 /* TableProUITests */, 5AEA8B292F6808270040461A /* EtcdDriverPlugin */, @@ -1981,6 +2048,13 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + 5A86F003400000000 /* Resources */ = { + isa = PBXResourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; 5A87A000400000000 /* Resources */ = { isa = PBXResourcesBuildPhase; buildActionMask = 2147483647; @@ -2180,6 +2254,13 @@ ); runOnlyForDeploymentPostprocessing = 0; }; + 5A86F003200000000 /* Sources */ = { + isa = PBXSourcesBuildPhase; + buildActionMask = 2147483647; + files = ( + ); + runOnlyForDeploymentPostprocessing = 0; + }; 5A87A000200000000 /* Sources */ = { isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; @@ -2345,6 +2426,11 @@ target = 5A86F001000000000 /* JSONImport */; targetProxy = 5A86F001B00000000 /* PBXContainerItemProxy */; }; + 5A86F003C00000000 /* PBXTargetDependency */ = { + isa = PBXTargetDependency; + target = 5A86F003000000000 /* CSVImport */; + targetProxy = 5A86F003B00000000 /* PBXContainerItemProxy */; + }; 5ABBED7F2FB55E1400A78382 /* PBXTargetDependency */ = { isa = PBXTargetDependency; target = 5ABBED712FB55E1400A78382 /* CSVInspectorPlugin */; @@ -3725,6 +3811,52 @@ }; name = Release; }; + 5A86F003600000000 /* Debug */ = { + isa = XCBuildConfiguration; + buildSettings = { + CODE_SIGN_STYLE = Automatic; + COMBINE_HIDPI_IMAGES = YES; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = ""; + GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_FILE = Plugins/CSVImportPlugin/Info.plist; + INFOPLIST_KEY_NSPrincipalClass = "$(PRODUCT_MODULE_NAME).CSVImportPlugin"; + LD_RUNPATH_SEARCH_PATHS = "@executable_path/../Frameworks"; + MACOSX_DEPLOYMENT_TARGET = 14.0; + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = com.TablePro.CSVImportPlugin; + PRODUCT_NAME = "$(TARGET_NAME)"; + SDKROOT = macosx; + SKIP_INSTALL = YES; + SUPPORTED_PLATFORMS = macosx; + SWIFT_VERSION = 5.9; + WRAPPER_EXTENSION = tableplugin; + }; + name = Debug; + }; + 5A86F003700000000 /* Release */ = { + isa = XCBuildConfiguration; + buildSettings = { + CODE_SIGN_STYLE = Automatic; + COMBINE_HIDPI_IMAGES = YES; + CURRENT_PROJECT_VERSION = 1; + DEVELOPMENT_TEAM = ""; + GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_FILE = Plugins/CSVImportPlugin/Info.plist; + INFOPLIST_KEY_NSPrincipalClass = "$(PRODUCT_MODULE_NAME).CSVImportPlugin"; + LD_RUNPATH_SEARCH_PATHS = "@executable_path/../Frameworks"; + MACOSX_DEPLOYMENT_TARGET = 14.0; + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = com.TablePro.CSVImportPlugin; + PRODUCT_NAME = "$(TARGET_NAME)"; + SDKROOT = macosx; + SKIP_INSTALL = YES; + SUPPORTED_PLATFORMS = macosx; + SWIFT_VERSION = 5.9; + WRAPPER_EXTENSION = tableplugin; + }; + name = Release; + }; 5A87A000600000000 /* Debug */ = { isa = XCBuildConfiguration; buildSettings = { @@ -4336,6 +4468,15 @@ defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; + 5A86F003800000000 /* Build configuration list for PBXNativeTarget "CSVImport" */ = { + isa = XCConfigurationList; + buildConfigurations = ( + 5A86F003600000000 /* Debug */, + 5A86F003700000000 /* Release */, + ); + defaultConfigurationIsVisible = 0; + defaultConfigurationName = Release; + }; 5A87A000800000000 /* Build configuration list for PBXNativeTarget "CassandraDriver" */ = { isa = XCConfigurationList; buildConfigurations = ( diff --git a/TablePro/Core/Plugins/JSONImportTypeMapper.swift b/TablePro/Core/Plugins/ImportTypeMapper.swift similarity index 97% rename from TablePro/Core/Plugins/JSONImportTypeMapper.swift rename to TablePro/Core/Plugins/ImportTypeMapper.swift index 8a221535b..d9da17a9f 100644 --- a/TablePro/Core/Plugins/JSONImportTypeMapper.swift +++ b/TablePro/Core/Plugins/ImportTypeMapper.swift @@ -1,12 +1,12 @@ // -// JSONImportTypeMapper.swift +// ImportTypeMapper.swift // TablePro // import Foundation import TableProPluginKit -enum JSONImportTypeMapper { +enum ImportTypeMapper { static func sqlType(for type: PluginImportFieldType, databaseType: DatabaseType) -> String { switch databaseType { case .postgresql, .redshift, .cockroachdb: diff --git a/TablePro/Views/Import/JSONImportSheet.swift b/TablePro/Views/Import/RowImportSheet.swift similarity index 74% rename from TablePro/Views/Import/JSONImportSheet.swift rename to TablePro/Views/Import/RowImportSheet.swift index 407c10f56..d7f562f73 100644 --- a/TablePro/Views/Import/JSONImportSheet.swift +++ b/TablePro/Views/Import/RowImportSheet.swift @@ -1,10 +1,11 @@ // -// JSONImportSheet.swift +// RowImportSheet.swift // TablePro // -// Dedicated import sheet for row-based formats (JSON / NDJSON): -// map each source field to a column in an existing table, or create a new -// table with columns inferred from the data. +// Import sheet for row-based formats (JSON, NDJSON, CSV): map each source +// field to a column in an existing table, or create a new table with columns +// inferred from the data. The format plugin supplies the icon, name, and the +// field-detection options shown in this sheet. // import Combine @@ -12,8 +13,8 @@ import os import SwiftUI import TableProPluginKit -struct JSONImportSheet: View { - private static let logger = Logger(subsystem: "com.TablePro", category: "JSONImportSheet") +struct RowImportSheet: View { + private static let logger = Logger(subsystem: "com.TablePro", category: "RowImportSheet") @Binding var isPresented: Bool let connection: DatabaseConnection @@ -85,7 +86,7 @@ struct JSONImportSheet: View { footerView .padding() } - .frame(width: 720, height: 600) + .frame(width: 720, height: 640) .task { await loadTables() await loadNewColumns() @@ -96,6 +97,9 @@ struct JSONImportSheet: View { guard destination == .existingTable, let table = newValue else { return } Task { await loadExistingContext(table: table) } } + .onChange(of: currentPlugin?.fieldDetectionSignature) { _, _ in + Task { await redetectFields() } + } .onDisappear { importTask?.cancel() } .sheet(isPresented: $showProgressDialog) { if let service = importService { @@ -118,13 +122,13 @@ struct JSONImportSheet: View { private var headerView: some View { HStack(alignment: .top, spacing: 12) { - Image(systemName: "curlybraces") + Image(systemName: currentPlugin.map { type(of: $0).iconName } ?? "tablecells") .font(.title) .foregroundStyle(.blue) VStack(alignment: .leading, spacing: 2) { Text(fileURL.lastPathComponent) .font(.headline) - Text("Import JSON rows into a table") + Text("Import rows into a table") .font(.subheadline) .foregroundStyle(.secondary) } @@ -237,94 +241,144 @@ struct JSONImportSheet: View { } private var mappingTable: some View { - ScrollView { - Grid(alignment: .leading, horizontalSpacing: 12, verticalSpacing: 6) { - GridRow { - Toggle("", isOn: allMappingsIncluded) - .labelsHidden() - .help(String(localized: "Import all fields")) - Text("JSON field").font(.caption).foregroundStyle(.secondary) - Text("Column").font(.caption).foregroundStyle(.secondary) - } - Divider().gridCellColumns(3) - - ForEach(mappings) { row in - GridRow { - Toggle("", isOn: mappingBinding(row).include).labelsHidden() - VStack(alignment: .leading, spacing: 1) { - Text(row.field.name).lineLimit(1) - if let sample = row.field.sampleValue, !sample.isEmpty { - Text(sample).font(.caption).foregroundStyle(.secondary).lineLimit(1) - } - } - Picker("", selection: mappingBinding(row).targetColumn) { - Text("Skip").tag(String?.none) - ForEach(targetColumns, id: \.self) { column in - Text(column).tag(String?.some(column)) - } - } - .labelsHidden() - .frame(maxWidth: 240, alignment: .leading) - .disabled(!row.include) + VStack(spacing: 0) { + HStack(spacing: 12) { + Toggle("", isOn: allMappingsIncluded) + .labelsHidden() + .help(String(localized: "Import all fields")) + .frame(width: 16) + Text("Field") + .font(.caption) + .foregroundStyle(.secondary) + .frame(maxWidth: .infinity, alignment: .leading) + Text("Column") + .font(.caption) + .foregroundStyle(.secondary) + .frame(width: 240, alignment: .leading) + } + .padding(.horizontal) + .padding(.vertical, 6) + Divider() + + ScrollView { + VStack(spacing: 6) { + ForEach(mappings) { row in + mappingRow(row) } } + .padding(.horizontal) + .padding(.vertical, 8) } - .padding(.horizontal) - .padding(.vertical, 8) + } + } + + private func mappingRow(_ row: FieldMapping) -> some View { + HStack(spacing: 12) { + Toggle("", isOn: mappingBinding(row).include) + .labelsHidden() + .frame(width: 16) + VStack(alignment: .leading, spacing: 1) { + Text(row.field.name).lineLimit(1) + if let sample = row.field.sampleValue, !sample.isEmpty { + Text(sample).font(.caption).foregroundStyle(.secondary).lineLimit(1) + } + } + .frame(maxWidth: .infinity, alignment: .leading) + Picker("", selection: mappingBinding(row).targetColumn) { + Text("Skip").tag(String?.none) + ForEach(targetColumns, id: \.self) { column in + Text(column).tag(String?.some(column)) + } + } + .labelsHidden() + .frame(width: 240, alignment: .leading) + .disabled(!row.include) } } private var newColumnsTable: some View { - ScrollView { - Grid(alignment: .leading, horizontalSpacing: 10, verticalSpacing: 6) { - GridRow { - Toggle("", isOn: allColumnsIncluded) - .labelsHidden() - .help(String(localized: "Create all columns")) - Text("Column").font(.caption).foregroundStyle(.secondary) - Text("Type").font(.caption).foregroundStyle(.secondary) - Text("Key").font(.caption).foregroundStyle(.secondary) - Text("Null").font(.caption).foregroundStyle(.secondary) - Text("Default").font(.caption).foregroundStyle(.secondary) + VStack(spacing: 0) { + HStack(spacing: 10) { + Toggle("", isOn: allColumnsIncluded) + .labelsHidden() + .help(String(localized: "Create all columns")) + .frame(width: 16) + Text("Column") + .font(.caption) + .foregroundStyle(.secondary) + .frame(width: 150, alignment: .leading) + Text("Type") + .font(.caption) + .foregroundStyle(.secondary) + .frame(width: 150, alignment: .leading) + Text("Key") + .font(.caption) + .foregroundStyle(.secondary) + .frame(width: 30) + Text("Null") + .font(.caption) + .foregroundStyle(.secondary) + .frame(width: 30) + Text("Default") + .font(.caption) + .foregroundStyle(.secondary) + .frame(maxWidth: .infinity, alignment: .leading) + } + .padding(.horizontal) + .padding(.vertical, 6) + Divider() + + ScrollView { + VStack(spacing: 6) { + ForEach(newColumns) { row in + newColumnRow(row) + } } - Divider().gridCellColumns(6) - - ForEach(newColumns) { row in - GridRow { - Toggle("", isOn: columnBinding(row).include).labelsHidden() - TextField("name", text: columnBinding(row).name) - .textFieldStyle(.roundedBorder) - .frame(width: 150) - .disabled(!row.include) - Menu { - ForEach(typeOptions(including: row.type), id: \.self) { type in - Button { - columnBinding(row).type.wrappedValue = type - } label: { - if type.caseInsensitiveCompare(row.type) == .orderedSame { - Label(type, systemImage: "checkmark") - } else { - Text(type) - } - } - } - } label: { - Text(row.type) - .frame(maxWidth: .infinity, alignment: .leading) + .padding(.horizontal) + .padding(.vertical, 8) + } + } + } + + private func newColumnRow(_ row: NewColumn) -> some View { + HStack(spacing: 10) { + Toggle("", isOn: columnBinding(row).include) + .labelsHidden() + .frame(width: 16) + TextField("name", text: columnBinding(row).name) + .textFieldStyle(.roundedBorder) + .frame(width: 150) + .disabled(!row.include) + Menu { + ForEach(typeOptions(including: row.type), id: \.self) { type in + Button { + columnBinding(row).type.wrappedValue = type + } label: { + if type.caseInsensitiveCompare(row.type) == .orderedSame { + Label(type, systemImage: "checkmark") + } else { + Text(type) } - .frame(width: 150) - .disabled(!row.include) - Toggle("", isOn: columnBinding(row).isPrimaryKey).labelsHidden().disabled(!row.include) - Toggle("", isOn: columnBinding(row).isNullable).labelsHidden().disabled(!row.include) - TextField("", text: columnBinding(row).defaultValue) - .textFieldStyle(.roundedBorder) - .frame(minWidth: 120) - .disabled(!row.include) } } + } label: { + Text(row.type) + .frame(maxWidth: .infinity, alignment: .leading) } - .padding(.horizontal) - .padding(.vertical, 8) + .frame(width: 150) + .disabled(!row.include) + Toggle("", isOn: columnBinding(row).isPrimaryKey) + .labelsHidden() + .frame(width: 30) + .disabled(!row.include) + Toggle("", isOn: columnBinding(row).isNullable) + .labelsHidden() + .frame(width: 30) + .disabled(!row.include) + TextField("", text: columnBinding(row).defaultValue) + .textFieldStyle(.roundedBorder) + .frame(maxWidth: .infinity) + .disabled(!row.include) } } @@ -436,7 +490,7 @@ struct JSONImportSheet: View { field: field, include: true, name: field.name, - type: JSONImportTypeMapper.sqlType(for: field.inferredType, databaseType: connection.type), + type: ImportTypeMapper.sqlType(for: field.inferredType, databaseType: connection.type), isPrimaryKey: false, isNullable: true, defaultValue: "" @@ -470,6 +524,18 @@ struct JSONImportSheet: View { } } + @MainActor + private func redetectFields() async { + switch destination { + case .existingTable: + guard let table = selectedTargetTable else { return } + await loadExistingContext(table: table) + case .newTable: + newColumnsLoaded = false + await loadNewColumns() + } + } + // MARK: - Import private func performImport() { @@ -481,7 +547,7 @@ struct JSONImportSheet: View { let name = newTableName.trimmingCharacters(in: .whitespaces) guard !name.isEmpty, let sql = buildCreateTableSQL(tableName: name) else { importError = NSError( - domain: "JSONImport", code: -1, + domain: "RowImport", code: -1, userInfo: [NSLocalizedDescriptionKey: String(localized: "Could not build the CREATE TABLE statement")] ) showErrorDialog = true diff --git a/TablePro/Views/Main/MainContentView.swift b/TablePro/Views/Main/MainContentView.swift index 00647bdff..89d2b72a7 100644 --- a/TablePro/Views/Main/MainContentView.swift +++ b/TablePro/Views/Main/MainContentView.swift @@ -241,7 +241,7 @@ struct MainContentView: View { } ) if let url = coordinator.importFileURL { - JSONImportSheet( + RowImportSheet( isPresented: rowDismiss, connection: connection, fileURL: url, diff --git a/TableProTests/Core/Plugins/ImportTypeMapperTests.swift b/TableProTests/Core/Plugins/ImportTypeMapperTests.swift new file mode 100644 index 000000000..6c09bba6b --- /dev/null +++ b/TableProTests/Core/Plugins/ImportTypeMapperTests.swift @@ -0,0 +1,42 @@ +// +// ImportTypeMapperTests.swift +// TableProTests +// + +import Foundation +@testable import TablePro +import TableProPluginKit +import Testing + +@Suite("Import Type Mapper") +struct ImportTypeMapperTests { + @Test("PostgreSQL maps inferred types to native SQL types") + func testPostgres() { + #expect(ImportTypeMapper.sqlType(for: .integer, databaseType: .postgresql) == "BIGINT") + #expect(ImportTypeMapper.sqlType(for: .real, databaseType: .postgresql) == "DOUBLE PRECISION") + #expect(ImportTypeMapper.sqlType(for: .boolean, databaseType: .postgresql) == "BOOLEAN") + #expect(ImportTypeMapper.sqlType(for: .json, databaseType: .postgresql) == "JSONB") + #expect(ImportTypeMapper.sqlType(for: .text, databaseType: .postgresql) == "TEXT") + } + + @Test("MySQL maps inferred types to native SQL types") + func testMySQL() { + #expect(ImportTypeMapper.sqlType(for: .integer, databaseType: .mysql) == "BIGINT") + #expect(ImportTypeMapper.sqlType(for: .boolean, databaseType: .mysql) == "TINYINT(1)") + #expect(ImportTypeMapper.sqlType(for: .json, databaseType: .mysql) == "JSON") + } + + @Test("SQLite uses its storage classes") + func testSQLite() { + #expect(ImportTypeMapper.sqlType(for: .integer, databaseType: .sqlite) == "INTEGER") + #expect(ImportTypeMapper.sqlType(for: .real, databaseType: .sqlite) == "REAL") + #expect(ImportTypeMapper.sqlType(for: .json, databaseType: .sqlite) == "TEXT") + } + + @Test("Unhandled database types fall back to generic SQL types") + func testFallback() { + #expect(ImportTypeMapper.sqlType(for: .text, databaseType: .clickhouse) == "TEXT") + #expect(ImportTypeMapper.sqlType(for: .integer, databaseType: .clickhouse) == "INTEGER") + #expect(ImportTypeMapper.sqlType(for: .boolean, databaseType: .clickhouse) == "BOOLEAN") + } +} diff --git a/TableProTests/Core/Plugins/JSONImportTypeMapperTests.swift b/TableProTests/Core/Plugins/JSONImportTypeMapperTests.swift deleted file mode 100644 index e03d4383a..000000000 --- a/TableProTests/Core/Plugins/JSONImportTypeMapperTests.swift +++ /dev/null @@ -1,42 +0,0 @@ -// -// JSONImportTypeMapperTests.swift -// TableProTests -// - -import Foundation -@testable import TablePro -import TableProPluginKit -import Testing - -@Suite("JSON Import Type Mapper") -struct JSONImportTypeMapperTests { - @Test("PostgreSQL maps inferred types to native SQL types") - func testPostgres() { - #expect(JSONImportTypeMapper.sqlType(for: .integer, databaseType: .postgresql) == "BIGINT") - #expect(JSONImportTypeMapper.sqlType(for: .real, databaseType: .postgresql) == "DOUBLE PRECISION") - #expect(JSONImportTypeMapper.sqlType(for: .boolean, databaseType: .postgresql) == "BOOLEAN") - #expect(JSONImportTypeMapper.sqlType(for: .json, databaseType: .postgresql) == "JSONB") - #expect(JSONImportTypeMapper.sqlType(for: .text, databaseType: .postgresql) == "TEXT") - } - - @Test("MySQL maps inferred types to native SQL types") - func testMySQL() { - #expect(JSONImportTypeMapper.sqlType(for: .integer, databaseType: .mysql) == "BIGINT") - #expect(JSONImportTypeMapper.sqlType(for: .boolean, databaseType: .mysql) == "TINYINT(1)") - #expect(JSONImportTypeMapper.sqlType(for: .json, databaseType: .mysql) == "JSON") - } - - @Test("SQLite uses its storage classes") - func testSQLite() { - #expect(JSONImportTypeMapper.sqlType(for: .integer, databaseType: .sqlite) == "INTEGER") - #expect(JSONImportTypeMapper.sqlType(for: .real, databaseType: .sqlite) == "REAL") - #expect(JSONImportTypeMapper.sqlType(for: .json, databaseType: .sqlite) == "TEXT") - } - - @Test("Unhandled database types fall back to generic SQL types") - func testFallback() { - #expect(JSONImportTypeMapper.sqlType(for: .text, databaseType: .clickhouse) == "TEXT") - #expect(JSONImportTypeMapper.sqlType(for: .integer, databaseType: .clickhouse) == "INTEGER") - #expect(JSONImportTypeMapper.sqlType(for: .boolean, databaseType: .clickhouse) == "BOOLEAN") - } -} diff --git a/TableProTests/PluginTestSources/CSVDialect.swift b/TableProTests/PluginTestSources/CSVDialect.swift deleted file mode 120000 index 86975805d..000000000 --- a/TableProTests/PluginTestSources/CSVDialect.swift +++ /dev/null @@ -1 +0,0 @@ -../../Plugins/CSVInspectorPlugin/CSVDialect.swift \ No newline at end of file diff --git a/TableProTests/PluginTestSources/CSVStreamingParser.swift b/TableProTests/PluginTestSources/CSVStreamingParser.swift deleted file mode 120000 index ade260193..000000000 --- a/TableProTests/PluginTestSources/CSVStreamingParser.swift +++ /dev/null @@ -1 +0,0 @@ -../../Plugins/CSVInspectorPlugin/CSVStreamingParser.swift \ No newline at end of file diff --git a/TableProTests/PluginTestSources/CSVTypeInferrer.swift b/TableProTests/PluginTestSources/CSVTypeInferrer.swift deleted file mode 120000 index 1a0f494d7..000000000 --- a/TableProTests/PluginTestSources/CSVTypeInferrer.swift +++ /dev/null @@ -1 +0,0 @@ -../../Plugins/CSVInspectorPlugin/CSVTypeInferrer.swift \ No newline at end of file diff --git a/TableProTests/Plugins/CSVImportPluginTests.swift b/TableProTests/Plugins/CSVImportPluginTests.swift new file mode 100644 index 000000000..c01a965be --- /dev/null +++ b/TableProTests/Plugins/CSVImportPluginTests.swift @@ -0,0 +1,210 @@ +// +// CSVImportPluginTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +@Suite("CSV Import Plugin") +struct CSVImportPluginTests { + private func data(_ text: String) -> Data { + Data(text.utf8) + } + + private func fields(_ name: String, _ list: [PluginImportField]) -> PluginImportField? { + list.first { $0.name == name } + } + + // MARK: - Dialect resolution + + @Test("Auto dialect detects the comma delimiter") + func testAutoDelimiter() { + let dialect = CSVImportParsing.resolveDialect(in: data("a,b,c\n1,2,3\n"), options: CSVImportOptions()) + #expect(dialect.delimiter == 0x2C) + } + + @Test("Explicit delimiter overrides detection") + func testDelimiterOverride() { + var options = CSVImportOptions() + options.delimiter = .semicolon + let dialect = CSVImportParsing.resolveDialect(in: data("a,b\n1,2\n"), options: options) + #expect(dialect.delimiter == 0x3B) + } + + @Test("Quote character and forced encoding are applied") + func testQuoteAndEncodingOverride() { + var options = CSVImportOptions() + options.quoteCharacter = .singleQuote + options.encoding = .isoLatin1 + let dialect = CSVImportParsing.resolveDialect(in: data("a,b\n1,2\n"), options: options) + #expect(dialect.quoteChar == 0x27) + #expect(dialect.encoding == .isoLatin1) + } + + // MARK: - Column names + + @Test("Header names are trimmed and empty headers get a placeholder") + func testColumnNamesFromHeader() { + let names = CSVImportParsing.columnNames(header: [" id ", "", "name"], columnCount: 3) + #expect(names == ["id", "Column 2", "name"]) + } + + @Test("Duplicate header names are made unique") + func testColumnNamesDeduplicated() { + let names = CSVImportParsing.columnNames(header: ["x", "x", "x"], columnCount: 3) + #expect(names == ["x", "x 2", "x 3"]) + } + + @Test("Without a header, names are synthesized positionally") + func testColumnNamesSynthesized() { + let names = CSVImportParsing.columnNames(header: nil, columnCount: 3) + #expect(names == ["Column 1", "Column 2", "Column 3"]) + } + + // MARK: - Cell values + + @Test("Empty fields become NULL by default") + func testEmptyAsNull() { + #expect(CSVImportParsing.cellValue(from: "", options: CSVImportOptions()) == .null) + } + + @Test("Empty fields stay empty text when emptyAsNull is off") + func testEmptyAsText() { + var options = CSVImportOptions() + options.emptyAsNull = false + #expect(CSVImportParsing.cellValue(from: "", options: options) == .text("")) + } + + @Test("A configured NULL token becomes NULL") + func testNullToken() { + var options = CSVImportOptions() + options.nullString = "\\N" + #expect(CSVImportParsing.cellValue(from: "\\N", options: options) == .null) + #expect(CSVImportParsing.cellValue(from: "value", options: options) == .text("value")) + } + + @Test("Whitespace is trimmed only when requested") + func testTrimWhitespace() { + var options = CSVImportOptions() + options.trimWhitespace = true + #expect(CSVImportParsing.cellValue(from: " hi ", options: options) == .text("hi")) + #expect(CSVImportParsing.cellValue(from: " hi ", options: CSVImportOptions()) == .text(" hi ")) + } + + @Test("Trimming an all-space field yields NULL when emptyAsNull is on") + func testTrimToNull() { + var options = CSVImportOptions() + options.trimWhitespace = true + #expect(CSVImportParsing.cellValue(from: " ", options: options) == .null) + } + + // MARK: - Row mapping + + @Test("Fields map to column names by position") + func testRowMapping() { + let row = CSVImportParsing.row(fields: ["1", "Alice"], columnNames: ["id", "name"], options: CSVImportOptions()) + #expect(row["id"] == .text("1")) + #expect(row["name"] == .text("Alice")) + } + + @Test("Missing trailing fields become NULL") + func testRaggedShortRow() { + let row = CSVImportParsing.row(fields: ["1"], columnNames: ["id", "name"], options: CSVImportOptions()) + #expect(row["id"] == .text("1")) + #expect(row["name"] == .null) + } + + @Test("Extra fields beyond the column count are ignored") + func testRaggedLongRow() { + let row = CSVImportParsing.row(fields: ["1", "Alice", "extra"], columnNames: ["id", "name"], options: CSVImportOptions()) + #expect(row.count == 2) + #expect(row["name"] == .text("Alice")) + } + + // MARK: - Type mapping + + @Test("Inspector types map to import field types, date falls back to text") + func testImportFieldTypeMapping() { + #expect(CSVImportParsing.importFieldType(for: .integer) == .integer) + #expect(CSVImportParsing.importFieldType(for: .real) == .real) + #expect(CSVImportParsing.importFieldType(for: .boolean) == .boolean) + #expect(CSVImportParsing.importFieldType(for: .text) == .text) + #expect(CSVImportParsing.importFieldType(for: .date) == .text) + } + + @Test("Blank rows are detected") + func testIsBlank() { + #expect(CSVImportParsing.isBlank([""])) + #expect(CSVImportParsing.isBlank(["", ""])) + #expect(!CSVImportParsing.isBlank(["", "x"])) + } + + // MARK: - Field detection + + @Test("Detects header names, sample values, and inferred types") + func testDetectFields() { + let csv = "id,name,score,active\n1,Alice,1.5,true\n2,Bob,2.0,false\n" + let result = CSVImportParsing.detectFields(in: data(csv), options: CSVImportOptions()) + #expect(result.map(\.name) == ["id", "name", "score", "active"]) + #expect(fields("id", result)?.inferredType == .integer) + #expect(fields("name", result)?.inferredType == .text) + #expect(fields("score", result)?.inferredType == .real) + #expect(fields("active", result)?.inferredType == .boolean) + #expect(fields("name", result)?.sampleValue == "Alice") + } + + @Test("Quoted fields keep embedded delimiters and newlines") + func testDetectQuotedFields() { + let csv = "name,note\n\"a,b\",\"line1\nline2\"\n" + let result = CSVImportParsing.detectFields(in: data(csv), options: CSVImportOptions()) + #expect(result.map(\.name) == ["name", "note"]) + #expect(fields("name", result)?.sampleValue == "a,b") + #expect(fields("note", result)?.sampleValue == "line1\nline2") + } + + @Test("Doubled quotes decode to a single quote") + func testDetectDoubledQuotes() { + let csv = "label\n\"say \"\"hi\"\"\"\n" + let result = CSVImportParsing.detectFields(in: data(csv), options: CSVImportOptions()) + #expect(fields("label", result)?.sampleValue == "say \"hi\"") + } + + @Test("Header-less detection uses positional names") + func testDetectWithoutHeader() { + var options = CSVImportOptions() + options.hasHeaderRow = false + let result = CSVImportParsing.detectFields(in: data("1,Alice\n2,Bob\n"), options: options) + #expect(result.map(\.name) == ["Column 1", "Column 2"]) + #expect(fields("Column 1", result)?.inferredType == .integer) + } + + @Test("Semicolon-delimited files are auto-detected") + func testDetectSemicolon() { + let result = CSVImportParsing.detectFields(in: data("a;b;c\n1;2;3\n"), options: CSVImportOptions()) + #expect(result.map(\.name) == ["a", "b", "c"]) + } + + @Test("Trim option applies during detection, matching imported values") + func testDetectTrimAffectsInference() { + let csv = "n\n 1 \n 2 \n" + var options = CSVImportOptions() + options.trimWhitespace = true + let trimmed = CSVImportParsing.detectFields(in: data(csv), options: options) + #expect(fields("n", trimmed)?.inferredType == .integer) + #expect(fields("n", trimmed)?.sampleValue == "1") + + let untrimmed = CSVImportParsing.detectFields(in: data(csv), options: CSVImportOptions()) + #expect(fields("n", untrimmed)?.inferredType == .text) + } + + @Test("NULL token values are excluded from detection samples") + func testDetectNullTokenExcluded() { + var options = CSVImportOptions() + options.nullString = "\\N" + let result = CSVImportParsing.detectFields(in: data("n\n\\N\n5\n"), options: options) + #expect(fields("n", result)?.inferredType == .integer) + #expect(fields("n", result)?.sampleValue == "5") + } +} diff --git a/TableProTests/Plugins/RowImportRunnerTests.swift b/TableProTests/Plugins/RowImportRunnerTests.swift new file mode 100644 index 000000000..1b93d854a --- /dev/null +++ b/TableProTests/Plugins/RowImportRunnerTests.swift @@ -0,0 +1,200 @@ +// +// RowImportRunnerTests.swift +// TableProTests +// + +import Foundation +import TableProPluginKit +import Testing + +private struct MockSinkError: Error {} + +private final class MockImportSink: PluginImportDataSink, @unchecked Sendable { + let databaseTypeId = "mock" + let targetTable: String? = "table" + + private(set) var calls: [String] = [] + var onInsertRow: ([String: PluginCellValue]) throws -> Void = { _ in } + var onInsertRows: ([[String: PluginCellValue]]) throws -> Void = { _ in } + + func execute(statement: String) async throws { + calls.append("execute") + } + + func insertRow(_ values: [String: PluginCellValue]) async throws { + calls.append("insertRow") + try onInsertRow(values) + } + + func insertRows(_ rows: [[String: PluginCellValue]]) async throws { + calls.append("insertRows(\(rows.count))") + try onInsertRows(rows) + } + + func deleteAllRowsFromTargetTable() async throws { + calls.append("deleteAll") + } + + func beginTransaction() async throws { + calls.append("begin") + } + + func commitTransaction() async throws { + calls.append("commit") + } + + func rollbackTransaction() async throws { + calls.append("rollback") + } + + func disableForeignKeyChecks() async throws {} + func enableForeignKeyChecks() async throws {} +} + +@Suite("Row Import Runner") +struct RowImportRunnerTests { + private func entry(_ line: Int, _ value: String = "v") -> RowImportRunner.Entry { + (line, ["c": .text(value)]) + } + + private func makeProgress() -> PluginImportProgress { + PluginImportProgress(progress: Progress()) + } + + private func provider(_ groups: [[RowImportRunner.Entry]]) -> () async throws -> [RowImportRunner.Entry]? { + var remaining = groups + return { + remaining.isEmpty ? nil : remaining.removeFirst() + } + } + + private func configuration( + _ errorHandling: ImportErrorHandling, + wrapInTransaction: Bool = true, + deleteExistingRows: Bool = false, + maxRecordedErrors: Int = 1_000 + ) -> RowImportRunner.Configuration { + RowImportRunner.Configuration( + errorHandling: errorHandling, + wrapInTransaction: wrapInTransaction, + deleteExistingRows: deleteExistingRows, + maxRecordedErrors: maxRecordedErrors + ) + } + + @Test("Delete existing rows runs inside the transaction") + func testDeleteRunsInsideTransaction() async throws { + let sink = MockImportSink() + let outcome = try await RowImportRunner.run( + configuration: configuration(.stopAndRollback, deleteExistingRows: true), + sink: sink, + progress: makeProgress(), + nextBatch: provider([[entry(1), entry(2)]]) + ) + #expect(sink.calls == ["begin", "deleteAll", "insertRows(2)", "commit"]) + #expect(outcome.inserted == 2) + } + + @Test("Stop and rollback rolls back and reports the failed row range") + func testStopAndRollbackRollsBack() async throws { + let sink = MockImportSink() + sink.onInsertRows = { _ in throw MockSinkError() } + await #expect(throws: PluginImportError.self) { + _ = try await RowImportRunner.run( + configuration: configuration(.stopAndRollback), + sink: sink, + progress: makeProgress(), + nextBatch: provider([[entry(1), entry(2)]]) + ) + } + #expect(sink.calls.contains("rollback")) + #expect(!sink.calls.contains("commit")) + } + + @Test("Stop and commit keeps rows inserted before the error") + func testStopAndCommitCommitsPartialWork() async throws { + let sink = MockImportSink() + var batchesSeen = 0 + sink.onInsertRows = { _ in + batchesSeen += 1 + if batchesSeen == 2 { throw MockSinkError() } + } + await #expect(throws: PluginImportError.self) { + _ = try await RowImportRunner.run( + configuration: configuration(.stopAndCommit), + sink: sink, + progress: makeProgress(), + nextBatch: provider([[entry(1)], [entry(2)]]) + ) + } + #expect(sink.calls.contains("commit")) + #expect(!sink.calls.contains("rollback")) + } + + @Test("Skip mode inserts row by row, never retries a batch, and skips only failures") + func testSkipModeInsertsRowByRowWithoutBatchRetry() async throws { + let sink = MockImportSink() + sink.onInsertRow = { values in + if values["c"] == .text("bad") { throw MockSinkError() } + } + let outcome = try await RowImportRunner.run( + configuration: configuration(.skipAndContinue), + sink: sink, + progress: makeProgress(), + nextBatch: provider([[entry(1), entry(2, "bad"), entry(3)]]) + ) + #expect(outcome.inserted == 2) + #expect(outcome.skipped == 1) + #expect(outcome.errors.map(\.line) == [2]) + #expect(sink.calls.filter { $0 == "insertRow" }.count == 3) + #expect(!sink.calls.contains { $0.hasPrefix("insertRows") }) + #expect(!sink.calls.contains("begin")) + #expect(!sink.calls.contains("commit")) + #expect(!sink.calls.contains("rollback")) + } + + @Test("Skip mode caps recorded errors but keeps counting skips") + func testErrorCapLimitsRecordedErrors() async throws { + let sink = MockImportSink() + sink.onInsertRow = { _ in throw MockSinkError() } + let outcome = try await RowImportRunner.run( + configuration: configuration(.skipAndContinue, maxRecordedErrors: 2), + sink: sink, + progress: makeProgress(), + nextBatch: provider([[entry(1), entry(2), entry(3), entry(4)]]) + ) + #expect(outcome.inserted == 0) + #expect(outcome.skipped == 4) + #expect(outcome.errors.count == 2) + } + + @Test("Cancellation rolls back even in stop-and-commit mode") + func testCancellationRollsBack() async throws { + let sink = MockImportSink() + let progress = makeProgress() + progress.cancel() + await #expect(throws: PluginImportCancellationError.self) { + _ = try await RowImportRunner.run( + configuration: configuration(.stopAndCommit), + sink: sink, + progress: progress, + nextBatch: provider([[entry(1)]]) + ) + } + #expect(sink.calls.contains("rollback")) + #expect(!sink.calls.contains("commit")) + } + + @Test("Transaction is skipped when wrap is off") + func testNoTransactionWhenWrapOff() async throws { + let sink = MockImportSink() + let outcome = try await RowImportRunner.run( + configuration: configuration(.stopAndRollback, wrapInTransaction: false), + sink: sink, + progress: makeProgress(), + nextBatch: provider([[entry(1)]]) + ) + #expect(outcome.inserted == 1) + #expect(sink.calls == ["insertRows(1)"]) + } +} diff --git a/docs/features/import-export.mdx b/docs/features/import-export.mdx index e4c761eb9..bf3ce96d6 100644 --- a/docs/features/import-export.mdx +++ b/docs/features/import-export.mdx @@ -1,11 +1,11 @@ --- title: Import & Export -description: Export to CSV, JSON, SQL, MQL, or XLSX. Import SQL files with transaction safety and progress tracking. +description: Export to CSV, JSON, SQL, MQL, or XLSX. Import SQL, JSON, and CSV files with column mapping, transaction safety, and progress tracking. --- # Import & Export -Export data in five formats (CSV, JSON, SQL, MQL, XLSX), import SQL files with gzip support, and paste tabular data from the clipboard into the grid. +Export data in five formats (CSV, JSON, SQL, MQL, XLSX), import SQL, JSON, and CSV files (with gzip support for SQL), and paste tabular data from the clipboard into the grid. ## Export Data @@ -200,7 +200,7 @@ Paste tabular data directly into the data grid. Press `Cmd+V` after selecting a ## Import Data -Import `.sql` and `.sql.gz` files (statements execute directly against your database: backups, migrations, seed data), or `.json` / `.jsonl` files into a table (see [Import JSON Data](#import-json-data)). +Import `.sql` and `.sql.gz` files (statements execute directly against your database: backups, migrations, seed data), `.json` / `.jsonl` files into a table (see [Import JSON Data](#import-json-data)), or `.csv` / `.tsv` files into a table (see [Import CSV Data](#import-csv-data)). **MongoDB**: SQL import is not available. Use `mongoimport` or the MQL shell. @@ -270,6 +270,29 @@ Choose a destination: Rows insert through parameterized statements, so JSON values are never concatenated into SQL. Nested objects and arrays are stored as JSON text. The on-error, transaction, and "delete existing rows" options work the same as SQL import. +### Import CSV Data + +Choose **File** > **Import** > **From CSV** and pick a `.csv` or `.tsv` file. TablePro opens the same row import sheet used for JSON, with CSV-specific parsing options. The delimiter, quote character, encoding, and line ending are auto-detected; change any of them and the field mapping re-reads the file. + +Choose a destination: + +- **Existing table**: pick the table, then map each CSV column to a table column. Columns are auto-matched by header name; toggle one off to skip it, or remap it. +- **New table**: name the table and review the columns TablePro infers from the data. Each column's name, type, primary key, nullable flag, and default are editable before the table is created. + +CSV parsing options: + +| Option | Description | Default | +|--------|-------------|---------| +| Delimiter | Comma, semicolon, tab, or pipe | Auto-detect | +| Quote character | Double or single quote | Double quote (`"`) | +| Encoding | UTF-8, ISO Latin 1, or Windows-1252 | Auto-detect | +| First row is a header | Use row 1 as column names; off imports every row as data | Yes | +| Trim leading and trailing spaces | Trim each field before import | No | +| Treat empty values as NULL | Insert NULL for empty fields instead of empty text | Yes | +| NULL text | An extra value imported as NULL, for example `\N` | None | + +Quoted fields keep embedded commas and newlines (RFC 4180), and doubled quotes (`""`) decode to a single quote. Rows insert through parameterized statements in batches. The on-error, transaction, and "delete existing rows" options work the same as SQL import. + ## Progress and Errors During import, a progress bar shows statements processed and overall completion.