From 646d4def73273c226fc8df710e3f79dc905b074f Mon Sep 17 00:00:00 2001 From: "Allen, Timothy" Date: Mon, 17 Mar 2025 09:55:19 -0400 Subject: [PATCH 1/5] Fix patterns for DATE_TIMEs. --- src/bin/extract_metadata.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/bin/extract_metadata.c b/src/bin/extract_metadata.c index 95c17140..68bd4b60 100644 --- a/src/bin/extract_metadata.c +++ b/src/bin/extract_metadata.c @@ -155,15 +155,15 @@ static int handle_variable_sav(int index, readstat_variable_t *variable, const c } else if (hasPrefix(vformat, "YMDHMS16") == 0) { // e.g. 2013-01-31 1:02 format = EXTRACT_METADATA_FORMAT_DATE_TIME; - pattern = "yyyy-MM-dd h:mm"; + pattern = "yyyy-MM-dd hh:mm"; } else if (hasPrefix(vformat, "YMDHMS19") == 0) { // e.g. 2013-01-31 1:02:33 format = EXTRACT_METADATA_FORMAT_DATE_TIME; - pattern = "yyyy-MM-dd h:mm:ss"; + pattern = "yyyy-MM-dd hh:mm:ss"; } else if (hasPrefix(vformat, "YMDHMS19.2") == 0) { // e.g. 2013-01-31 1:02:33.72 format = EXTRACT_METADATA_FORMAT_DATE_TIME; - pattern = "yyyy-MM-dd h:mm:ss.SS+"; + pattern = "yyyy-MM-dd hh:mm:ss.SS+"; } else if (hasPrefix(vformat, "MTIME5") == 0) { // e.g. 1754:36 format = EXTRACT_METADATA_FORMAT_TIME; From 8edb1714ba335953e9a150fd69bc43f7a7e4fae7 Mon Sep 17 00:00:00 2001 From: "Allen, Timothy" Date: Mon, 17 Mar 2025 13:39:46 -0400 Subject: [PATCH 2/5] Create is_date_time. --- src/bin/read_csv/csv_metadata.h | 1 + src/bin/read_csv/mod_dta.c | 44 ++++++++++++++++++++++++++++++++- src/bin/read_csv/read_csv.c | 1 + 3 files changed, 45 insertions(+), 1 deletion(-) diff --git a/src/bin/read_csv/csv_metadata.h b/src/bin/read_csv/csv_metadata.h index 7c950850..8867461a 100644 --- a/src/bin/read_csv/csv_metadata.h +++ b/src/bin/read_csv/csv_metadata.h @@ -12,6 +12,7 @@ typedef struct csv_metadata { void *user_ctx; readstat_variable_t *variables; int* is_date; + int* is_date_time; struct json_metadata *json_md; rs_read_module_t *output_module; } csv_metadata; diff --git a/src/bin/read_csv/mod_dta.c b/src/bin/read_csv/mod_dta.c index 8ed7736a..020478d2 100644 --- a/src/bin/read_csv/mod_dta.c +++ b/src/bin/read_csv/mod_dta.c @@ -193,6 +193,7 @@ void produce_column_header_dta(void *csv_metadata, const char *column, readstat_ extract_metadata_type_t coltype = column_type(c->json_md, column, c->output_format); if (coltype == EXTRACT_METADATA_TYPE_NUMERIC) { extract_metadata_format_t colformat = column_format(c->json_md, column); + switch (colformat) { case EXTRACT_METADATA_FORMAT_NUMBER: case EXTRACT_METADATA_FORMAT_PERCENT: @@ -206,7 +207,7 @@ void produce_column_header_dta(void *csv_metadata, const char *column, readstat_ break; case EXTRACT_METADATA_FORMAT_TIME: case EXTRACT_METADATA_FORMAT_DATE_TIME: - var->type = READSTAT_TYPE_INT32; + var->type = READSTAT_TYPE_DOUBLE; snprintf(var->format, sizeof(var->format), "%s", "%tC"); // %tC => is equivalent to coordinated universal time (UTC) break; @@ -385,10 +386,48 @@ static readstat_value_t value_double_dta(const char *s, size_t len, struct csv_m return value; } +static readstat_value_t value_double_date_time_dta(const char *s, size_t len, struct csv_metadata *c) { + char *dest; + readstat_variable_t *var = &c->variables[c->columns]; + double val = strtod(s, &dest); + if (dest == s) { + fprintf(stderr, "not a number: %s\n", (char*)s); + exit(EXIT_FAILURE); + } + int missing_ranges_count = readstat_variable_get_missing_ranges_count(var); + for (int i=0; i= lo && val <= hi) { + readstat_value_t value = { + .type = READSTAT_TYPE_DOUBLE, + .is_tagged_missing = 1, + .tag = 'a' + i, + .v = { .double_value = val } + }; + return value; + } + } + + readstat_value_t value = { + .type = READSTAT_TYPE_DOUBLE, + .is_tagged_missing = 0, + .v = { .double_value = val } + }; + return value; +} + void produce_csv_value_dta(void *csv_metadata, const char *s, size_t len) { struct csv_metadata *c = (struct csv_metadata *)csv_metadata; readstat_variable_t *var = &c->variables[c->columns]; int is_date = c->is_date[c->columns]; + int is_date_time = c->is_date_time[c->columns]; int obs_index = c->rows - 1; // TODO: ??? readstat_value_t value; @@ -396,6 +435,9 @@ void produce_csv_value_dta(void *csv_metadata, const char *s, size_t len) { value = value_sysmiss(s, len, c); } else if (is_date) { value = value_int32_date_dta(s, len, c); + } else if (is_date_time) { + printf("WE ARE IN is_date_time, YAY!!!"); + value = value_double_date_time_dta(s, len, c); } else if (var->type == READSTAT_TYPE_DOUBLE) { value = value_double_dta(s, len, c); } else if (var->type == READSTAT_TYPE_STRING) { diff --git a/src/bin/read_csv/read_csv.c b/src/bin/read_csv/read_csv.c index bdc0d548..c59728a1 100644 --- a/src/bin/read_csv/read_csv.c +++ b/src/bin/read_csv/read_csv.c @@ -46,6 +46,7 @@ static void produce_column_header(struct csv_metadata *c, void *s, size_t len) { extract_metadata_format_t colformat = column_format(c->json_md, column); c->is_date[c->columns] = colformat == EXTRACT_METADATA_FORMAT_DATE; + c->is_date_time[c->columns] = colformat == EXTRACT_METADATA_FORMAT_DATE_TIME; if (c->output_module->header) { c->output_module->header(c, column, var); } From 2376f4728805cd3bca319f853682da6da93413e8 Mon Sep 17 00:00:00 2001 From: "Allen, Timothy" Date: Tue, 18 Mar 2025 10:58:43 -0400 Subject: [PATCH 3/5] Complete timestamp support for STATA with leap seconds accounted for. --- src/bin/read_csv/mod_dta.c | 113 +++++++++++++++++++++++++++--------- src/bin/read_csv/read_csv.c | 5 ++ 2 files changed, 90 insertions(+), 28 deletions(-) diff --git a/src/bin/read_csv/mod_dta.c b/src/bin/read_csv/mod_dta.c index 020478d2..d8799d14 100644 --- a/src/bin/read_csv/mod_dta.c +++ b/src/bin/read_csv/mod_dta.c @@ -193,7 +193,6 @@ void produce_column_header_dta(void *csv_metadata, const char *column, readstat_ extract_metadata_type_t coltype = column_type(c->json_md, column, c->output_format); if (coltype == EXTRACT_METADATA_TYPE_NUMERIC) { extract_metadata_format_t colformat = column_format(c->json_md, column); - switch (colformat) { case EXTRACT_METADATA_FORMAT_NUMBER: case EXTRACT_METADATA_FORMAT_PERCENT: @@ -387,39 +386,98 @@ static readstat_value_t value_double_dta(const char *s, size_t len, struct csv_m } static readstat_value_t value_double_date_time_dta(const char *s, size_t len, struct csv_metadata *c) { - char *dest; - readstat_variable_t *var = &c->variables[c->columns]; - double val = strtod(s, &dest); - if (dest == s) { - fprintf(stderr, "not a number: %s\n", (char*)s); + // Handle empty or NULL strings as missing values + if (s == NULL || len == 0 || *s == '\0') { + readstat_value_t value = { + .type = READSTAT_TYPE_DOUBLE, + .is_system_missing = 1, + .v = { .double_value = NAN } + }; + return value; + } + + // Truncate the date string to 23 characters to remove the timezone offset and + // microseconds, if present. STATA does not support timezones or microseconds. + char date_time[24]; + strncpy(date_time, s, 23); + date_time[23] = '\0'; + + // Parse date-time components + int year, month, day, hour, minute, second, msecs = 0; + int matched = sscanf( + date_time, + "%d-%d-%d %d:%d:%d.%d", + &year, &month, &day, &hour, &minute, &second, &msecs + ); + if (matched < 6 || matched > 8) { + fprintf(stderr, "%s:%d not a valid date-time: %s (expected format: yyyy-mm-dd hh:MM:SS with optional milliseconds. Datetime string is truncated at 23 characters to ignore microseconds and timezone information.)\n", __FILE__, __LINE__, date_time); exit(EXIT_FAILURE); } - int missing_ranges_count = readstat_variable_get_missing_ranges_count(var); - for (int i=0; i= lo && val <= hi) { - readstat_value_t value = { - .type = READSTAT_TYPE_DOUBLE, - .is_tagged_missing = 1, - .tag = 'a' + i, - .v = { .double_value = val } - }; - return value; - } + + // Get days since the epoch for the date + char days_since_epoch_string[11]; + sprintf(days_since_epoch_string, "%04d-%02d-%02d", year, month, day); + char* dest; + int days_since_epoch = readstat_dta_num_days(days_since_epoch_string, &dest); + + // Add the hours, minutes, and seconds to the days + double msecs_since_epoch = 86400000.0 * days_since_epoch + hour * 3600000.0 + minute * 60000.0 + second * 1000.0 + msecs * 1.0; + + // Adjust for leap seconds; 27 have occurred as of writing this code + // https://en.m.wikipedia.org/wiki/Leap_second + typedef struct { + int year; + int month; + int day; + } leap_second_date; + + leap_second_date leap_seconds[] = { + {1972, 6, 30}, {1972, 12, 31}, // +2 seconds in 1972 + {1973, 12, 31}, // +1 second in 1973 + {1974, 12, 31}, // +1 second in 1974 + {1975, 12, 31}, // +1 second in 1975 + {1976, 12, 31}, // +1 second in 1976 + {1977, 12, 31}, // +1 second in 1977 + {1978, 12, 31}, // +1 second in 1978 + {1979, 12, 31}, // +1 second in 1979 + {1981, 6, 30}, // +1 second in 1981 + {1982, 6, 30}, // +1 second in 1982 + {1983, 6, 30}, // +1 second in 1983 + {1985, 6, 30}, // +1 second in 1985 + {1987, 12, 31}, // +1 second in 1987 + {1989, 12, 31}, // +1 second in 1989 + {1990, 12, 31}, // +1 second in 1990 + {1992, 6, 30}, // +1 second in 1992 + {1993, 6, 30}, // +1 second in 1993 + {1994, 6, 30}, // +1 second in 1994 + {1995, 12, 31}, // +1 second in 1995 + {1997, 6, 30}, // +1 second in 1997 + {1998, 12, 31}, // +1 second in 1998 + {2005, 12, 31}, // +1 second in 2005 + {2008, 12, 31}, // +1 second in 2008 + {2012, 6, 30}, // +1 second in 2012 + {2015, 6, 30}, // +1 second in 2015 + {2016, 12, 31} // +1 second in 2016 + }; + + int leap_second_count = sizeof(leap_seconds) / sizeof(leap_seconds[0]); + int leap_seconds_to_add = 0; + + for (int i = 0; i < leap_second_count; i++) { + // If the date is after this leap second, add one second + if ( + (year > leap_seconds[i].year) || + (year == leap_seconds[i].year && month > leap_seconds[i].month) || + (year == leap_seconds[i].year && month == leap_seconds[i].month && day > leap_seconds[i].day) + ) { leap_seconds_to_add++; } } + msecs_since_epoch += leap_seconds_to_add * 1000.0; readstat_value_t value = { .type = READSTAT_TYPE_DOUBLE, - .is_tagged_missing = 0, - .v = { .double_value = val } + .v = { .double_value = msecs_since_epoch } }; + return value; } @@ -436,7 +494,6 @@ void produce_csv_value_dta(void *csv_metadata, const char *s, size_t len) { } else if (is_date) { value = value_int32_date_dta(s, len, c); } else if (is_date_time) { - printf("WE ARE IN is_date_time, YAY!!!"); value = value_double_date_time_dta(s, len, c); } else if (var->type == READSTAT_TYPE_DOUBLE) { value = value_double_dta(s, len, c); diff --git a/src/bin/read_csv/read_csv.c b/src/bin/read_csv/read_csv.c index c59728a1..65fb343a 100644 --- a/src/bin/read_csv/read_csv.c +++ b/src/bin/read_csv/read_csv.c @@ -76,6 +76,7 @@ static void csv_metadata_cell(void *s, size_t len, void *data) if (c->rows == 0) { c->variables = realloc(c->variables, (c->columns+1) * sizeof(readstat_variable_t)); c->is_date = realloc(c->is_date, (c->columns+1) * sizeof(int)); + c->is_date_time = realloc(c->is_date_time, (c->columns+1) * sizeof(int)); produce_column_header(c, s, len); } else if (c->rows >= 1 && c->handle.value && c->output_module->csv_value) { c->output_module->csv_value(c, s, len); @@ -185,6 +186,10 @@ readstat_error_t readstat_parse_csv(readstat_parser_t *parser, free(md->is_date); md->is_date = NULL; } + if (md->is_date_time) { + free(md->is_date_time); + md->is_date_time = NULL; + } csv_free(p); io->close(io->io_ctx); return retval; From e5451bb896bfcf0a2c24cc5c4c49431c23282533 Mon Sep 17 00:00:00 2001 From: "Allen, Timothy" Date: Mon, 24 Mar 2025 10:21:43 -0400 Subject: [PATCH 4/5] Switch to snprintf for safer operations. --- src/bin/read_csv/mod_dta.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/bin/read_csv/mod_dta.c b/src/bin/read_csv/mod_dta.c index d8799d14..94a099d6 100644 --- a/src/bin/read_csv/mod_dta.c +++ b/src/bin/read_csv/mod_dta.c @@ -399,8 +399,7 @@ static readstat_value_t value_double_date_time_dta(const char *s, size_t len, st // Truncate the date string to 23 characters to remove the timezone offset and // microseconds, if present. STATA does not support timezones or microseconds. char date_time[24]; - strncpy(date_time, s, 23); - date_time[23] = '\0'; + snprintf(date_time, sizeof(date_time), "%s", s); // Parse date-time components int year, month, day, hour, minute, second, msecs = 0; @@ -416,7 +415,7 @@ static readstat_value_t value_double_date_time_dta(const char *s, size_t len, st // Get days since the epoch for the date char days_since_epoch_string[11]; - sprintf(days_since_epoch_string, "%04d-%02d-%02d", year, month, day); + snprintf(days_since_epoch_string, sizeof(days_since_epoch_string), "%04d-%02d-%02d", year, month, day); char* dest; int days_since_epoch = readstat_dta_num_days(days_since_epoch_string, &dest); From 97b6bc1ae4b0332a5730f9bfe93f0ede4d6a08d8 Mon Sep 17 00:00:00 2001 From: "Allen, Timothy" Date: Fri, 17 Jul 2026 14:51:25 -0400 Subject: [PATCH 5/5] Truncate UTF-8 heavy string fields by a few characters, rather than dropping the row. --- src/bin/read_csv/mod_dta.c | 9 ++++-- src/stata/readstat_dta_write.c | 58 ++++++++++++++++++++++++++-------- 2 files changed, 50 insertions(+), 17 deletions(-) diff --git a/src/bin/read_csv/mod_dta.c b/src/bin/read_csv/mod_dta.c index 94a099d6..2dcd97fd 100644 --- a/src/bin/read_csv/mod_dta.c +++ b/src/bin/read_csv/mod_dta.c @@ -151,7 +151,8 @@ static void produce_missingness_discrete_dta(struct csv_metadata *c, jsmntok_t* dta_add_missing_date(var, get_dta_days_from_token(js, missing_value_token)); } else if (var->type == READSTAT_TYPE_DOUBLE) { dta_add_missing_double(var, get_double_from_token(js, missing_value_token)); - } else if (var->type == READSTAT_TYPE_STRING) { + } else if (var->type == READSTAT_TYPE_STRING || + var->type == READSTAT_TYPE_STRING_REF) { } else { fprintf(stderr, "%s:%d Unsupported column type %d\n", __FILE__, __LINE__, var->type); exit(EXIT_FAILURE); @@ -302,7 +303,8 @@ void produce_value_label_dta(void *csv_metadata, const char* column) { produce_value_label_int32_date_dta(column, c, code, label); } else if (coltype == READSTAT_TYPE_DOUBLE) { produce_value_label_double_dta(column, c, code, label); - } else if (coltype == READSTAT_TYPE_STRING) { + } else if (coltype == READSTAT_TYPE_STRING || + coltype == READSTAT_TYPE_STRING_REF) { } else { fprintf(stderr, "%s:%d unsupported column type %d for value label for column %s\n", __FILE__, __LINE__, coltype, column); exit(EXIT_FAILURE); @@ -496,7 +498,8 @@ void produce_csv_value_dta(void *csv_metadata, const char *s, size_t len) { value = value_double_date_time_dta(s, len, c); } else if (var->type == READSTAT_TYPE_DOUBLE) { value = value_double_dta(s, len, c); - } else if (var->type == READSTAT_TYPE_STRING) { + } else if (var->type == READSTAT_TYPE_STRING || + var->type == READSTAT_TYPE_STRING_REF) { value = value_string(s, len, c); } else { fprintf(stderr, "%s:%d unsupported variable type %d\n", __FILE__, __LINE__, var->type); diff --git a/src/stata/readstat_dta_write.c b/src/stata/readstat_dta_write.c index 5519b031..209e0ded 100644 --- a/src/stata/readstat_dta_write.c +++ b/src/stata/readstat_dta_write.c @@ -1268,26 +1268,58 @@ static readstat_error_t dta_write_double(void *row, const readstat_variable_t *v return dta_write_raw_double(row, value); } +/* Chops a UTF-8 string, if necessary, to fit in str2045 STATA size */ +static size_t dta_utf8_safe_length(const char *value, size_t max_len) { + size_t p = max_len; + + /* Back up past any continuation bytes that straddle the cut point. */ + while (p > 0 && ((unsigned char)value[p] & 0xC0) == 0x80) + p--; + + /* p is now at a non-continuation byte (ASCII or lead byte). Determine + * whether the full sequence starting at p fits within max_len. */ + if (p < max_len) { + unsigned char b = (unsigned char)value[p]; + size_t seq_len; + if ((b & 0x80) == 0x00) seq_len = 1; /* ASCII */ + else if ((b & 0xE0) == 0xC0) seq_len = 2; + else if ((b & 0xF0) == 0xE0) seq_len = 3; + else if ((b & 0xF8) == 0xF0) seq_len = 4; + else seq_len = 1; /* invalid; treat as single byte */ + + if (p + seq_len <= max_len) + p += seq_len; /* full sequence fits */ + /* else: incomplete sequence — leave p before the lead byte */ + } + + return p; +} + static readstat_error_t dta_write_string(void *row, const readstat_variable_t *var, const char *value) { size_t max_len = var->storage_width; if (value == NULL || value[0] == '\0') { memset(row, '\0', max_len); } else { size_t value_len = strlen(value); - if (value_len > max_len) - return READSTAT_ERROR_STRING_VALUE_IS_TOO_LONG; - - strncpy((char *)row, value, max_len); + if (value_len > max_len) { + /* The string's UTF-8 byte length exceeds the column width (e.g. a + * near-2045-character string whose Unicode bytes push it over the + * str2045 cap). Truncate at the last valid UTF-8 boundary so the + * row is preserved rather than dropped. */ + size_t safe_len = dta_utf8_safe_length(value, max_len); + memset(row, '\0', max_len); + memcpy((char *)row, value, safe_len); + } else { + strncpy((char *)row, value, max_len); + } } return READSTAT_OK; } static readstat_error_t dta_118_write_string_ref(void *row, const readstat_variable_t *var, readstat_string_ref_t *ref) { - if (ref == NULL) - return READSTAT_ERROR_STRING_REF_IS_REQUIRED; - - int16_t v = ref->first_v; - int64_t o = ref->first_o; + /* A NULL ref encodes a missing strL: write (v=0, o=0). */ + int16_t v = ref ? ref->first_v : 0; + int64_t o = ref ? ref->first_o : 0; char *row_bytes = (char *)row; memcpy(&row_bytes[0], &v, sizeof(int16_t)); if (!machine_is_little_endian()) { @@ -1298,11 +1330,9 @@ static readstat_error_t dta_118_write_string_ref(void *row, const readstat_varia } static readstat_error_t dta_117_write_string_ref(void *row, const readstat_variable_t *var, readstat_string_ref_t *ref) { - if (ref == NULL) - return READSTAT_ERROR_STRING_REF_IS_REQUIRED; - - int32_t v = ref->first_v; - int32_t o = ref->first_o; + /* A NULL ref encodes a missing strL: write (v=0, o=0). */ + int32_t v = ref ? ref->first_v : 0; + int32_t o = ref ? ref->first_o : 0; char *row_bytes = (char *)row; memcpy(&row_bytes[0], &v, sizeof(int32_t)); memcpy(&row_bytes[4], &o, sizeof(int32_t));