diff --git a/src/bin/read_csv/mod_dta.c b/src/bin/read_csv/mod_dta.c index 94a099d6..2dcd97fd 100644 --- a/src/bin/read_csv/mod_dta.c +++ b/src/bin/read_csv/mod_dta.c @@ -151,7 +151,8 @@ static void produce_missingness_discrete_dta(struct csv_metadata *c, jsmntok_t* dta_add_missing_date(var, get_dta_days_from_token(js, missing_value_token)); } else if (var->type == READSTAT_TYPE_DOUBLE) { dta_add_missing_double(var, get_double_from_token(js, missing_value_token)); - } else if (var->type == READSTAT_TYPE_STRING) { + } else if (var->type == READSTAT_TYPE_STRING || + var->type == READSTAT_TYPE_STRING_REF) { } else { fprintf(stderr, "%s:%d Unsupported column type %d\n", __FILE__, __LINE__, var->type); exit(EXIT_FAILURE); @@ -302,7 +303,8 @@ void produce_value_label_dta(void *csv_metadata, const char* column) { produce_value_label_int32_date_dta(column, c, code, label); } else if (coltype == READSTAT_TYPE_DOUBLE) { produce_value_label_double_dta(column, c, code, label); - } else if (coltype == READSTAT_TYPE_STRING) { + } else if (coltype == READSTAT_TYPE_STRING || + coltype == READSTAT_TYPE_STRING_REF) { } else { fprintf(stderr, "%s:%d unsupported column type %d for value label for column %s\n", __FILE__, __LINE__, coltype, column); exit(EXIT_FAILURE); @@ -496,7 +498,8 @@ void produce_csv_value_dta(void *csv_metadata, const char *s, size_t len) { value = value_double_date_time_dta(s, len, c); } else if (var->type == READSTAT_TYPE_DOUBLE) { value = value_double_dta(s, len, c); - } else if (var->type == READSTAT_TYPE_STRING) { + } else if (var->type == READSTAT_TYPE_STRING || + var->type == READSTAT_TYPE_STRING_REF) { value = value_string(s, len, c); } else { fprintf(stderr, "%s:%d unsupported variable type %d\n", __FILE__, __LINE__, var->type); diff --git a/src/stata/readstat_dta_write.c b/src/stata/readstat_dta_write.c index 5519b031..209e0ded 100644 --- a/src/stata/readstat_dta_write.c +++ b/src/stata/readstat_dta_write.c @@ -1268,26 +1268,58 @@ static readstat_error_t dta_write_double(void *row, const readstat_variable_t *v return dta_write_raw_double(row, value); } +/* Chops a UTF-8 string, if necessary, to fit in str2045 STATA size */ +static size_t dta_utf8_safe_length(const char *value, size_t max_len) { + size_t p = max_len; + + /* Back up past any continuation bytes that straddle the cut point. */ + while (p > 0 && ((unsigned char)value[p] & 0xC0) == 0x80) + p--; + + /* p is now at a non-continuation byte (ASCII or lead byte). Determine + * whether the full sequence starting at p fits within max_len. */ + if (p < max_len) { + unsigned char b = (unsigned char)value[p]; + size_t seq_len; + if ((b & 0x80) == 0x00) seq_len = 1; /* ASCII */ + else if ((b & 0xE0) == 0xC0) seq_len = 2; + else if ((b & 0xF0) == 0xE0) seq_len = 3; + else if ((b & 0xF8) == 0xF0) seq_len = 4; + else seq_len = 1; /* invalid; treat as single byte */ + + if (p + seq_len <= max_len) + p += seq_len; /* full sequence fits */ + /* else: incomplete sequence — leave p before the lead byte */ + } + + return p; +} + static readstat_error_t dta_write_string(void *row, const readstat_variable_t *var, const char *value) { size_t max_len = var->storage_width; if (value == NULL || value[0] == '\0') { memset(row, '\0', max_len); } else { size_t value_len = strlen(value); - if (value_len > max_len) - return READSTAT_ERROR_STRING_VALUE_IS_TOO_LONG; - - strncpy((char *)row, value, max_len); + if (value_len > max_len) { + /* The string's UTF-8 byte length exceeds the column width (e.g. a + * near-2045-character string whose Unicode bytes push it over the + * str2045 cap). Truncate at the last valid UTF-8 boundary so the + * row is preserved rather than dropped. */ + size_t safe_len = dta_utf8_safe_length(value, max_len); + memset(row, '\0', max_len); + memcpy((char *)row, value, safe_len); + } else { + strncpy((char *)row, value, max_len); + } } return READSTAT_OK; } static readstat_error_t dta_118_write_string_ref(void *row, const readstat_variable_t *var, readstat_string_ref_t *ref) { - if (ref == NULL) - return READSTAT_ERROR_STRING_REF_IS_REQUIRED; - - int16_t v = ref->first_v; - int64_t o = ref->first_o; + /* A NULL ref encodes a missing strL: write (v=0, o=0). */ + int16_t v = ref ? ref->first_v : 0; + int64_t o = ref ? ref->first_o : 0; char *row_bytes = (char *)row; memcpy(&row_bytes[0], &v, sizeof(int16_t)); if (!machine_is_little_endian()) { @@ -1298,11 +1330,9 @@ static readstat_error_t dta_118_write_string_ref(void *row, const readstat_varia } static readstat_error_t dta_117_write_string_ref(void *row, const readstat_variable_t *var, readstat_string_ref_t *ref) { - if (ref == NULL) - return READSTAT_ERROR_STRING_REF_IS_REQUIRED; - - int32_t v = ref->first_v; - int32_t o = ref->first_o; + /* A NULL ref encodes a missing strL: write (v=0, o=0). */ + int32_t v = ref ? ref->first_v : 0; + int32_t o = ref ? ref->first_o : 0; char *row_bytes = (char *)row; memcpy(&row_bytes[0], &v, sizeof(int32_t)); memcpy(&row_bytes[4], &o, sizeof(int32_t));