Truncate UTF-8 heavy fields when writing STATA

src/bin/read_csv/mod_dta.c

@@ -151,7 +151,8 @@ static void produce_missingness_discrete_dta(struct csv_metadata *c, jsmntok_t*
             dta_add_missing_date(var, get_dta_days_from_token(js, missing_value_token));
         } else if (var->type == READSTAT_TYPE_DOUBLE) {
             dta_add_missing_double(var, get_double_from_token(js, missing_value_token));
-        } else if (var->type == READSTAT_TYPE_STRING) {
+        } else if (var->type == READSTAT_TYPE_STRING ||
+                   var->type == READSTAT_TYPE_STRING_REF) {
         } else {
             fprintf(stderr, "%s:%d Unsupported column type %d\n", __FILE__, __LINE__, var->type);
             exit(EXIT_FAILURE);
@@ -302,7 +303,8 @@ void produce_value_label_dta(void *csv_metadata, const char* column) {
             produce_value_label_int32_date_dta(column, c, code, label);
         } else if (coltype == READSTAT_TYPE_DOUBLE) {
             produce_value_label_double_dta(column, c, code, label);
-        } else if (coltype == READSTAT_TYPE_STRING) {
+        } else if (coltype == READSTAT_TYPE_STRING ||
+                   coltype == READSTAT_TYPE_STRING_REF) {
         } else {
             fprintf(stderr, "%s:%d unsupported column type %d for value label for column %s\n", __FILE__, __LINE__, coltype, column);
             exit(EXIT_FAILURE);
@@ -496,7 +498,8 @@ void produce_csv_value_dta(void *csv_metadata, const char *s, size_t len) {
         value = value_double_date_time_dta(s, len, c);
     } else if (var->type == READSTAT_TYPE_DOUBLE) {
         value = value_double_dta(s, len, c);
-    } else if (var->type == READSTAT_TYPE_STRING) {
+    } else if (var->type == READSTAT_TYPE_STRING ||
+               var->type == READSTAT_TYPE_STRING_REF) {
         value = value_string(s, len, c);
     } else {
         fprintf(stderr, "%s:%d unsupported variable type %d\n", __FILE__, __LINE__, var->type);

src/stata/readstat_dta_write.c

@@ -1268,26 +1268,58 @@ static readstat_error_t dta_write_double(void *row, const readstat_variable_t *v
     return dta_write_raw_double(row, value);
 }
 
+/* Chops a UTF-8 string, if necessary, to fit in str2045 STATA size */
+static size_t dta_utf8_safe_length(const char *value, size_t max_len) {
+    size_t p = max_len;
+
+    /* Back up past any continuation bytes that straddle the cut point. */
+    while (p > 0 && ((unsigned char)value[p] & 0xC0) == 0x80)
+        p--;
+
+    /* p is now at a non-continuation byte (ASCII or lead byte).  Determine
+     * whether the full sequence starting at p fits within max_len. */
+    if (p < max_len) {
+        unsigned char b = (unsigned char)value[p];
+        size_t seq_len;
+        if      ((b & 0x80) == 0x00) seq_len = 1;  /* ASCII */
+        else if ((b & 0xE0) == 0xC0) seq_len = 2;
+        else if ((b & 0xF0) == 0xE0) seq_len = 3;
+        else if ((b & 0xF8) == 0xF0) seq_len = 4;
+        else                          seq_len = 1;  /* invalid; treat as single byte */
+
+        if (p + seq_len <= max_len)
+            p += seq_len;  /* full sequence fits */
+        /* else: incomplete sequence — leave p before the lead byte */
+    }
+
+    return p;
+}
+
 static readstat_error_t dta_write_string(void *row, const readstat_variable_t *var, const char *value) {
     size_t max_len = var->storage_width;
     if (value == NULL || value[0] == '\0') {
         memset(row, '\0', max_len);
     } else {
         size_t value_len = strlen(value);
-        if (value_len > max_len)
-            return READSTAT_ERROR_STRING_VALUE_IS_TOO_LONG;
-
-        strncpy((char *)row, value, max_len);
+        if (value_len > max_len) {
+            /* The string's UTF-8 byte length exceeds the column width (e.g. a
+             * near-2045-character string whose Unicode bytes push it over the
+             * str2045 cap). Truncate at the last valid UTF-8 boundary so the
+             * row is preserved rather than dropped. */
+            size_t safe_len = dta_utf8_safe_length(value, max_len);
+            memset(row, '\0', max_len);
+            memcpy((char *)row, value, safe_len);
+        } else {
+            strncpy((char *)row, value, max_len);
+        }
     }
     return READSTAT_OK;
 }
 
 static readstat_error_t dta_118_write_string_ref(void *row, const readstat_variable_t *var, readstat_string_ref_t *ref) {
-    if (ref == NULL)
-        return READSTAT_ERROR_STRING_REF_IS_REQUIRED;
-
-    int16_t v = ref->first_v;
-    int64_t o = ref->first_o;
+    /* A NULL ref encodes a missing strL: write (v=0, o=0). */
+    int16_t v = ref ? ref->first_v : 0;
+    int64_t o = ref ? ref->first_o : 0;
     char *row_bytes = (char *)row;
     memcpy(&row_bytes[0], &v, sizeof(int16_t));
     if (!machine_is_little_endian()) {
@@ -1298,11 +1330,9 @@ static readstat_error_t dta_118_write_string_ref(void *row, const readstat_varia
 }
 
 static readstat_error_t dta_117_write_string_ref(void *row, const readstat_variable_t *var, readstat_string_ref_t *ref) {
-    if (ref == NULL)
-        return READSTAT_ERROR_STRING_REF_IS_REQUIRED;
-
-    int32_t v = ref->first_v;
-    int32_t o = ref->first_o;
+    /* A NULL ref encodes a missing strL: write (v=0, o=0). */
+    int32_t v = ref ? ref->first_v : 0;
+    int32_t o = ref ? ref->first_o : 0;
     char *row_bytes = (char *)row;
     memcpy(&row_bytes[0], &v, sizeof(int32_t));
     memcpy(&row_bytes[4], &o, sizeof(int32_t));