Skip to content

Support deleted rows in SAS data files #366

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
hpoettker wants to merge 1 commit into
base: dev
Choose a base branch
from
Open

Support deleted rows in SAS data files #366

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Binary file added resources/datetime.sas7bdat
View file
Open in desktop
Binary file not shown.
12 changes: 7 additions & 5 deletions src/sas/readstat_sas.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,18 +111,20 @@ typedef enum sas_subheader_type_e {
#define SAS_PAGE_TYPE_AMD 0x0400
#define SAS_PAGE_TYPE_MASK 0x0F00

#define SAS_PAGE_TYPE_META2 0x4000
#define SAS_PAGE_TYPE_COMP 0x9000
#define SAS_PAGE_TYPE_DELETED_ROWS 0x0080
#define SAS_PAGE_TYPE_META2 0x4000
#define SAS_PAGE_TYPE_COMP 0x9000

#define SAS_SUBHEADER_POINTER_SIZE_32BIT 12
#define SAS_SUBHEADER_POINTER_SIZE_64BIT 24

#define SAS_PAGE_HEADER_SIZE_32BIT 24
#define SAS_PAGE_HEADER_SIZE_64BIT 40

#define SAS_COMPRESSION_NONE 0x00
#define SAS_COMPRESSION_TRUNC 0x01
#define SAS_COMPRESSION_ROW 0x04
#define SAS_COMPRESSION_NONE 0x00
#define SAS_COMPRESSION_TRUNC 0x01
#define SAS_COMPRESSION_ROW 0x04
#define SAS_COMPRESSION_DELETED_ROW 0x05

#define SAS_COMPRESSION_SIGNATURE_RLE "SASYZCRL"
#define SAS_COMPRESSION_SIGNATURE_RDC "SASYZCR2"
Expand Down
76 changes: 70 additions & 6 deletions src/sas/readstat_sas7bdat_read.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
#include <string.h>
#include <math.h>
#include <inttypes.h>
#include <limits.h>
#include "readstat_sas.h"
#include "readstat_sas_rle.h"
#include "../readstat_iconv.h"
Expand Down Expand Up @@ -45,9 +46,12 @@ typedef struct sas7bdat_ctx_s {

uint32_t row_length;
uint32_t page_row_count;
uint32_t total_row_count;
uint32_t parsed_row_count;
uint32_t parsed_deleted_row_count;
uint32_t column_count;
uint32_t row_limit;
uint32_t deleted_row_count;
uint32_t row_offset;

uint64_t header_size;
Expand Down Expand Up @@ -232,7 +236,7 @@ static readstat_error_t sas7bdat_parse_column_size_subheader(const char *subhead
static readstat_error_t sas7bdat_parse_row_size_subheader(const char *subheader, size_t len, sas7bdat_ctx_t *ctx) {
readstat_error_t retval = READSTAT_OK;
uint64_t total_row_count;
uint64_t row_length, page_row_count;
uint64_t row_length, deleted_row_count, page_row_count;

if (len < (ctx->u64 ? 250: 190)) {
retval = READSTAT_ERROR_PARSE;
Expand All @@ -242,13 +246,22 @@ static readstat_error_t sas7bdat_parse_row_size_subheader(const char *subheader,
if (ctx->u64) {
row_length = sas_read8(&subheader[40], ctx->bswap);
total_row_count = sas_read8(&subheader[48], ctx->bswap);
deleted_row_count = sas_read8(&subheader[56], ctx->bswap);
page_row_count = sas_read8(&subheader[120], ctx->bswap);
} else {
row_length = sas_read4(&subheader[20], ctx->bswap);
total_row_count = sas_read4(&subheader[24], ctx->bswap);
deleted_row_count = sas_read4(&subheader[28], ctx->bswap);
page_row_count = sas_read4(&subheader[60], ctx->bswap);
}

if (deleted_row_count > total_row_count) {
retval = READSTAT_ERROR_PARSE;
goto cleanup;
}
ctx->total_row_count = total_row_count;
ctx->deleted_row_count = deleted_row_count;

sas_text_ref_t file_label_ref = sas7bdat_parse_text_ref(&subheader[len-130], ctx);
if (file_label_ref.length) {
if ((retval = sas7bdat_copy_text_ref(ctx->file_label, sizeof(ctx->file_label),
Expand All @@ -275,7 +288,7 @@ static readstat_error_t sas7bdat_parse_row_size_subheader(const char *subheader,
}

ctx->page_row_count = page_row_count;
uint64_t total_row_count_after_skipping = total_row_count;
uint64_t total_row_count_after_skipping = total_row_count - deleted_row_count;
if (total_row_count > ctx->row_offset) {
total_row_count_after_skipping -= ctx->row_offset;
} else {
Expand Down Expand Up @@ -393,6 +406,14 @@ static readstat_error_t sas7bdat_parse_column_format_subheader(const char *subhe
return retval;
}

static readstat_error_t sas7bdat_register_deleted_row(sas7bdat_ctx_t *ctx) {
if (ctx->parsed_deleted_row_count >= ctx->deleted_row_count) {
return READSTAT_ERROR_PARSE;
}
ctx->parsed_deleted_row_count++;
return READSTAT_OK;
}

static readstat_error_t sas7bdat_handle_data_value(readstat_variable_t *variable,
col_info_t *col_info, const char *col_data, sas7bdat_ctx_t *ctx) {
readstat_error_t retval = READSTAT_OK;
Expand Down Expand Up @@ -490,7 +511,15 @@ static readstat_error_t sas7bdat_parse_single_row(const char *data, sas7bdat_ctx
return retval;
}

static readstat_error_t sas7bdat_parse_rows(const char *data, size_t len, sas7bdat_ctx_t *ctx) {
static unsigned char sas7bdat_read_bitmap(const unsigned char *bitmap, int index) {
unsigned char current_byte = bitmap[index / CHAR_BIT];
unsigned char mask = 1 << (CHAR_BIT - 1 - index % CHAR_BIT);

return current_byte & mask;
}

static readstat_error_t sas7bdat_parse_rows(const char *data, size_t len,
const unsigned char *deleted_bitmap, sas7bdat_ctx_t *ctx) {
readstat_error_t retval = READSTAT_OK;
int i;
size_t row_offset=0;
Expand All @@ -499,8 +528,13 @@ static readstat_error_t sas7bdat_parse_rows(const char *data, size_t len, sas7bd
retval = READSTAT_ERROR_ROW_WIDTH_MISMATCH;
goto cleanup;
}
if ((retval = sas7bdat_parse_single_row(&data[row_offset], ctx)) != READSTAT_OK)
if (deleted_bitmap != NULL && sas7bdat_read_bitmap(deleted_bitmap, i)) {
if ((retval = sas7bdat_register_deleted_row(ctx)) != READSTAT_OK) {
goto cleanup;
}
} else if ((retval = sas7bdat_parse_single_row(&data[row_offset], ctx)) != READSTAT_OK) {
goto cleanup;
}

row_offset += ctx->row_length;
}
Expand Down Expand Up @@ -932,7 +966,7 @@ static readstat_error_t sas7bdat_parse_page_pass1(const char *page, size_t page_
goto cleanup;
}
}
} else if (shp_info.compression == SAS_COMPRESSION_ROW) {
} else if (shp_info.compression == SAS_COMPRESSION_ROW || shp_info.compression == SAS_COMPRESSION_DELETED_ROW) {
/* void */
} else {
retval = READSTAT_ERROR_UNSUPPORTED_COMPRESSION;
Expand All @@ -948,6 +982,25 @@ static readstat_error_t sas7bdat_parse_page_pass1(const char *page, size_t page_
return retval;
}

static readstat_error_t sas7bdat_parse_deleted_row_bitmap(const char *page, const char *data,
size_t page_size, const unsigned char **deleted_row_bitmap, sas7bdat_ctx_t *ctx) {
uint64_t page_unused_bytes;
if (ctx->u64) {
page_unused_bytes = sas_read8(&page[24], ctx->bswap);
} else {
page_unused_bytes = sas_read4(&page[12], ctx->bswap);
}
uint32_t row_count = ctx->page_row_count < ctx->total_row_count ? ctx->page_row_count : ctx->total_row_count;
uint64_t deleted_row_bitmap_offset = row_count * ctx->row_length + page_unused_bytes;
uint32_t required_bytes = row_count / CHAR_BIT + (row_count % CHAR_BIT == 0 ? 0 : 1);

if ((data - page) + deleted_row_bitmap_offset + required_bytes > page_size) {
return READSTAT_ERROR_PARSE;
}
*deleted_row_bitmap = (const unsigned char *)data + deleted_row_bitmap_offset;
return READSTAT_OK;
}

static readstat_error_t sas7bdat_parse_page_pass2(const char *page, size_t page_size, sas7bdat_ctx_t *ctx) {
uint16_t page_type;

Expand Down Expand Up @@ -1008,6 +1061,10 @@ static readstat_error_t sas7bdat_parse_page_pass2(const char *page, size_t page_
if ((retval = sas7bdat_parse_subheader_compressed(page + shp_info.offset, shp_info.len, ctx)) != READSTAT_OK) {
goto cleanup;
}
} else if (shp_info.compression == SAS_COMPRESSION_DELETED_ROW) {
if ((retval = sas7bdat_register_deleted_row(ctx)) != READSTAT_OK) {
goto cleanup;
}
} else {
retval = READSTAT_ERROR_UNSUPPORTED_COMPRESSION;
goto cleanup;
Expand Down Expand Up @@ -1037,7 +1094,14 @@ static readstat_error_t sas7bdat_parse_page_pass2(const char *page, size_t page_
goto cleanup;
}
if (ctx->handle.value) {
retval = sas7bdat_parse_rows(data, page + page_size - data, ctx);
const unsigned char *deleted_row_bitmap = NULL;
if (page_type & SAS_PAGE_TYPE_DELETED_ROWS) {
if ((retval = sas7bdat_parse_deleted_row_bitmap(page, data, page_size,
&deleted_row_bitmap, ctx)) != READSTAT_OK) {
goto cleanup;
}
}
retval = sas7bdat_parse_rows(data, page + page_size - data, deleted_row_bitmap, ctx);
}
}
cleanup:
Expand Down
59 changes: 58 additions & 1 deletion src/test/test_buffer.c
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,20 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>

#include "test_buffer.h"

rt_buffer_t *buffer_init(void) {
rt_buffer_t *buffer = calloc(1, sizeof(rt_buffer_t));
if (!buffer) {
return NULL;
}
buffer->size = 1024;
buffer->bytes = malloc(buffer->size);

if (!buffer->bytes) {
free(buffer);
return NULL;
}
return buffer;
}

Expand Down Expand Up @@ -36,3 +44,52 @@ void buffer_ctx_reset(rt_buffer_ctx_t *buffer_ctx) {
buffer_reset(buffer_ctx->buffer);
buffer_ctx->pos = 0;
}

readstat_error_t buffer_read_from_resource(rt_buffer_t *buffer, char *resource_name) {
const char *path_prefix = "resources/";
int retval = READSTAT_OK;

FILE* file = NULL;
char *resource_path = malloc(strlen(path_prefix) + strlen(resource_name) + 1);
if (resource_path == NULL) {
retval = READSTAT_ERROR_MALLOC;
goto cleanup;
}

strcpy(resource_path, path_prefix);
strcat(resource_path, resource_name);
file = fopen(resource_path, "rb");
if (!file) {
retval = READSTAT_ERROR_OPEN;
goto cleanup;
}

const size_t CHUNK_SIZE = 1024;
while (1) {
buffer_grow(buffer, CHUNK_SIZE);
if (!buffer->bytes) {
free(buffer);
retval = READSTAT_ERROR_MALLOC;
goto cleanup;
}

size_t bytes_read = fread(buffer->bytes + buffer->used, 1, CHUNK_SIZE, file);
buffer->used += bytes_read;
if (ferror(file)) {
buffer_free(buffer);
retval = READSTAT_ERROR_READ;
goto cleanup;
}

if (bytes_read < CHUNK_SIZE) {
break;
}
}

cleanup:
free(resource_path);
if (file) {
fclose(file);
}
return retval;
}
3 changes: 3 additions & 0 deletions src/test/test_buffer.h
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
#include "../readstat.h"

typedef struct rt_buffer_s {
size_t used;
Expand All @@ -17,3 +18,5 @@ void buffer_free(rt_buffer_t *buffer);

rt_buffer_ctx_t *buffer_ctx_init(rt_buffer_t *buffer);
void buffer_ctx_reset(rt_buffer_ctx_t *buffer_ctx);

readstat_error_t buffer_read_from_resource(rt_buffer_t *buffer, char *resource_name);
4 changes: 3 additions & 1 deletion src/test/test_error.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,7 +181,9 @@ static void print_value(readstat_value_t value) {

void print_error(rt_error_t *error) {
if (error->file) {
printf("Test \"%s\" failed: %s\n", error->file->label, error->msg);
rt_test_file_t *file = error->file;
char *test_name = file->label[0] ? file->label : file->resource_name;
printf("Test \"%s\" failed: %s\n", test_name, error->msg);
} else {
printf("Test failed: %s\n", error->msg);
}
Expand Down
60 changes: 60 additions & 0 deletions src/test/test_list.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2317,6 +2317,66 @@ static rt_test_group_t _test_groups[] = {
}
}
}
},

{
.label = "Resource tests",
.tests = {
{
.resource_name = "datetime.sas7bdat",
.table_name = "DATETIME",
.test_formats = RT_FORMAT_SAS7BDAT_32BIT_COMP_NONE,
.rows = 3,
.columns_count = 5,
.columns = {
{
.name = "Date1",
.type = READSTAT_TYPE_DOUBLE,
.values = {
{ .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 5 } },
{ .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 2 } },
{ .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 5 } }
}
},
{
.name = "Date2",
.type = READSTAT_TYPE_DOUBLE,
.values = {
{ .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 3 } },
{ .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 4 } },
{ .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 3 } }
}
},
{
.name = "DateTime",
.type = READSTAT_TYPE_DOUBLE,
.values = {
{ .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = -8907752836 } },
{ .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 9538991236 } },
{ .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = -8907752836 } }
}
},
{
.name = "DateTimeHi",
.type = READSTAT_TYPE_DOUBLE,
.values = {
{ .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = -8907752836.85477447509766 } },
{ .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 0 } },
{ .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 9538991236.85477447509766 } }
}
},
{
.name = "Taiw",
.type = READSTAT_TYPE_DOUBLE,
.values = {
{ .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = -17532 } },
{ .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = 1 } },
{ .type = READSTAT_TYPE_DOUBLE, .v = { .double_value = -17532 } }
}
}
}
}
}
}
};

Loading
Loading