From 7f653ae05e69f935e9a59c10b9bad2cf5122c6b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Clemens=20D=C3=B6rrh=C3=B6fer?= Date: Tue, 23 Sep 2014 18:06:19 +0200 Subject: [PATCH 01/70] MariaDB 10.1 page encryption v0.1 Alpha --- dbug/dbug.c | 48 + include/keyfile.h | 19 + include/my_aes.h | 38 + include/my_dbug.h | 3 + mysql-test/r/enc.result | 20 + mysql-test/t/enc.test | 28 + mysys/keyfile.c | 110 + mysys_ssl/my_aes.cc | 118 +- storage/xtradb/CMakeLists.txt | 1 + storage/xtradb/buf/buf0buf.cc | 9 +- storage/xtradb/fil/fil0fil.cc | 19 +- storage/xtradb/fil/fil0fil.cc.orig | 6885 ++++++++++++++++++ storage/xtradb/fil/fil0pageencryption.cc | 537 ++ storage/xtradb/handler/ha_innodb.cc | 64 +- storage/xtradb/handler/ha_innodb.h | 2 + storage/xtradb/include/dict0dict.h | 4 +- storage/xtradb/include/dict0dict.ic | 41 +- storage/xtradb/include/dict0mem.h | 38 +- storage/xtradb/include/dict0pagecompress.ic | 25 + storage/xtradb/include/fil0fil.h | 9 + storage/xtradb/include/fil0pageencryption.h | 97 + storage/xtradb/include/fsp0fsp.h | 55 +- storage/xtradb/include/fsp0fsp.ic | 2 + storage/xtradb/include/fsp0pageencryption.h | 59 + storage/xtradb/include/fsp0pageencryption.ic | 120 + storage/xtradb/include/os0file.h | 16 +- storage/xtradb/include/os0file.ic | 8 +- storage/xtradb/include/srv0mon.h | 5 + storage/xtradb/include/srv0srv.h | 13 + storage/xtradb/os/os0file.cc | 150 +- storage/xtradb/os/os0file.cc.orig | 6683 +++++++++++++++++ storage/xtradb/os/os0file.cc.rej | 20 + storage/xtradb/srv/srv0mon.cc | 9 + storage/xtradb/srv/srv0srv.cc | 4 + unittest/eperi/CMakeLists.txt | 42 + unittest/eperi/enc0enc-t.cc | 47 + unittest/eperi/enc0enc-t.h | 13 + unittest/eperi/eperi-t.c | 10 + unittest/eperi/eperi_aes-t.c | 361 + unittest/eperi/eperi_keyfile-t.c | 38 + unittest/eperi/johnny-t.cc | 131 + unittest/eperi/johnny-t.h | 19 + unittest/eperi/keys.txt | 11 + unittest/eperi/kf.txt | 1 + unittest/eperi/kfo.txt | 1 + unittest/eperi/pfs_jim-t.cc | 126 + unittest/eperi/xaa | 0 unittest/eperi/xab | 0 unittest/eperi/xac | 0 unittest/eperi/xad | 0 unittest/eperi/xae | 0 unittest/eperi/xaf | 0 52 files changed, 16022 insertions(+), 37 deletions(-) create mode 100644 include/keyfile.h create mode 100644 mysql-test/r/enc.result create mode 100644 mysql-test/t/enc.test create mode 100644 mysys/keyfile.c create mode 100644 storage/xtradb/fil/fil0fil.cc.orig create mode 100644 storage/xtradb/fil/fil0pageencryption.cc create mode 100644 storage/xtradb/include/fil0pageencryption.h create mode 100644 storage/xtradb/include/fsp0pageencryption.h create mode 100644 storage/xtradb/include/fsp0pageencryption.ic create mode 100644 storage/xtradb/os/os0file.cc.orig create mode 100644 storage/xtradb/os/os0file.cc.rej create mode 100644 unittest/eperi/CMakeLists.txt create mode 100644 unittest/eperi/enc0enc-t.cc create mode 100644 unittest/eperi/enc0enc-t.h create mode 100644 unittest/eperi/eperi-t.c create mode 100644 unittest/eperi/eperi_aes-t.c create mode 100644 unittest/eperi/eperi_keyfile-t.c create mode 100644 unittest/eperi/johnny-t.cc create mode 100644 unittest/eperi/johnny-t.h create mode 100644 unittest/eperi/keys.txt create mode 100644 unittest/eperi/kf.txt create mode 100644 unittest/eperi/kfo.txt create mode 100644 unittest/eperi/pfs_jim-t.cc create mode 100644 unittest/eperi/xaa create mode 100644 unittest/eperi/xab create mode 100644 unittest/eperi/xac create mode 100644 unittest/eperi/xad create mode 100644 unittest/eperi/xae create mode 100644 unittest/eperi/xaf diff --git a/dbug/dbug.c b/dbug/dbug.c index dffd7a44cd8f0..8ab183dc85869 100644 --- a/dbug/dbug.c +++ b/dbug/dbug.c @@ -85,6 +85,7 @@ #undef SAFE_MUTEX #include #include +#include #ifndef DBUG_OFF @@ -2184,6 +2185,53 @@ const char* _db_get_func_(void) return cs->func; } + +void dump_buffer(unsigned n, const unsigned char* buf) { +#if defined(UNIV_DEBUG) +int on_this_line = 0; +int counter = 0; +int cc =0; +char ch =0; + +FILE* stream = stderr; +fflush(stream); +fprintf(stream, "%06X: ", counter); +while (n-- > 0) { + fprintf(stream, "%02X ", *buf++); + on_this_line += 1; + if (on_this_line == 16 || n == 0) { + int i; + fprintf(stream, " "); + int cc = on_this_line; + if (cc != 16) { + + + for (i = on_this_line; i < 16; i++) { + fprintf(stream," " ); + } + } + for (i = on_this_line; i > 0; i--) { + ch =isprint(buf[-i]) ? buf[-i] : '.'; + fprintf(stream,"%c",ch); + } + + fprintf(stream,"\n" ); + + on_this_line = 0; + if (n!=0) fprintf(stream, "%06X: ", ++counter); + + + } else { + counter++; + } +} +fprintf( stream, "\n"); +fflush(stream); +#endif +} + + + #else /* diff --git a/include/keyfile.h b/include/keyfile.h new file mode 100644 index 0000000000000..01d9b0a1d451d --- /dev/null +++ b/include/keyfile.h @@ -0,0 +1,19 @@ +#include + +struct keyentry { + int id; + char *iv; + char *key; +}; + +int +parseFile(FILE * fp, struct keyentry **allKeys, const int k_len); + +int +parseLine(const char *line, struct keyentry *entry); + +int +isComment(char *line); + +char* +trim(char *in); diff --git a/include/my_aes.h b/include/my_aes.h index 58a7891902338..0d4486448a1da 100644 --- a/include/my_aes.h +++ b/include/my_aes.h @@ -1,6 +1,11 @@ #ifndef MY_AES_INCLUDED #define MY_AES_INCLUDED +#define AES_OK 0 +#define AES_BAD_DATA -1 +#define AES_BAD_KEYSIZE -5 +#define AES_KEY_CREATION_FAILED -10 + /* Copyright (c) 2002, 2006 MySQL AB, 2009 Sun Microsystems, Inc. Use is subject to license terms. @@ -26,6 +31,23 @@ C_MODE_START #define AES_KEY_LENGTH 128 /* Must be 128 192 or 256 */ +typedef unsigned long int ulint; + + +/* + my_aes_encrypt_cbc- Crypt buffer with AES encryption algorithm using cbc mode. + source - Pointer to data for encryption + source_length - size of encryption data + dest - buffer to place encrypted data (must be large enough) + key - Key to be used for encryption + kel_length - Length of the key. Will handle keys of any length + + returns - size of encrypted data, or negative in case of error. +*/ +int my_aes_encrypt_cbc(const char* source, ulint source_length, + char* dest, ulint *dest_length, + const char* key, uint8 key_length, + const char* iv, uint8 iv_length); /* my_aes_encrypt - Crypt buffer with AES encryption algorithm. @@ -41,6 +63,22 @@ C_MODE_START int my_aes_encrypt(const char *source, int source_length, char *dest, const char *key, int key_length); +/* + my_aes_decrypt_cbc - DeCrypt buffer with AES encryption algorithm using + cbc Mode. + source - Pointer to data for decryption + source_length - size of encrypted data + dest - buffer to place decrypted data (must be large enough) + key - Key to be used for decryption + kel_length - Length of the key. Will handle keys of any length + + returns - size of original data, or negative in case of error. +*/ +int my_aes_decrypt_cbc(const char* source, ulint source_length, + char* dest, ulint *dest_length, + const char* key, uint8 key_length, + const char* iv, uint8 iv_length); + /* my_aes_decrypt - DeCrypt buffer with AES encryption algorithm. source - Pointer to data for decryption diff --git a/include/my_dbug.h b/include/my_dbug.h index bcf2015466dec..3837e35f1417a 100644 --- a/include/my_dbug.h +++ b/include/my_dbug.h @@ -52,6 +52,9 @@ extern void _db_return_(uint _line_, struct _db_stack_frame_ *_stack_frame_); extern void _db_pargs_(uint _line_,const char *keyword); extern void _db_doprnt_(const char *format,...) ATTRIBUTE_FORMAT(printf, 1, 2); + +extern void dump_buffer(unsigned n, const unsigned char* buf); + extern void _db_dump_(uint _line_,const char *keyword, const unsigned char *memory, size_t length); extern void _db_end_(void); diff --git a/mysql-test/r/enc.result b/mysql-test/r/enc.result new file mode 100644 index 0000000000000..46b53558e23ae --- /dev/null +++ b/mysql-test/r/enc.result @@ -0,0 +1,20 @@ +DROP TABLE IF EXISTS t1; +DROP DATABASE IF EXISTS test; +CREATE DATABASE test; +USE test; +set @save_storage_engine= @@storage_engine; +set storage_engine=InnoDB; +CREATE TABLE t1 (id int) +PAGE_ENCRYPTION='abc'; +ERROR HY000: Incorrect value 'abc' for option 'PAGE_ENCRYPTION' +CREATE TABLE t1 (id int) +PAGE_ENCRYPTION=1 +PAGE_ENCRYPTION_KEY='0xFFC'; +ERROR HY000: Incorrect value '0xFFC' for option 'PAGE_ENCRYPTION_KEY' +CREATE TABLE t1 (id int(11)) +PAGE_ENCRYPTION=1 +PAGE_ENCRYPTION_KEY=42; +INSERT INTO t1(id) values(1); +SELECT * FROM t1; +id +1 diff --git a/mysql-test/t/enc.test b/mysql-test/t/enc.test new file mode 100644 index 0000000000000..6e93d4765d816 --- /dev/null +++ b/mysql-test/t/enc.test @@ -0,0 +1,28 @@ +-- source include/have_xtradb.inc + +--disable_warnings +DROP TABLE IF EXISTS t1; +DROP DATABASE IF EXISTS test; +--enable_warnings + +CREATE DATABASE test; +USE test; +set @save_storage_engine= @@storage_engine; +set storage_engine=InnoDB; + +--error ER_BAD_OPTION_VALUE +CREATE TABLE t1 (id int) + PAGE_ENCRYPTION='abc'; + +--error ER_BAD_OPTION_VALUE +CREATE TABLE t1 (id int) + PAGE_ENCRYPTION=1 + PAGE_ENCRYPTION_KEY='0xFFC'; + +CREATE TABLE t1 (id int(11)) + PAGE_ENCRYPTION=1 + PAGE_ENCRYPTION_KEY=42; + +INSERT INTO t1(id) values(1); +SELECT * FROM t1; + diff --git a/mysys/keyfile.c b/mysys/keyfile.c new file mode 100644 index 0000000000000..43ba8fbaf6b02 --- /dev/null +++ b/mysys/keyfile.c @@ -0,0 +1,110 @@ +#include +#include +#include +#include + +#define E_WRONG_NUMBER_OF_MATCHES 10 + +int +isComment(char *line) +{ + const char *error_p; + int offset; + int m_len = (int) strlen(line); + + pcre *pattern = pcre_compile( + "\\s*#.*", + 0, + &error_p, + &offset, + NULL); + int rc,i; + int ovector[30]; + rc = pcre_exec( + pattern, + NULL, + line, + m_len, + 0, + 0, + ovector, + 30 + ); + if(rc < 0) { + return 0; + } else { + return 1; + } +} + +int +parseFile(FILE * fp, struct keyentry **allKeys, const int k_len) +{ + if(NULL == fp) { + return 100; + } + char *line = NULL; + size_t len = 0; + ssize_t read; + + while(( read = getline(&line, &len, fp)) != -1 ) { + struct keyentry *entry = (struct keyentry*) malloc(sizeof(struct keyentry)); + if( parseLine(line, entry) == 0 && entry->id < k_len) { + allKeys[entry->id] = entry; + } + } + return 0; +} + +int +parseLine(const char *line, struct keyentry *entry) +{ + const char *error_p; + int offset; + + pcre *pattern = pcre_compile( + "([0-9]+);([0-9,a-f,A-F]+);([0-9,a-f,A-F]+)", + 0, + &error_p, + &offset, + NULL); + if( error_p != NULL ) { + fprintf(stderr, "Error: %s\n", error_p); + fprintf(stderr, "Offset: %d\n", offset); + } + int m_len = (int) strlen(line); + char *buffer = (char*) malloc(400*sizeof(char)); + int rc,i; + int ovector[30]; + rc = pcre_exec( + pattern, + NULL, + line, + m_len, + 0, + 0, + ovector, + 30 + ); + if(rc == 4 && !isComment(line)) { + char *substring_start = line + ovector[2]; + int substr_length = ovector[3] - ovector[2]; + sprintf( buffer, "%.*s", substr_length, substring_start ); + entry->id = atoi(buffer); + + substring_start = line + ovector[4]; + substr_length = ovector[5] - ovector[4]; + entry->iv = malloc(substr_length*sizeof(char)); + sprintf( entry->iv, "%.*s", substr_length, substring_start ); + + substring_start = line + ovector[6]; + substr_length = ovector[7] - ovector[6]; + entry->key = malloc(substr_length*sizeof(char)); + sprintf( entry->key, "%.*s", substr_length, substring_start ); + return 0; + } else + { + return E_WRONG_NUMBER_OF_MATCHES; + } + return 0; +} diff --git a/mysys_ssl/my_aes.cc b/mysys_ssl/my_aes.cc index 9327bc32a3b60..7cb068df02fe8 100644 --- a/mysys_ssl/my_aes.cc +++ b/mysys_ssl/my_aes.cc @@ -101,6 +101,24 @@ static int my_aes_create_key(const char *key, int key_length, uint8 *rkey) return 0; } +/** + Decode Hexencoded String to uint8[]. + my_aes_hexToUint() + @param iv [in] Pointer to hexadecimal encoded IV String + @param dest [out] Pointer to output uint8 array. Memory needs to be allocated by caller + @param iv_length [in] Size of destination array. + */ +void +my_aes_hexToUint(const char* iv, unsigned char *dest, int dest_length) +{ + const char *pos = iv; + int count = 0; + for(count = 0; count < dest_length; count++) + { + sscanf(pos, "%2hhx", &dest[count]); + pos += 2 * sizeof(char); + } +} /** Crypt buffer with AES encryption algorithm. @@ -118,7 +136,105 @@ static int my_aes_create_key(const char *key, int key_length, uint8 *rkey) < 0 Error */ -int my_aes_encrypt(const char* source, int source_length, char* dest, +int my_aes_encrypt_cbc(const char* source, ulint source_length, + char* dest, ulint *dest_length, + const char* key, uint8 key_length, + const char* iv, uint8 iv_length) +{ +#if defined(HAVE_OPENSSL) + MyCipherCtx ctx; + int u_len, f_len; + /* The real key to be used for encryption */ + unsigned char rkey[key_length]; + my_aes_hexToUint(key, rkey, key_length); + + unsigned char riv[iv_length]; + my_aes_hexToUint(iv, riv, iv_length); + const EVP_CIPHER* cipher; + switch(key_length) { + case 16: + cipher = EVP_aes_128_cbc(); + break; + case 24: + cipher = EVP_aes_192_cbc(); + break; + case 32: + cipher = EVP_aes_256_cbc(); + break; + default: + return AES_BAD_KEYSIZE; + } + //Initialize Encryption Engine here, default software Engine is default + ENGINE *engine = NULL; + + if (! EVP_EncryptInit_ex(&ctx.ctx, cipher, engine, rkey, riv)) + return AES_BAD_DATA; /* Error */ + int test = EVP_CIPHER_CTX_key_length(&ctx.ctx); + OPENSSL_assert(EVP_CIPHER_CTX_key_length(&ctx.ctx) == key_length); + OPENSSL_assert(EVP_CIPHER_CTX_iv_length(&ctx.ctx) == iv_length); + OPENSSL_assert(EVP_CIPHER_CTX_block_size(&ctx.ctx) == 16); + printf("Blocksize: %d \n", EVP_CIPHER_CTX_block_size(&ctx.ctx)); + if (! EVP_EncryptUpdate(&ctx.ctx, (unsigned char *) dest, &u_len, + (unsigned const char *) source, source_length)) + return AES_BAD_DATA; /* Error */ + if (! EVP_EncryptFinal_ex(&ctx.ctx, (unsigned char *) dest + u_len, &f_len)) + return AES_BAD_DATA; /* Error */ + *dest_length = (ulint) (u_len + f_len); +#endif + return AES_OK; +} + +int my_aes_decrypt_cbc(const char* source, ulint source_length, + char* dest, ulint *dest_length, + const char* key, uint8 key_length, + const char* iv, uint8 iv_length) +{ +#if defined(HAVE_OPENSSL) + MyCipherCtx ctx; + int u_len, f_len; + + /* The real key to be used for decryption */ + unsigned char rkey[key_length]; + my_aes_hexToUint(key, rkey, key_length); + + unsigned char riv[iv_length]; + my_aes_hexToUint(iv, riv, iv_length); + + const EVP_CIPHER* cipher; + switch(key_length) { + case 16: + cipher = EVP_aes_128_cbc(); + break; + case 24: + cipher = EVP_aes_192_cbc(); + break; + case 32: + cipher = EVP_aes_256_cbc(); + break; + default: + return AES_BAD_KEYSIZE; + } + //Initialize Encryption Engine here, default software Engine is default + ENGINE *engine = NULL; + + if (! EVP_DecryptInit_ex(&ctx.ctx, cipher, engine, rkey, riv)) + return AES_BAD_DATA; /* Error */ + OPENSSL_assert(EVP_CIPHER_CTX_key_length(&ctx.ctx) == key_length); + OPENSSL_assert(EVP_CIPHER_CTX_iv_length(&ctx.ctx) == iv_length); + OPENSSL_assert(EVP_CIPHER_CTX_block_size(&ctx.ctx) == 16); + if (! EVP_DecryptUpdate(&ctx.ctx, (unsigned char *) dest, &u_len, + (unsigned char *)source, source_length)) + return AES_BAD_DATA; /* Error */ + if (! EVP_DecryptFinal_ex(&ctx.ctx, (unsigned char *) dest + u_len, &f_len)) + return AES_BAD_DATA; /* Error */ + *dest_length = (ulint) (u_len + f_len); +#endif + return AES_OK; +} + + +int +my_aes_encrypt(const char* source, int source_length, char* dest, const char* key, int key_length) { #if defined(HAVE_YASSL) diff --git a/storage/xtradb/CMakeLists.txt b/storage/xtradb/CMakeLists.txt index c29888b8b15bc..d1df187fb84d6 100644 --- a/storage/xtradb/CMakeLists.txt +++ b/storage/xtradb/CMakeLists.txt @@ -337,6 +337,7 @@ SET(INNOBASE_SOURCES eval/eval0proc.cc fil/fil0fil.cc fil/fil0pagecompress.cc + fil/fil0pageencryption.cc fsp/fsp0fsp.cc fut/fut0fut.cc fut/fut0lst.cc diff --git a/storage/xtradb/buf/buf0buf.cc b/storage/xtradb/buf/buf0buf.cc index 359b15f4a6b8c..57146a694847a 100644 --- a/storage/xtradb/buf/buf0buf.cc +++ b/storage/xtradb/buf/buf0buf.cc @@ -57,6 +57,9 @@ Created 11/5/1995 Heikki Tuuri #include "trx0trx.h" #include "srv0start.h" +#include "fil0pageencryption.h" + + /* prototypes for new functions added to ha_innodb.cc */ trx_t* innobase_get_trx(); @@ -528,12 +531,13 @@ buf_page_is_corrupted( ulint zip_size) /*!< in: size of compressed page; 0 for uncompressed pages */ { + ulint page_encrypted = fil_page_is_encrypted(read_buf); ulint checksum_field1; ulint checksum_field2; ibool crc32_inited = FALSE; ib_uint32_t crc32 = ULINT32_UNDEFINED; - if (!zip_size + if (!page_encrypted && !zip_size && memcmp(read_buf + FIL_PAGE_LSN + 4, read_buf + UNIV_PAGE_SIZE - FIL_PAGE_END_LSN_OLD_CHKSUM + 4, 4)) { @@ -586,6 +590,9 @@ buf_page_is_corrupted( if (zip_size) { return(!page_zip_verify_checksum(read_buf, zip_size)); } + if (page_encrypted) { + return (FALSE); + } checksum_field1 = mach_read_from_4( read_buf + FIL_PAGE_SPACE_OR_CHKSUM); diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 017e96e611154..96a80aaab6be0 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -56,6 +56,10 @@ Created 10/25/1995 Heikki Tuuri static ulint srv_data_read, srv_data_written; #endif /* !UNIV_HOTBACKUP */ #include "fil0pagecompress.h" + +#include "fil0pageencryption.h" +#include "fsp0pageencryption.h" + #include "zlib.h" #ifdef __linux__ #include @@ -5267,7 +5271,7 @@ fil_extend_space_to_desired_size( success = os_aio(OS_FILE_WRITE, OS_AIO_SYNC, node->name, node->handle, buf, offset, page_size * n_pages, - NULL, NULL, space_id, NULL, 0, 0, 0); + NULL, NULL, space_id, NULL, 0, 0, 0, 0, 0); #endif /* UNIV_HOTBACKUP */ if (success) { os_has_said_disk_full = FALSE; @@ -5660,6 +5664,9 @@ _fil_io( ibool ignore_nonexistent_pages; ibool page_compressed = FALSE; ulint page_compression_level = 0; + ibool page_encrypted = FALSE; + ulint page_encryption_key = 0; + is_log = type & OS_FILE_LOG; type = type & ~OS_FILE_LOG; @@ -5729,6 +5736,11 @@ _fil_io( page_compressed = fsp_flags_is_page_compressed(space->flags); page_compression_level = fsp_flags_get_page_compression_level(space->flags); + + page_encrypted = fsp_flags_is_page_encrypted(space->flags); + page_encryption_key = fsp_flags_get_page_encryption_key(space->flags); + + /* If we are deleting a tablespace we don't allow any read operations on that. However, we do allow write operations. */ if (space == 0 || (type == OS_FILE_READ && space->stop_new_ops)) { @@ -5873,9 +5885,8 @@ _fil_io( } /* Queue the aio request */ - ret = os_aio(type, mode | wake_later, node->name, node->handle, buf, - offset, len, node, message, space_id, trx, - page_compressed, page_compression_level, write_size); + ret = os_aio(type, mode | wake_later, node->name, node->handle, buf, + offset, len, node, message, space_id, trx, page_compressed, page_compression_level, write_size, page_encrypted, page_encryption_key); #else /* In ibbackup do normal i/o, not aio */ diff --git a/storage/xtradb/fil/fil0fil.cc.orig b/storage/xtradb/fil/fil0fil.cc.orig new file mode 100644 index 0000000000000..96a80aaab6be0 --- /dev/null +++ b/storage/xtradb/fil/fil0fil.cc.orig @@ -0,0 +1,6885 @@ +/***************************************************************************** + +Copyright (c) 1995, 2014, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2013, 2014, SkySQL Ab. All Rights Reserved. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA + +*****************************************************************************/ + +/**************************************************//** +@file fil/fil0fil.cc +The tablespace memory cache + +Created 10/25/1995 Heikki Tuuri +*******************************************************/ + +#include "fil0fil.h" + +#include +#include + +#include "mem0mem.h" +#include "hash0hash.h" +#include "os0file.h" +#include "mach0data.h" +#include "buf0buf.h" +#include "buf0flu.h" +#include "log0recv.h" +#include "fsp0fsp.h" +#include "srv0srv.h" +#include "srv0start.h" +#include "mtr0mtr.h" +#include "mtr0log.h" +#include "dict0dict.h" +#include "page0page.h" +#include "page0zip.h" +#include "trx0sys.h" +#include "row0mysql.h" +#ifndef UNIV_HOTBACKUP +# include "buf0lru.h" +# include "ibuf0ibuf.h" +# include "sync0sync.h" +# include "os0sync.h" +#else /* !UNIV_HOTBACKUP */ +# include "srv0srv.h" +static ulint srv_data_read, srv_data_written; +#endif /* !UNIV_HOTBACKUP */ +#include "fil0pagecompress.h" + +#include "fil0pageencryption.h" +#include "fsp0pageencryption.h" + +#include "zlib.h" +#ifdef __linux__ +#include +#include +#include +#endif +#include "row0mysql.h" + +/* + IMPLEMENTATION OF THE TABLESPACE MEMORY CACHE + ============================================= + +The tablespace cache is responsible for providing fast read/write access to +tablespaces and logs of the database. File creation and deletion is done +in other modules which know more of the logic of the operation, however. + +A tablespace consists of a chain of files. The size of the files does not +have to be divisible by the database block size, because we may just leave +the last incomplete block unused. When a new file is appended to the +tablespace, the maximum size of the file is also specified. At the moment, +we think that it is best to extend the file to its maximum size already at +the creation of the file, because then we can avoid dynamically extending +the file when more space is needed for the tablespace. + +A block's position in the tablespace is specified with a 32-bit unsigned +integer. The files in the chain are thought to be catenated, and the block +corresponding to an address n is the nth block in the catenated file (where +the first block is named the 0th block, and the incomplete block fragments +at the end of files are not taken into account). A tablespace can be extended +by appending a new file at the end of the chain. + +Our tablespace concept is similar to the one of Oracle. + +To acquire more speed in disk transfers, a technique called disk striping is +sometimes used. This means that logical block addresses are divided in a +round-robin fashion across several disks. Windows NT supports disk striping, +so there we do not need to support it in the database. Disk striping is +implemented in hardware in RAID disks. We conclude that it is not necessary +to implement it in the database. Oracle 7 does not support disk striping, +either. + +Another trick used at some database sites is replacing tablespace files by +raw disks, that is, the whole physical disk drive, or a partition of it, is +opened as a single file, and it is accessed through byte offsets calculated +from the start of the disk or the partition. This is recommended in some +books on database tuning to achieve more speed in i/o. Using raw disk +certainly prevents the OS from fragmenting disk space, but it is not clear +if it really adds speed. We measured on the Pentium 100 MHz + NT + NTFS file +system + EIDE Conner disk only a negligible difference in speed when reading +from a file, versus reading from a raw disk. + +To have fast access to a tablespace or a log file, we put the data structures +to a hash table. Each tablespace and log file is given an unique 32-bit +identifier. + +Some operating systems do not support many open files at the same time, +though NT seems to tolerate at least 900 open files. Therefore, we put the +open files in an LRU-list. If we need to open another file, we may close the +file at the end of the LRU-list. When an i/o-operation is pending on a file, +the file cannot be closed. We take the file nodes with pending i/o-operations +out of the LRU-list and keep a count of pending operations. When an operation +completes, we decrement the count and return the file node to the LRU-list if +the count drops to zero. */ + +/** When mysqld is run, the default directory "." is the mysqld datadir, +but in the MySQL Embedded Server Library and ibbackup it is not the default +directory, and we must set the base file path explicitly */ +UNIV_INTERN const char* fil_path_to_mysql_datadir = "."; + +/** The number of fsyncs done to the log */ +UNIV_INTERN ulint fil_n_log_flushes = 0; + +/** Number of pending redo log flushes */ +UNIV_INTERN ulint fil_n_pending_log_flushes = 0; +/** Number of pending tablespace flushes */ +UNIV_INTERN ulint fil_n_pending_tablespace_flushes = 0; + +/** Number of files currently open */ +UNIV_INTERN ulint fil_n_file_opened = 0; + +/** The null file address */ +UNIV_INTERN fil_addr_t fil_addr_null = {FIL_NULL, 0}; + +#ifdef UNIV_PFS_MUTEX +/* Key to register fil_system_mutex with performance schema */ +UNIV_INTERN mysql_pfs_key_t fil_system_mutex_key; +#endif /* UNIV_PFS_MUTEX */ + +#ifdef UNIV_PFS_RWLOCK +/* Key to register file space latch with performance schema */ +UNIV_INTERN mysql_pfs_key_t fil_space_latch_key; +#endif /* UNIV_PFS_RWLOCK */ + +/** File node of a tablespace or the log data space */ +struct fil_node_t { + fil_space_t* space; /*!< backpointer to the space where this node + belongs */ + char* name; /*!< path to the file */ + ibool open; /*!< TRUE if file open */ + os_file_t handle; /*!< OS handle to the file, if file open */ + os_event_t sync_event;/*!< Condition event to group and + serialize calls to fsync */ + ibool is_raw_disk;/*!< TRUE if the 'file' is actually a raw + device or a raw disk partition */ + ulint size; /*!< size of the file in database pages, 0 if + not known yet; the possible last incomplete + megabyte may be ignored if space == 0 */ + ulint n_pending; + /*!< count of pending i/o's on this file; + closing of the file is not allowed if + this is > 0 */ + ulint n_pending_flushes; + /*!< count of pending flushes on this file; + closing of the file is not allowed if + this is > 0 */ + ibool being_extended; + /*!< TRUE if the node is currently + being extended. */ + ib_int64_t modification_counter;/*!< when we write to the file we + increment this by one */ + ib_int64_t flush_counter;/*!< up to what + modification_counter value we have + flushed the modifications to disk */ + UT_LIST_NODE_T(fil_node_t) chain; + /*!< link field for the file chain */ + UT_LIST_NODE_T(fil_node_t) LRU; + /*!< link field for the LRU list */ + ulint magic_n;/*!< FIL_NODE_MAGIC_N */ +}; + +/** Value of fil_node_t::magic_n */ +#define FIL_NODE_MAGIC_N 89389 + +/** Tablespace or log data space: let us call them by a common name space */ +struct fil_space_t { + char* name; /*!< space name = the path to the first file in + it */ + ulint id; /*!< space id */ + ib_int64_t tablespace_version; + /*!< in DISCARD/IMPORT this timestamp + is used to check if we should ignore + an insert buffer merge request for a + page because it actually was for the + previous incarnation of the space */ + ibool mark; /*!< this is set to TRUE at database startup if + the space corresponds to a table in the InnoDB + data dictionary; so we can print a warning of + orphaned tablespaces */ + ibool stop_ios;/*!< TRUE if we want to rename the + .ibd file of tablespace and want to + stop temporarily posting of new i/o + requests on the file */ + ibool stop_new_ops; + /*!< we set this TRUE when we start + deleting a single-table tablespace. + When this is set following new ops + are not allowed: + * read IO request + * ibuf merge + * file flush + Note that we can still possibly have + new write operations because we don't + check this flag when doing flush + batches. */ + ulint purpose;/*!< FIL_TABLESPACE, FIL_LOG, or + FIL_ARCH_LOG */ + UT_LIST_BASE_NODE_T(fil_node_t) chain; + /*!< base node for the file chain */ + ulint size; /*!< space size in pages; 0 if a single-table + tablespace whose size we do not know yet; + last incomplete megabytes in data files may be + ignored if space == 0 */ + ulint flags; /*!< tablespace flags; see + fsp_flags_is_valid(), + fsp_flags_get_zip_size() */ + ulint n_reserved_extents; + /*!< number of reserved free extents for + ongoing operations like B-tree page split */ + ulint n_pending_flushes; /*!< this is positive when flushing + the tablespace to disk; dropping of the + tablespace is forbidden if this is positive */ + ulint n_pending_ops;/*!< this is positive when we + have pending operations against this + tablespace. The pending operations can + be ibuf merges or lock validation code + trying to read a block. + Dropping of the tablespace is forbidden + if this is positive */ + hash_node_t hash; /*!< hash chain node */ + hash_node_t name_hash;/*!< hash chain the name_hash table */ +#ifndef UNIV_HOTBACKUP + prio_rw_lock_t latch; /*!< latch protecting the file space storage + allocation */ +#endif /* !UNIV_HOTBACKUP */ + UT_LIST_NODE_T(fil_space_t) unflushed_spaces; + /*!< list of spaces with at least one unflushed + file we have written to */ + bool is_in_unflushed_spaces; + /*!< true if this space is currently in + unflushed_spaces */ + ibool is_corrupt; + UT_LIST_NODE_T(fil_space_t) space_list; + /*!< list of all spaces */ + ulint magic_n;/*!< FIL_SPACE_MAGIC_N */ +}; + +/** Value of fil_space_t::magic_n */ +#define FIL_SPACE_MAGIC_N 89472 + +/** The tablespace memory cache; also the totality of logs (the log +data space) is stored here; below we talk about tablespaces, but also +the ib_logfiles form a 'space' and it is handled here */ +struct fil_system_t { +#ifndef UNIV_HOTBACKUP + ib_mutex_t mutex; /*!< The mutex protecting the cache */ +#endif /* !UNIV_HOTBACKUP */ + hash_table_t* spaces; /*!< The hash table of spaces in the + system; they are hashed on the space + id */ + hash_table_t* name_hash; /*!< hash table based on the space + name */ + UT_LIST_BASE_NODE_T(fil_node_t) LRU; + /*!< base node for the LRU list of the + most recently used open files with no + pending i/o's; if we start an i/o on + the file, we first remove it from this + list, and return it to the start of + the list when the i/o ends; + log files and the system tablespace are + not put to this list: they are opened + after the startup, and kept open until + shutdown */ + UT_LIST_BASE_NODE_T(fil_space_t) unflushed_spaces; + /*!< base node for the list of those + tablespaces whose files contain + unflushed writes; those spaces have + at least one file node where + modification_counter > flush_counter */ + ulint n_open; /*!< number of files currently open */ + ulint max_n_open; /*!< n_open is not allowed to exceed + this */ + ib_int64_t modification_counter;/*!< when we write to a file we + increment this by one */ + ulint max_assigned_id;/*!< maximum space id in the existing + tables, or assigned during the time + mysqld has been up; at an InnoDB + startup we scan the data dictionary + and set here the maximum of the + space id's of the tables there */ + ib_int64_t tablespace_version; + /*!< a counter which is incremented for + every space object memory creation; + every space mem object gets a + 'timestamp' from this; in DISCARD/ + IMPORT this is used to check if we + should ignore an insert buffer merge + request */ + UT_LIST_BASE_NODE_T(fil_space_t) space_list; + /*!< list of all file spaces */ + ibool space_id_reuse_warned; + /* !< TRUE if fil_space_create() + has issued a warning about + potential space_id reuse */ +}; + +/** The tablespace memory cache. This variable is NULL before the module is +initialized. */ +static fil_system_t* fil_system = NULL; + +/** Determine if (i) is a user tablespace id or not. */ +# define fil_is_user_tablespace_id(i) ((i) > srv_undo_tablespaces_open) + +/** Determine if user has explicitly disabled fsync(). */ +#ifndef __WIN__ +# define fil_buffering_disabled(s) \ + (((s)->purpose == FIL_TABLESPACE \ + && srv_unix_file_flush_method == SRV_UNIX_O_DIRECT_NO_FSYNC)\ + || ((s)->purpose == FIL_LOG \ + && srv_unix_file_flush_method == SRV_UNIX_ALL_O_DIRECT)) + +#else /* __WIN__ */ +# define fil_buffering_disabled(s) (0) +#endif /* __WIN__ */ + +#ifdef UNIV_DEBUG +/** Try fil_validate() every this many times */ +# define FIL_VALIDATE_SKIP 17 + +/******************************************************************//** +Checks the consistency of the tablespace cache some of the time. +@return TRUE if ok or the check was skipped */ +static +ibool +fil_validate_skip(void) +/*===================*/ +{ + /** The fil_validate() call skip counter. Use a signed type + because of the race condition below. */ + static int fil_validate_count = FIL_VALIDATE_SKIP; + + /* There is a race condition below, but it does not matter, + because this call is only for heuristic purposes. We want to + reduce the call frequency of the costly fil_validate() check + in debug builds. */ + if (--fil_validate_count > 0) { + return(TRUE); + } + + fil_validate_count = FIL_VALIDATE_SKIP; + return(fil_validate()); +} +#endif /* UNIV_DEBUG */ + +/********************************************************************//** +Determines if a file node belongs to the least-recently-used list. +@return TRUE if the file belongs to fil_system->LRU mutex. */ +UNIV_INLINE +ibool +fil_space_belongs_in_lru( +/*=====================*/ + const fil_space_t* space) /*!< in: file space */ +{ + return(space->purpose == FIL_TABLESPACE + && fil_is_user_tablespace_id(space->id)); +} + +/********************************************************************//** +NOTE: you must call fil_mutex_enter_and_prepare_for_io() first! + +Prepares a file node for i/o. Opens the file if it is closed. Updates the +pending i/o's field in the node and the system appropriately. Takes the node +off the LRU list if it is in the LRU list. The caller must hold the fil_sys +mutex. +@return false if the file can't be opened, otherwise true */ +static +bool +fil_node_prepare_for_io( +/*====================*/ + fil_node_t* node, /*!< in: file node */ + fil_system_t* system, /*!< in: tablespace memory cache */ + fil_space_t* space); /*!< in: space */ +/********************************************************************//** +Updates the data structures when an i/o operation finishes. Updates the +pending i/o's field in the node appropriately. */ +static +void +fil_node_complete_io( +/*=================*/ + fil_node_t* node, /*!< in: file node */ + fil_system_t* system, /*!< in: tablespace memory cache */ + ulint type); /*!< in: OS_FILE_WRITE or OS_FILE_READ; marks + the node as modified if + type == OS_FILE_WRITE */ +/*******************************************************************//** +Frees a space object from the tablespace memory cache. Closes the files in +the chain but does not delete them. There must not be any pending i/o's or +flushes on the files. +@return TRUE on success */ +static +ibool +fil_space_free( +/*===========*/ + ulint id, /* in: space id */ + ibool x_latched); /* in: TRUE if caller has space->latch + in X mode */ +/********************************************************************//** +Reads data from a space to a buffer. Remember that the possible incomplete +blocks at the end of file are ignored: they are not taken into account when +calculating the byte offset within a space. +@return DB_SUCCESS, or DB_TABLESPACE_DELETED if we are trying to do +i/o on a tablespace which does not exist */ +UNIV_INLINE +dberr_t +fil_read( +/*=====*/ + bool sync, /*!< in: true if synchronous aio is desired */ + ulint space_id, /*!< in: space id */ + ulint zip_size, /*!< in: compressed page size in bytes; + 0 for uncompressed pages */ + ulint block_offset, /*!< in: offset in number of blocks */ + ulint byte_offset, /*!< in: remainder of offset in bytes; in aio + this must be divisible by the OS block size */ + ulint len, /*!< in: how many bytes to read; this must not + cross a file boundary; in aio this must be a + block size multiple */ + void* buf, /*!< in/out: buffer where to store data read; + in aio this must be appropriately aligned */ + void* message, /*!< in: message for aio handler if non-sync + aio used, else ignored */ + ulint* write_size) /*!< in/out: Actual write size initialized + after fist successfull trim + operation for this page and if + initialized we do not trim again if + actual page size does not decrease. */ +{ + return(fil_io(OS_FILE_READ, sync, space_id, zip_size, block_offset, + byte_offset, len, buf, message, write_size)); +} + +/********************************************************************//** +Writes data to a space from a buffer. Remember that the possible incomplete +blocks at the end of file are ignored: they are not taken into account when +calculating the byte offset within a space. +@return DB_SUCCESS, or DB_TABLESPACE_DELETED if we are trying to do +i/o on a tablespace which does not exist */ +UNIV_INLINE +dberr_t +fil_write( +/*======*/ + bool sync, /*!< in: true if synchronous aio is desired */ + ulint space_id, /*!< in: space id */ + ulint zip_size, /*!< in: compressed page size in bytes; + 0 for uncompressed pages */ + ulint block_offset, /*!< in: offset in number of blocks */ + ulint byte_offset, /*!< in: remainder of offset in bytes; in aio + this must be divisible by the OS block size */ + ulint len, /*!< in: how many bytes to write; this must + not cross a file boundary; in aio this must + be a block size multiple */ + void* buf, /*!< in: buffer from which to write; in aio + this must be appropriately aligned */ + void* message, /*!< in: message for aio handler if non-sync + aio used, else ignored */ + ulint* write_size) /*!< in/out: Actual write size initialized + after fist successfull trim + operation for this page and if + initialized we do not trim again if + actual page size does not decrease. */ +{ + ut_ad(!srv_read_only_mode); + + return(fil_io(OS_FILE_WRITE, sync, space_id, zip_size, block_offset, + byte_offset, len, buf, message, write_size)); +} + +/*******************************************************************//** +Returns the table space by a given id, NULL if not found. */ +fil_space_t* +fil_space_get_by_id( +/*================*/ + ulint id) /*!< in: space id */ +{ + fil_space_t* space; + + ut_ad(mutex_own(&fil_system->mutex)); + + HASH_SEARCH(hash, fil_system->spaces, id, + fil_space_t*, space, + ut_ad(space->magic_n == FIL_SPACE_MAGIC_N), + space->id == id); + + return(space); +} + +/****************************************************************//** +Get space id from fil node */ +ulint +fil_node_get_space_id( +/*==================*/ + fil_node_t* node) /*!< in: Compressed node*/ +{ + ut_ad(node); + ut_ad(node->space); + + return (node->space->id); +} + +/*******************************************************************//** +Returns the table space by a given name, NULL if not found. */ +UNIV_INLINE +fil_space_t* +fil_space_get_by_name( +/*==================*/ + const char* name) /*!< in: space name */ +{ + fil_space_t* space; + ulint fold; + + ut_ad(mutex_own(&fil_system->mutex)); + + fold = ut_fold_string(name); + + HASH_SEARCH(name_hash, fil_system->name_hash, fold, + fil_space_t*, space, + ut_ad(space->magic_n == FIL_SPACE_MAGIC_N), + !strcmp(name, space->name)); + + return(space); +} + +#ifndef UNIV_HOTBACKUP +/*******************************************************************//** +Returns the version number of a tablespace, -1 if not found. +@return version number, -1 if the tablespace does not exist in the +memory cache */ +UNIV_INTERN +ib_int64_t +fil_space_get_version( +/*==================*/ + ulint id) /*!< in: space id */ +{ + fil_space_t* space; + ib_int64_t version = -1; + + ut_ad(fil_system); + + mutex_enter(&fil_system->mutex); + + space = fil_space_get_by_id(id); + + if (space) { + version = space->tablespace_version; + } + + mutex_exit(&fil_system->mutex); + + return(version); +} + +/*******************************************************************//** +Returns the latch of a file space. +@return latch protecting storage allocation */ +UNIV_INTERN +prio_rw_lock_t* +fil_space_get_latch( +/*================*/ + ulint id, /*!< in: space id */ + ulint* flags) /*!< out: tablespace flags */ +{ + fil_space_t* space; + + ut_ad(fil_system); + + mutex_enter(&fil_system->mutex); + + space = fil_space_get_by_id(id); + + ut_a(space); + + if (flags) { + *flags = space->flags; + } + + mutex_exit(&fil_system->mutex); + + return(&(space->latch)); +} + +/*******************************************************************//** +Returns the type of a file space. +@return FIL_TABLESPACE or FIL_LOG */ +UNIV_INTERN +ulint +fil_space_get_type( +/*===============*/ + ulint id) /*!< in: space id */ +{ + fil_space_t* space; + + ut_ad(fil_system); + + mutex_enter(&fil_system->mutex); + + space = fil_space_get_by_id(id); + + ut_a(space); + + mutex_exit(&fil_system->mutex); + + return(space->purpose); +} +#endif /* !UNIV_HOTBACKUP */ + +/**********************************************************************//** +Checks if all the file nodes in a space are flushed. The caller must hold +the fil_system mutex. +@return true if all are flushed */ +static +bool +fil_space_is_flushed( +/*=================*/ + fil_space_t* space) /*!< in: space */ +{ + fil_node_t* node; + + ut_ad(mutex_own(&fil_system->mutex)); + + node = UT_LIST_GET_FIRST(space->chain); + + while (node) { + if (node->modification_counter > node->flush_counter) { + + ut_ad(!fil_buffering_disabled(space)); + return(false); + } + + node = UT_LIST_GET_NEXT(chain, node); + } + + return(true); +} + +/*******************************************************************//** +Appends a new file to the chain of files of a space. File must be closed. +@return pointer to the file name, or NULL on error */ +UNIV_INTERN +char* +fil_node_create( +/*============*/ + const char* name, /*!< in: file name (file must be closed) */ + ulint size, /*!< in: file size in database blocks, rounded + downwards to an integer */ + ulint id, /*!< in: space id where to append */ + ibool is_raw) /*!< in: TRUE if a raw device or + a raw disk partition */ +{ + fil_node_t* node; + fil_space_t* space; + + ut_a(fil_system); + ut_a(name); + + mutex_enter(&fil_system->mutex); + + node = static_cast(mem_zalloc(sizeof(fil_node_t))); + + node->name = mem_strdup(name); + + ut_a(!is_raw || srv_start_raw_disk_in_use); + + node->sync_event = os_event_create(); + node->is_raw_disk = is_raw; + node->size = size; + node->magic_n = FIL_NODE_MAGIC_N; + + space = fil_space_get_by_id(id); + + if (!space) { + ut_print_timestamp(stderr); + fprintf(stderr, + " InnoDB: Error: Could not find tablespace %lu for\n" + "InnoDB: file ", (ulong) id); + ut_print_filename(stderr, name); + fputs(" in the tablespace memory cache.\n", stderr); + mem_free(node->name); + + mem_free(node); + + mutex_exit(&fil_system->mutex); + + return(NULL); + } + + space->size += size; + + node->space = space; + + UT_LIST_ADD_LAST(chain, space->chain, node); + + if (id < SRV_LOG_SPACE_FIRST_ID && fil_system->max_assigned_id < id) { + + fil_system->max_assigned_id = id; + } + + mutex_exit(&fil_system->mutex); + + return(node->name); +} + +/********************************************************************//** +Opens a file of a node of a tablespace. The caller must own the fil_system +mutex. +@return false if the file can't be opened, otherwise true */ +static +bool +fil_node_open_file( +/*===============*/ + fil_node_t* node, /*!< in: file node */ + fil_system_t* system, /*!< in: tablespace memory cache */ + fil_space_t* space) /*!< in: space */ +{ + os_offset_t size_bytes; + ibool ret; + ibool success; + byte* buf2; + byte* page; + ulint space_id; + ulint flags=0; + ulint page_size; + ulint atomic_writes=0; + + ut_ad(mutex_own(&(system->mutex))); + ut_a(node->n_pending == 0); + ut_a(node->open == FALSE); + + if (node->size == 0) { + /* It must be a single-table tablespace and we do not know the + size of the file yet. First we open the file in the normal + mode, no async I/O here, for simplicity. Then do some checks, + and close the file again. + NOTE that we could not use the simple file read function + os_file_read() in Windows to read from a file opened for + async I/O! */ + + node->handle = os_file_create_simple_no_error_handling( + innodb_file_data_key, node->name, OS_FILE_OPEN, + OS_FILE_READ_ONLY, &success, 0); + if (!success) { + /* The following call prints an error message */ + os_file_get_last_error(true); + + ut_print_timestamp(stderr); + + ib_logf(IB_LOG_LEVEL_WARN, "InnoDB: Error: cannot " + "open %s\n. InnoDB: Have you deleted .ibd " + "files under a running mysqld server?\n", + node->name); + + return(false); + } + + size_bytes = os_file_get_size(node->handle); + ut_a(size_bytes != (os_offset_t) -1); +#ifdef UNIV_HOTBACKUP + if (space->id == 0) { + node->size = (ulint) (size_bytes / UNIV_PAGE_SIZE); + os_file_close(node->handle); + goto add_size; + } +#endif /* UNIV_HOTBACKUP */ + ut_a(space->purpose != FIL_LOG); + ut_a(fil_is_user_tablespace_id(space->id)); + + if (size_bytes < FIL_IBD_FILE_INITIAL_SIZE * UNIV_PAGE_SIZE) { + fprintf(stderr, + "InnoDB: Error: the size of single-table" + " tablespace file %s\n" + "InnoDB: is only "UINT64PF"," + " should be at least %lu!\n", + node->name, + size_bytes, + (ulong) (FIL_IBD_FILE_INITIAL_SIZE + * UNIV_PAGE_SIZE)); + + ut_a(0); + } + + /* Read the first page of the tablespace */ + + buf2 = static_cast(ut_malloc(2 * UNIV_PAGE_SIZE)); + /* Align the memory for file i/o if we might have O_DIRECT + set */ + page = static_cast(ut_align(buf2, UNIV_PAGE_SIZE)); + + success = os_file_read(node->handle, page, 0, UNIV_PAGE_SIZE, + space->flags); + + space_id = fsp_header_get_space_id(page); + flags = fsp_header_get_flags(page); + page_size = fsp_flags_get_page_size(flags); + atomic_writes = fsp_flags_get_atomic_writes(flags); + + ut_free(buf2); + + /* Close the file now that we have read the space id from it */ + + os_file_close(node->handle); + + if (UNIV_UNLIKELY(space_id != space->id)) { + fprintf(stderr, + "InnoDB: Error: tablespace id is %lu" + " in the data dictionary\n" + "InnoDB: but in file %s it is %lu!\n", + space->id, node->name, space_id); + + ut_error; + } + + if (UNIV_UNLIKELY(space_id == ULINT_UNDEFINED + || space_id == 0)) { + fprintf(stderr, + "InnoDB: Error: tablespace id %lu" + " in file %s is not sensible\n", + (ulong) space_id, node->name); + + ut_error; + } + + if (UNIV_UNLIKELY(fsp_flags_get_page_size(space->flags) + != page_size)) { + fprintf(stderr, + "InnoDB: Error: tablespace file %s" + " has page size 0x%lx\n" + "InnoDB: but the data dictionary" + " expects page size 0x%lx!\n", + node->name, flags, + fsp_flags_get_page_size(space->flags)); + + ut_error; + } + + if (UNIV_UNLIKELY(space->flags != flags)) { + fprintf(stderr, + "InnoDB: Error: table flags are 0x%lx" + " in the data dictionary\n" + "InnoDB: but the flags in file %s are 0x%lx!\n", + space->flags, node->name, flags); + + ut_error; + } + + if (UNIV_UNLIKELY(space->flags != flags)) { + if (!dict_tf_verify_flags(space->flags, flags)) { + fprintf(stderr, + "InnoDB: Error: table flags are 0x%lx" + " in the data dictionary\n" + "InnoDB: but the flags in file %s are 0x%lx!\n", + space->flags, node->name, flags); + ut_error; + } + } + + if (size_bytes >= FSP_EXTENT_SIZE * UNIV_PAGE_SIZE) { + /* Truncate the size to whole extent size. */ + size_bytes = ut_2pow_round(size_bytes, + FSP_EXTENT_SIZE * + UNIV_PAGE_SIZE); + } + + if (!fsp_flags_is_compressed(flags)) { + node->size = (ulint) + (size_bytes + / fsp_flags_get_page_size(flags)); + } else { + node->size = (ulint) + (size_bytes + / fsp_flags_get_zip_size(flags)); + } + +#ifdef UNIV_HOTBACKUP +add_size: +#endif /* UNIV_HOTBACKUP */ + space->size += node->size; + } + + atomic_writes = fsp_flags_get_atomic_writes(space->flags); + + /* printf("Opening file %s\n", node->name); */ + + /* Open the file for reading and writing, in Windows normally in the + unbuffered async I/O mode, though global variables may make + os_file_create() to fall back to the normal file I/O mode. */ + + if (space->purpose == FIL_LOG) { + node->handle = os_file_create(innodb_file_log_key, + node->name, OS_FILE_OPEN, + OS_FILE_AIO, OS_LOG_FILE, + &ret, atomic_writes); + } else if (node->is_raw_disk) { + node->handle = os_file_create(innodb_file_data_key, + node->name, + OS_FILE_OPEN_RAW, + OS_FILE_AIO, OS_DATA_FILE, + &ret, atomic_writes); + } else { + node->handle = os_file_create(innodb_file_data_key, + node->name, OS_FILE_OPEN, + OS_FILE_AIO, OS_DATA_FILE, + &ret, atomic_writes); + } + + ut_a(ret); + + node->open = TRUE; + + system->n_open++; + fil_n_file_opened++; + + if (fil_space_belongs_in_lru(space)) { + + /* Put the node to the LRU list */ + UT_LIST_ADD_FIRST(LRU, system->LRU, node); + } + + return(true); +} + +/**********************************************************************//** +Closes a file. */ +static +void +fil_node_close_file( +/*================*/ + fil_node_t* node, /*!< in: file node */ + fil_system_t* system) /*!< in: tablespace memory cache */ +{ + ibool ret; + + ut_ad(node && system); + ut_ad(mutex_own(&(system->mutex))); + ut_a(node->open); + ut_a(node->n_pending == 0); + ut_a(node->n_pending_flushes == 0); + ut_a(!node->being_extended); +#ifndef UNIV_HOTBACKUP + ut_a(node->modification_counter == node->flush_counter + || srv_fast_shutdown == 2); +#endif /* !UNIV_HOTBACKUP */ + + ret = os_file_close(node->handle); + ut_a(ret); + + /* printf("Closing file %s\n", node->name); */ + + node->open = FALSE; + ut_a(system->n_open > 0); + system->n_open--; + fil_n_file_opened--; + + if (fil_space_belongs_in_lru(node->space)) { + + ut_a(UT_LIST_GET_LEN(system->LRU) > 0); + + /* The node is in the LRU list, remove it */ + UT_LIST_REMOVE(LRU, system->LRU, node); + } +} + +/********************************************************************//** +Tries to close a file in the LRU list. The caller must hold the fil_sys +mutex. +@return TRUE if success, FALSE if should retry later; since i/o's +generally complete in < 100 ms, and as InnoDB writes at most 128 pages +from the buffer pool in a batch, and then immediately flushes the +files, there is a good chance that the next time we find a suitable +node from the LRU list */ +static +ibool +fil_try_to_close_file_in_LRU( +/*=========================*/ + ibool print_info) /*!< in: if TRUE, prints information why it + cannot close a file */ +{ + fil_node_t* node; + + ut_ad(mutex_own(&fil_system->mutex)); + + if (print_info) { + fprintf(stderr, + "InnoDB: fil_sys open file LRU len %lu\n", + (ulong) UT_LIST_GET_LEN(fil_system->LRU)); + } + + for (node = UT_LIST_GET_LAST(fil_system->LRU); + node != NULL; + node = UT_LIST_GET_PREV(LRU, node)) { + + if (node->modification_counter == node->flush_counter + && node->n_pending_flushes == 0 + && !node->being_extended) { + + fil_node_close_file(node, fil_system); + + return(TRUE); + } + + if (!print_info) { + continue; + } + + if (node->n_pending_flushes > 0) { + fputs("InnoDB: cannot close file ", stderr); + ut_print_filename(stderr, node->name); + fprintf(stderr, ", because n_pending_flushes %lu\n", + (ulong) node->n_pending_flushes); + } + + if (node->modification_counter != node->flush_counter) { + fputs("InnoDB: cannot close file ", stderr); + ut_print_filename(stderr, node->name); + fprintf(stderr, + ", because mod_count %ld != fl_count %ld\n", + (long) node->modification_counter, + (long) node->flush_counter); + + } + + if (node->being_extended) { + fputs("InnoDB: cannot close file ", stderr); + ut_print_filename(stderr, node->name); + fprintf(stderr, ", because it is being extended\n"); + } + } + + return(FALSE); +} + +/*******************************************************************//** +Reserves the fil_system mutex and tries to make sure we can open at least one +file while holding it. This should be called before calling +fil_node_prepare_for_io(), because that function may need to open a file. */ +static +void +fil_mutex_enter_and_prepare_for_io( +/*===============================*/ + ulint space_id) /*!< in: space id */ +{ + fil_space_t* space; + ibool success; + ibool print_info = FALSE; + ulint count = 0; + ulint count2 = 0; + +retry: + mutex_enter(&fil_system->mutex); + + if (space_id == 0 || space_id >= SRV_LOG_SPACE_FIRST_ID) { + /* We keep log files and system tablespace files always open; + this is important in preventing deadlocks in this module, as + a page read completion often performs another read from the + insert buffer. The insert buffer is in tablespace 0, and we + cannot end up waiting in this function. */ + + return; + } + + space = fil_space_get_by_id(space_id); + + if (space != NULL && space->stop_ios) { + /* We are going to do a rename file and want to stop new i/o's + for a while */ + + if (count2 > 20000) { + fputs("InnoDB: Warning: tablespace ", stderr); + ut_print_filename(stderr, space->name); + fprintf(stderr, + " has i/o ops stopped for a long time %lu\n", + (ulong) count2); + } + + mutex_exit(&fil_system->mutex); + +#ifndef UNIV_HOTBACKUP + + /* Wake the i/o-handler threads to make sure pending + i/o's are performed */ + os_aio_simulated_wake_handler_threads(); + + /* The sleep here is just to give IO helper threads a + bit of time to do some work. It is not required that + all IO related to the tablespace being renamed must + be flushed here as we do fil_flush() in + fil_rename_tablespace() as well. */ + os_thread_sleep(20000); + +#endif /* UNIV_HOTBACKUP */ + + /* Flush tablespaces so that we can close modified + files in the LRU list */ + fil_flush_file_spaces(FIL_TABLESPACE); + + os_thread_sleep(20000); + + count2++; + + goto retry; + } + + if (fil_system->n_open < fil_system->max_n_open) { + + return; + } + + /* If the file is already open, no need to do anything; if the space + does not exist, we handle the situation in the function which called + this function */ + + if (!space || UT_LIST_GET_FIRST(space->chain)->open) { + + return; + } + + if (count > 1) { + print_info = TRUE; + } + + /* Too many files are open, try to close some */ +close_more: + success = fil_try_to_close_file_in_LRU(print_info); + + if (success && fil_system->n_open >= fil_system->max_n_open) { + + goto close_more; + } + + if (fil_system->n_open < fil_system->max_n_open) { + /* Ok */ + + return; + } + + if (count >= 2) { + ut_print_timestamp(stderr); + fprintf(stderr, + " InnoDB: Warning: too many (%lu) files stay open" + " while the maximum\n" + "InnoDB: allowed value would be %lu.\n" + "InnoDB: You may need to raise the value of" + " innodb_open_files in\n" + "InnoDB: my.cnf.\n", + (ulong) fil_system->n_open, + (ulong) fil_system->max_n_open); + + return; + } + + mutex_exit(&fil_system->mutex); + +#ifndef UNIV_HOTBACKUP + /* Wake the i/o-handler threads to make sure pending i/o's are + performed */ + os_aio_simulated_wake_handler_threads(); + + os_thread_sleep(20000); +#endif + /* Flush tablespaces so that we can close modified files in the LRU + list */ + + fil_flush_file_spaces(FIL_TABLESPACE); + + count++; + + goto retry; +} + +/*******************************************************************//** +Frees a file node object from a tablespace memory cache. */ +static +void +fil_node_free( +/*==========*/ + fil_node_t* node, /*!< in, own: file node */ + fil_system_t* system, /*!< in: tablespace memory cache */ + fil_space_t* space) /*!< in: space where the file node is chained */ +{ + ut_ad(node && system && space); + ut_ad(mutex_own(&(system->mutex))); + ut_a(node->magic_n == FIL_NODE_MAGIC_N); + ut_a(node->n_pending == 0); + ut_a(!node->being_extended); + + if (node->open) { + /* We fool the assertion in fil_node_close_file() to think + there are no unflushed modifications in the file */ + + node->modification_counter = node->flush_counter; + os_event_set(node->sync_event); + + if (fil_buffering_disabled(space)) { + + ut_ad(!space->is_in_unflushed_spaces); + ut_ad(fil_space_is_flushed(space)); + + } else if (space->is_in_unflushed_spaces + && fil_space_is_flushed(space)) { + + space->is_in_unflushed_spaces = false; + + UT_LIST_REMOVE(unflushed_spaces, + system->unflushed_spaces, + space); + } + + fil_node_close_file(node, system); + } + + space->size -= node->size; + + UT_LIST_REMOVE(chain, space->chain, node); + + os_event_free(node->sync_event); + mem_free(node->name); + mem_free(node); +} + +#ifdef UNIV_LOG_ARCHIVE +/****************************************************************//** +Drops files from the start of a file space, so that its size is cut by +the amount given. */ +UNIV_INTERN +void +fil_space_truncate_start( +/*=====================*/ + ulint id, /*!< in: space id */ + ulint trunc_len) /*!< in: truncate by this much; it is an error + if this does not equal to the combined size of + some initial files in the space */ +{ + fil_node_t* node; + fil_space_t* space; + + mutex_enter(&fil_system->mutex); + + space = fil_space_get_by_id(id); + + ut_a(space); + + while (trunc_len > 0) { + node = UT_LIST_GET_FIRST(space->chain); + + ut_a(node->size * UNIV_PAGE_SIZE <= trunc_len); + + trunc_len -= node->size * UNIV_PAGE_SIZE; + + fil_node_free(node, fil_system, space); + } + + mutex_exit(&fil_system->mutex); +} + +/****************************************************************//** +Check is there node in file space with given name. */ +UNIV_INTERN +ibool +fil_space_contains_node( +/*====================*/ + ulint id, /*!< in: space id */ + char* node_name) /*!< in: node name */ +{ + fil_node_t* node; + fil_space_t* space; + + mutex_enter(&fil_system->mutex); + + space = fil_space_get_by_id(id); + + ut_a(space); + + for (node = UT_LIST_GET_FIRST(space->chain); node != NULL; + node = UT_LIST_GET_NEXT(chain, node)) { + + if (ut_strcmp(node->name, node_name) == 0) { + mutex_exit(&fil_system->mutex); + return(TRUE); + } + + } + + mutex_exit(&fil_system->mutex); + return(FALSE); +} + +#endif /* UNIV_LOG_ARCHIVE */ + +/*******************************************************************//** +Creates a space memory object and puts it to the 'fil system' hash table. +If there is an error, prints an error message to the .err log. +@return TRUE if success */ +UNIV_INTERN +ibool +fil_space_create( +/*=============*/ + const char* name, /*!< in: space name */ + ulint id, /*!< in: space id */ + ulint flags, /*!< in: tablespace flags */ + ulint purpose)/*!< in: FIL_TABLESPACE, or FIL_LOG if log */ +{ + fil_space_t* space; + + DBUG_EXECUTE_IF("fil_space_create_failure", return(false);); + + ut_a(fil_system); + + /* Look for a matching tablespace and if found free it. */ + do { + mutex_enter(&fil_system->mutex); + + space = fil_space_get_by_name(name); + + if (space != 0) { + ib_logf(IB_LOG_LEVEL_WARN, + "Tablespace '%s' exists in the cache " + "with id %lu != %lu", + name, (ulong) space->id, (ulong) id); + + if (id == 0 || purpose != FIL_TABLESPACE) { + + mutex_exit(&fil_system->mutex); + + return(FALSE); + } + + ib_logf(IB_LOG_LEVEL_WARN, + "Freeing existing tablespace '%s' entry " + "from the cache with id %lu", + name, (ulong) id); + + ibool success = fil_space_free(space->id, FALSE); + ut_a(success); + + mutex_exit(&fil_system->mutex); + } + + } while (space != 0); + + space = fil_space_get_by_id(id); + + if (space != 0) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Trying to add tablespace '%s' with id %lu " + "to the tablespace memory cache, but tablespace '%s' " + "with id %lu already exists in the cache!", + name, (ulong) id, space->name, (ulong) space->id); + + mutex_exit(&fil_system->mutex); + + return(FALSE); + } + + space = static_cast(mem_zalloc(sizeof(*space))); + + space->name = mem_strdup(name); + space->id = id; + + fil_system->tablespace_version++; + space->tablespace_version = fil_system->tablespace_version; + space->mark = FALSE; + + if (purpose == FIL_TABLESPACE && !recv_recovery_on + && id > fil_system->max_assigned_id) { + + if (!fil_system->space_id_reuse_warned) { + fil_system->space_id_reuse_warned = TRUE; + + ib_logf(IB_LOG_LEVEL_WARN, + "Allocated tablespace %lu, old maximum " + "was %lu", + (ulong) id, + (ulong) fil_system->max_assigned_id); + } + + fil_system->max_assigned_id = id; + } + + space->purpose = purpose; + space->flags = flags; + + space->magic_n = FIL_SPACE_MAGIC_N; + + rw_lock_create(fil_space_latch_key, &space->latch, SYNC_FSP); + + HASH_INSERT(fil_space_t, hash, fil_system->spaces, id, space); + + HASH_INSERT(fil_space_t, name_hash, fil_system->name_hash, + ut_fold_string(name), space); + space->is_in_unflushed_spaces = false; + + space->is_corrupt = FALSE; + + UT_LIST_ADD_LAST(space_list, fil_system->space_list, space); + + mutex_exit(&fil_system->mutex); + + return(TRUE); +} + +/*******************************************************************//** +Assigns a new space id for a new single-table tablespace. This works simply by +incrementing the global counter. If 4 billion id's is not enough, we may need +to recycle id's. +@return TRUE if assigned, FALSE if not */ +UNIV_INTERN +ibool +fil_assign_new_space_id( +/*====================*/ + ulint* space_id) /*!< in/out: space id */ +{ + ulint id; + ibool success; + + mutex_enter(&fil_system->mutex); + + id = *space_id; + + if (id < fil_system->max_assigned_id) { + id = fil_system->max_assigned_id; + } + + id++; + + if (id > (SRV_LOG_SPACE_FIRST_ID / 2) && (id % 1000000UL == 0)) { + ut_print_timestamp(stderr); + fprintf(stderr, + "InnoDB: Warning: you are running out of new" + " single-table tablespace id's.\n" + "InnoDB: Current counter is %lu and it" + " must not exceed %lu!\n" + "InnoDB: To reset the counter to zero" + " you have to dump all your tables and\n" + "InnoDB: recreate the whole InnoDB installation.\n", + (ulong) id, + (ulong) SRV_LOG_SPACE_FIRST_ID); + } + + success = (id < SRV_LOG_SPACE_FIRST_ID); + + if (success) { + *space_id = fil_system->max_assigned_id = id; + } else { + ut_print_timestamp(stderr); + fprintf(stderr, + "InnoDB: You have run out of single-table" + " tablespace id's!\n" + "InnoDB: Current counter is %lu.\n" + "InnoDB: To reset the counter to zero you" + " have to dump all your tables and\n" + "InnoDB: recreate the whole InnoDB installation.\n", + (ulong) id); + *space_id = ULINT_UNDEFINED; + } + + mutex_exit(&fil_system->mutex); + + return(success); +} + +/*******************************************************************//** +Frees a space object from the tablespace memory cache. Closes the files in +the chain but does not delete them. There must not be any pending i/o's or +flushes on the files. +@return TRUE if success */ +static +ibool +fil_space_free( +/*===========*/ + /* out: TRUE if success */ + ulint id, /* in: space id */ + ibool x_latched) /* in: TRUE if caller has space->latch + in X mode */ +{ + fil_space_t* space; + fil_space_t* fnamespace; + + ut_ad(mutex_own(&fil_system->mutex)); + + space = fil_space_get_by_id(id); + + if (!space) { + ut_print_timestamp(stderr); + fprintf(stderr, + " InnoDB: Error: trying to remove tablespace %lu" + " from the cache but\n" + "InnoDB: it is not there.\n", (ulong) id); + + return(FALSE); + } + + HASH_DELETE(fil_space_t, hash, fil_system->spaces, id, space); + + fnamespace = fil_space_get_by_name(space->name); + ut_a(fnamespace); + ut_a(space == fnamespace); + + HASH_DELETE(fil_space_t, name_hash, fil_system->name_hash, + ut_fold_string(space->name), space); + + if (space->is_in_unflushed_spaces) { + + ut_ad(!fil_buffering_disabled(space)); + space->is_in_unflushed_spaces = false; + + UT_LIST_REMOVE(unflushed_spaces, fil_system->unflushed_spaces, + space); + } + + UT_LIST_REMOVE(space_list, fil_system->space_list, space); + + ut_a(space->magic_n == FIL_SPACE_MAGIC_N); + ut_a(0 == space->n_pending_flushes); + + for (fil_node_t* fil_node = UT_LIST_GET_FIRST(space->chain); + fil_node != NULL; + fil_node = UT_LIST_GET_FIRST(space->chain)) { + + fil_node_free(fil_node, fil_system, space); + } + + ut_a(0 == UT_LIST_GET_LEN(space->chain)); + + if (x_latched) { + rw_lock_x_unlock(&space->latch); + } + + rw_lock_free(&(space->latch)); + + mem_free(space->name); + mem_free(space); + + return(TRUE); +} + +/*******************************************************************//** +Returns a pointer to the file_space_t that is in the memory cache +associated with a space id. The caller must lock fil_system->mutex. +@return file_space_t pointer, NULL if space not found */ +UNIV_INLINE +fil_space_t* +fil_space_get_space( +/*================*/ + ulint id) /*!< in: space id */ +{ + fil_space_t* space; + fil_node_t* node; + + ut_ad(fil_system); + + space = fil_space_get_by_id(id); + if (space == NULL) { + return(NULL); + } + + if (space->size == 0 && space->purpose == FIL_TABLESPACE) { + ut_a(id != 0); + + mutex_exit(&fil_system->mutex); + + /* It is possible that the space gets evicted at this point + before the fil_mutex_enter_and_prepare_for_io() acquires + the fil_system->mutex. Check for this after completing the + call to fil_mutex_enter_and_prepare_for_io(). */ + fil_mutex_enter_and_prepare_for_io(id); + + /* We are still holding the fil_system->mutex. Check if + the space is still in memory cache. */ + space = fil_space_get_by_id(id); + if (space == NULL) { + return(NULL); + } + + /* The following code must change when InnoDB supports + multiple datafiles per tablespace. */ + ut_a(1 == UT_LIST_GET_LEN(space->chain)); + + node = UT_LIST_GET_FIRST(space->chain); + + /* It must be a single-table tablespace and we have not opened + the file yet; the following calls will open it and update the + size fields */ + + if (!fil_node_prepare_for_io(node, fil_system, space)) { + /* The single-table tablespace can't be opened, + because the ibd file is missing. */ + return(NULL); + } + fil_node_complete_io(node, fil_system, OS_FILE_READ); + } + + return(space); +} + +/*******************************************************************//** +Returns the path from the first fil_node_t found for the space ID sent. +The caller is responsible for freeing the memory allocated here for the +value returned. +@return own: A copy of fil_node_t::path, NULL if space ID is zero +or not found. */ +UNIV_INTERN +char* +fil_space_get_first_path( +/*=====================*/ + ulint id) /*!< in: space id */ +{ + fil_space_t* space; + fil_node_t* node; + char* path; + + ut_ad(fil_system); + ut_a(id); + + fil_mutex_enter_and_prepare_for_io(id); + + space = fil_space_get_space(id); + + if (space == NULL) { + mutex_exit(&fil_system->mutex); + + return(NULL); + } + + ut_ad(mutex_own(&fil_system->mutex)); + + node = UT_LIST_GET_FIRST(space->chain); + + path = mem_strdup(node->name); + + mutex_exit(&fil_system->mutex); + + return(path); +} + +/*******************************************************************//** +Returns the size of the space in pages. The tablespace must be cached in the +memory cache. +@return space size, 0 if space not found */ +UNIV_INTERN +ulint +fil_space_get_size( +/*===============*/ + ulint id) /*!< in: space id */ +{ + fil_space_t* space; + ulint size; + + ut_ad(fil_system); + mutex_enter(&fil_system->mutex); + + space = fil_space_get_space(id); + + size = space ? space->size : 0; + + mutex_exit(&fil_system->mutex); + + return(size); +} + +/*******************************************************************//** +Returns the flags of the space. The tablespace must be cached +in the memory cache. +@return flags, ULINT_UNDEFINED if space not found */ +UNIV_INTERN +ulint +fil_space_get_flags( +/*================*/ + ulint id) /*!< in: space id */ +{ + fil_space_t* space; + ulint flags; + + ut_ad(fil_system); + + if (!id) { + return(0); + } + + mutex_enter(&fil_system->mutex); + + space = fil_space_get_space(id); + + if (space == NULL) { + mutex_exit(&fil_system->mutex); + + return(ULINT_UNDEFINED); + } + + flags = space->flags; + + mutex_exit(&fil_system->mutex); + + return(flags); +} + +/*******************************************************************//** +Returns the compressed page size of the space, or 0 if the space +is not compressed. The tablespace must be cached in the memory cache. +@return compressed page size, ULINT_UNDEFINED if space not found */ +UNIV_INTERN +ulint +fil_space_get_zip_size( +/*===================*/ + ulint id) /*!< in: space id */ +{ + ulint flags; + + flags = fil_space_get_flags(id); + + if (flags && flags != ULINT_UNDEFINED) { + + return(fsp_flags_get_zip_size(flags)); + } + + return(flags); +} + +/*******************************************************************//** +Checks if the pair space, page_no refers to an existing page in a tablespace +file space. The tablespace must be cached in the memory cache. +@return TRUE if the address is meaningful */ +UNIV_INTERN +ibool +fil_check_adress_in_tablespace( +/*===========================*/ + ulint id, /*!< in: space id */ + ulint page_no)/*!< in: page number */ +{ + if (fil_space_get_size(id) > page_no) { + + return(TRUE); + } + + return(FALSE); +} + +/****************************************************************//** +Initializes the tablespace memory cache. */ +UNIV_INTERN +void +fil_init( +/*=====*/ + ulint hash_size, /*!< in: hash table size */ + ulint max_n_open) /*!< in: max number of open files */ +{ + ut_a(fil_system == NULL); + + ut_a(hash_size > 0); + ut_a(max_n_open > 0); + + fil_system = static_cast( + mem_zalloc(sizeof(fil_system_t))); + + mutex_create(fil_system_mutex_key, + &fil_system->mutex, SYNC_ANY_LATCH); + + fil_system->spaces = hash_create(hash_size); + fil_system->name_hash = hash_create(hash_size); + + UT_LIST_INIT(fil_system->LRU); + + fil_system->max_n_open = max_n_open; +} + +/*******************************************************************//** +Opens all log files and system tablespace data files. They stay open until the +database server shutdown. This should be called at a server startup after the +space objects for the log and the system tablespace have been created. The +purpose of this operation is to make sure we never run out of file descriptors +if we need to read from the insert buffer or to write to the log. */ +UNIV_INTERN +void +fil_open_log_and_system_tablespace_files(void) +/*==========================================*/ +{ + fil_space_t* space; + + mutex_enter(&fil_system->mutex); + + for (space = UT_LIST_GET_FIRST(fil_system->space_list); + space != NULL; + space = UT_LIST_GET_NEXT(space_list, space)) { + + fil_node_t* node; + + if (fil_space_belongs_in_lru(space)) { + + continue; + } + + for (node = UT_LIST_GET_FIRST(space->chain); + node != NULL; + node = UT_LIST_GET_NEXT(chain, node)) { + + if (!node->open) { + if (!fil_node_open_file(node, fil_system, + space)) { + /* This func is called during server's + startup. If some file of log or system + tablespace is missing, the server + can't start successfully. So we should + assert for it. */ + ut_a(0); + } + } + + if (fil_system->max_n_open < 10 + fil_system->n_open) { + + fprintf(stderr, + "InnoDB: Warning: you must" + " raise the value of" + " innodb_open_files in\n" + "InnoDB: my.cnf! Remember that" + " InnoDB keeps all log files" + " and all system\n" + "InnoDB: tablespace files open" + " for the whole time mysqld is" + " running, and\n" + "InnoDB: needs to open also" + " some .ibd files if the" + " file-per-table storage\n" + "InnoDB: model is used." + " Current open files %lu," + " max allowed" + " open files %lu.\n", + (ulong) fil_system->n_open, + (ulong) fil_system->max_n_open); + } + } + } + + mutex_exit(&fil_system->mutex); +} + +/*******************************************************************//** +Closes all open files. There must not be any pending i/o's or not flushed +modifications in the files. */ +UNIV_INTERN +void +fil_close_all_files(void) +/*=====================*/ +{ + fil_space_t* space; + + mutex_enter(&fil_system->mutex); + + space = UT_LIST_GET_FIRST(fil_system->space_list); + + while (space != NULL) { + fil_node_t* node; + fil_space_t* prev_space = space; + + for (node = UT_LIST_GET_FIRST(space->chain); + node != NULL; + node = UT_LIST_GET_NEXT(chain, node)) { + + if (node->open) { + fil_node_close_file(node, fil_system); + } + } + + space = UT_LIST_GET_NEXT(space_list, space); + + fil_space_free(prev_space->id, FALSE); + } + + mutex_exit(&fil_system->mutex); +} + +/*******************************************************************//** +Closes the redo log files. There must not be any pending i/o's or not +flushed modifications in the files. */ +UNIV_INTERN +void +fil_close_log_files( +/*================*/ + bool free) /*!< in: whether to free the memory object */ +{ + fil_space_t* space; + + mutex_enter(&fil_system->mutex); + + space = UT_LIST_GET_FIRST(fil_system->space_list); + + while (space != NULL) { + fil_node_t* node; + fil_space_t* prev_space = space; + + if (space->purpose != FIL_LOG) { + space = UT_LIST_GET_NEXT(space_list, space); + continue; + } + + for (node = UT_LIST_GET_FIRST(space->chain); + node != NULL; + node = UT_LIST_GET_NEXT(chain, node)) { + + if (node->open) { + fil_node_close_file(node, fil_system); + } + } + + space = UT_LIST_GET_NEXT(space_list, space); + + if (free) { + fil_space_free(prev_space->id, FALSE); + } + } + + mutex_exit(&fil_system->mutex); +} + +/*******************************************************************//** +Sets the max tablespace id counter if the given number is bigger than the +previous value. */ +UNIV_INTERN +void +fil_set_max_space_id_if_bigger( +/*===========================*/ + ulint max_id) /*!< in: maximum known id */ +{ + if (max_id >= SRV_LOG_SPACE_FIRST_ID) { + fprintf(stderr, + "InnoDB: Fatal error: max tablespace id" + " is too high, %lu\n", (ulong) max_id); + ut_error; + } + + mutex_enter(&fil_system->mutex); + + if (fil_system->max_assigned_id < max_id) { + + fil_system->max_assigned_id = max_id; + } + + mutex_exit(&fil_system->mutex); +} + +/****************************************************************//** +Writes the flushed lsn and the latest archived log number to the page header +of the first page of a data file of the system tablespace (space 0), +which is uncompressed. */ +static __attribute__((warn_unused_result)) +dberr_t +fil_write_lsn_and_arch_no_to_file( +/*==============================*/ + ulint space, /*!< in: space to write to */ + ulint sum_of_sizes, /*!< in: combined size of previous files + in space, in database pages */ + lsn_t lsn, /*!< in: lsn to write */ + ulint arch_log_no __attribute__((unused))) + /*!< in: archived log number to write */ +{ + byte* buf1; + byte* buf; + dberr_t err; + + buf1 = static_cast(mem_alloc(2 * UNIV_PAGE_SIZE)); + buf = static_cast(ut_align(buf1, UNIV_PAGE_SIZE)); + + err = fil_read(TRUE, space, 0, sum_of_sizes, 0, + UNIV_PAGE_SIZE, buf, NULL, 0); + if (err == DB_SUCCESS) { + mach_write_to_8(buf + FIL_PAGE_FILE_FLUSH_LSN, lsn); + + err = fil_write(TRUE, space, 0, sum_of_sizes, 0, + UNIV_PAGE_SIZE, buf, NULL, 0); + } + + mem_free(buf1); + + return(err); +} + +/****************************************************************//** +Writes the flushed lsn and the latest archived log number to the page +header of the first page of each data file in the system tablespace. +@return DB_SUCCESS or error number */ +UNIV_INTERN +dberr_t +fil_write_flushed_lsn_to_data_files( +/*================================*/ + lsn_t lsn, /*!< in: lsn to write */ + ulint arch_log_no) /*!< in: latest archived log file number */ +{ + fil_space_t* space; + fil_node_t* node; + dberr_t err; + + mutex_enter(&fil_system->mutex); + + for (space = UT_LIST_GET_FIRST(fil_system->space_list); + space != NULL; + space = UT_LIST_GET_NEXT(space_list, space)) { + + /* We only write the lsn to all existing data files which have + been open during the lifetime of the mysqld process; they are + represented by the space objects in the tablespace memory + cache. Note that all data files in the system tablespace 0 + and the UNDO log tablespaces (if separate) are always open. */ + + if (space->purpose == FIL_TABLESPACE + && !fil_is_user_tablespace_id(space->id)) { + ulint sum_of_sizes = 0; + + for (node = UT_LIST_GET_FIRST(space->chain); + node != NULL; + node = UT_LIST_GET_NEXT(chain, node)) { + + mutex_exit(&fil_system->mutex); + + err = fil_write_lsn_and_arch_no_to_file( + space->id, sum_of_sizes, lsn, + arch_log_no); + + if (err != DB_SUCCESS) { + + return(err); + } + + mutex_enter(&fil_system->mutex); + + sum_of_sizes += node->size; + } + } + } + + mutex_exit(&fil_system->mutex); + + return(DB_SUCCESS); +} + +/*******************************************************************//** +Checks the consistency of the first data page of a tablespace +at database startup. +@retval NULL on success, or if innodb_force_recovery is set +@return pointer to an error message string */ +static __attribute__((warn_unused_result)) +const char* +fil_check_first_page( +/*=================*/ + const page_t* page) /*!< in: data page */ +{ + ulint space_id; + ulint flags; + + if (srv_force_recovery >= SRV_FORCE_IGNORE_CORRUPT) { + return(NULL); + } + + space_id = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_ID + page); + flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); + + if (UNIV_PAGE_SIZE != fsp_flags_get_page_size(flags)) { + fprintf(stderr, "InnoDB: Error: Current page size %lu != page size on page %lu\n", + UNIV_PAGE_SIZE, fsp_flags_get_page_size(flags)); + + return("innodb-page-size mismatch"); + } + + if (!space_id && !flags) { + ulint nonzero_bytes = UNIV_PAGE_SIZE; + const byte* b = page; + + while (!*b && --nonzero_bytes) { + b++; + } + + if (!nonzero_bytes) { + return("space header page consists of zero bytes"); + } + } + + if (buf_page_is_corrupted( + false, page, fsp_flags_get_zip_size(flags))) { + return("checksum mismatch"); + } + + if (page_get_space_id(page) == space_id + && page_get_page_no(page) == 0) { + return(NULL); + } + + return("inconsistent data in space header"); +} + +/*******************************************************************//** +Reads the flushed lsn, arch no, and tablespace flag fields from a data +file at database startup. +@retval NULL on success, or if innodb_force_recovery is set +@return pointer to an error message string */ +UNIV_INTERN +const char* +fil_read_first_page( +/*================*/ + os_file_t data_file, /*!< in: open data file */ + ibool one_read_already, /*!< in: TRUE if min and max + parameters below already + contain sensible data */ + ulint* flags, /*!< out: tablespace flags */ + ulint* space_id, /*!< out: tablespace ID */ + lsn_t* min_flushed_lsn, /*!< out: min of flushed + lsn values in data files */ + lsn_t* max_flushed_lsn, /*!< out: max of flushed + lsn values in data files */ + ulint orig_space_id) /*!< in: original file space + id */ +{ + byte* buf; + byte* page; + lsn_t flushed_lsn; + const char* check_msg = NULL; + + buf = static_cast(ut_malloc(2 * UNIV_PAGE_SIZE)); + + /* Align the memory for a possible read from a raw device */ + + page = static_cast(ut_align(buf, UNIV_PAGE_SIZE)); + + os_file_read(data_file, page, 0, UNIV_PAGE_SIZE, + orig_space_id != ULINT_UNDEFINED ? + fil_space_is_page_compressed(orig_space_id) : + FALSE); + + *flags = fsp_header_get_flags(page); + + /* Page is page compressed page, need to decompress, before + continue. */ + if (fsp_flags_is_page_compressed(*flags)) { + ulint write_size=0; + fil_decompress_page(NULL, page, UNIV_PAGE_SIZE, &write_size); + } + + *space_id = fsp_header_get_space_id(page); + + flushed_lsn = mach_read_from_8(page + FIL_PAGE_FILE_FLUSH_LSN); + + if (!one_read_already) { + check_msg = fil_check_first_page(page); + } + + ut_free(buf); + + if (check_msg) { + return(check_msg); + } + + if (!one_read_already) { + *min_flushed_lsn = flushed_lsn; + *max_flushed_lsn = flushed_lsn; + + return(NULL); + } + + if (*min_flushed_lsn > flushed_lsn) { + *min_flushed_lsn = flushed_lsn; + } + if (*max_flushed_lsn < flushed_lsn) { + *max_flushed_lsn = flushed_lsn; + } + + return(NULL); +} + +/*================ SINGLE-TABLE TABLESPACES ==========================*/ + +#ifndef UNIV_HOTBACKUP +/*******************************************************************//** +Increments the count of pending operation, if space is not being deleted. +@return TRUE if being deleted, and operation should be skipped */ +UNIV_INTERN +ibool +fil_inc_pending_ops( +/*================*/ + ulint id) /*!< in: space id */ +{ + fil_space_t* space; + + mutex_enter(&fil_system->mutex); + + space = fil_space_get_by_id(id); + + if (space == NULL) { + fprintf(stderr, + "InnoDB: Error: trying to do an operation on a" + " dropped tablespace %lu\n", + (ulong) id); + } + + if (space == NULL || space->stop_new_ops) { + mutex_exit(&fil_system->mutex); + + return(TRUE); + } + + space->n_pending_ops++; + + mutex_exit(&fil_system->mutex); + + return(FALSE); +} + +/*******************************************************************//** +Decrements the count of pending operations. */ +UNIV_INTERN +void +fil_decr_pending_ops( +/*=================*/ + ulint id) /*!< in: space id */ +{ + fil_space_t* space; + + mutex_enter(&fil_system->mutex); + + space = fil_space_get_by_id(id); + + if (space == NULL) { + fprintf(stderr, + "InnoDB: Error: decrementing pending operation" + " of a dropped tablespace %lu\n", + (ulong) id); + } + + if (space != NULL) { + space->n_pending_ops--; + } + + mutex_exit(&fil_system->mutex); +} +#endif /* !UNIV_HOTBACKUP */ + +/********************************************************//** +Creates the database directory for a table if it does not exist yet. */ +static +void +fil_create_directory_for_tablename( +/*===============================*/ + const char* name) /*!< in: name in the standard + 'databasename/tablename' format */ +{ + const char* namend; + char* path; + ulint len; + + len = strlen(fil_path_to_mysql_datadir); + namend = strchr(name, '/'); + ut_a(namend); + path = static_cast(mem_alloc(len + (namend - name) + 2)); + + memcpy(path, fil_path_to_mysql_datadir, len); + path[len] = '/'; + memcpy(path + len + 1, name, namend - name); + path[len + (namend - name) + 1] = 0; + + srv_normalize_path_for_win(path); + + ut_a(os_file_create_directory(path, FALSE)); + mem_free(path); +} + +#ifndef UNIV_HOTBACKUP +/********************************************************//** +Writes a log record about an .ibd file create/rename/delete. */ +static +void +fil_op_write_log( +/*=============*/ + ulint type, /*!< in: MLOG_FILE_CREATE, + MLOG_FILE_CREATE2, + MLOG_FILE_DELETE, or + MLOG_FILE_RENAME */ + ulint space_id, /*!< in: space id */ + ulint log_flags, /*!< in: redo log flags (stored + in the page number field) */ + ulint flags, /*!< in: compressed page size + and file format + if type==MLOG_FILE_CREATE2, or 0 */ + const char* name, /*!< in: table name in the familiar + 'databasename/tablename' format, or + the file path in the case of + MLOG_FILE_DELETE */ + const char* new_name, /*!< in: if type is MLOG_FILE_RENAME, + the new table name in the + 'databasename/tablename' format */ + mtr_t* mtr) /*!< in: mini-transaction handle */ +{ + byte* log_ptr; + ulint len; + + log_ptr = mlog_open(mtr, 11 + 2 + 1); + + if (!log_ptr) { + /* Logging in mtr is switched off during crash recovery: + in that case mlog_open returns NULL */ + return; + } + + log_ptr = mlog_write_initial_log_record_for_file_op( + type, space_id, log_flags, log_ptr, mtr); + if (type == MLOG_FILE_CREATE2) { + mach_write_to_4(log_ptr, flags); + log_ptr += 4; + } + /* Let us store the strings as null-terminated for easier readability + and handling */ + + len = strlen(name) + 1; + + mach_write_to_2(log_ptr, len); + log_ptr += 2; + mlog_close(mtr, log_ptr); + + mlog_catenate_string(mtr, (byte*) name, len); + + if (type == MLOG_FILE_RENAME) { + len = strlen(new_name) + 1; + log_ptr = mlog_open(mtr, 2 + len); + ut_a(log_ptr); + mach_write_to_2(log_ptr, len); + log_ptr += 2; + mlog_close(mtr, log_ptr); + + mlog_catenate_string(mtr, (byte*) new_name, len); + } +} +#endif + +/*******************************************************************//** +Parses the body of a log record written about an .ibd file operation. That is, +the log record part after the standard (type, space id, page no) header of the +log record. + +If desired, also replays the delete or rename operation if the .ibd file +exists and the space id in it matches. Replays the create operation if a file +at that path does not exist yet. If the database directory for the file to be +created does not exist, then we create the directory, too. + +Note that ibbackup --apply-log sets fil_path_to_mysql_datadir to point to the +datadir that we should use in replaying the file operations. + +InnoDB recovery does not replay these fully since it always sets the space id +to zero. But ibbackup does replay them. TODO: If remote tablespaces are used, +ibbackup will only create tables in the default directory since MLOG_FILE_CREATE +and MLOG_FILE_CREATE2 only know the tablename, not the path. + +@return end of log record, or NULL if the record was not completely +contained between ptr and end_ptr */ +UNIV_INTERN +byte* +fil_op_log_parse_or_replay( +/*=======================*/ + byte* ptr, /*!< in: buffer containing the log record body, + or an initial segment of it, if the record does + not fir completely between ptr and end_ptr */ + byte* end_ptr, /*!< in: buffer end */ + ulint type, /*!< in: the type of this log record */ + ulint space_id, /*!< in: the space id of the tablespace in + question, or 0 if the log record should + only be parsed but not replayed */ + ulint log_flags) /*!< in: redo log flags + (stored in the page number parameter) */ +{ + ulint name_len; + ulint new_name_len; + const char* name; + const char* new_name = NULL; + ulint flags = 0; + + if (type == MLOG_FILE_CREATE2) { + if (end_ptr < ptr + 4) { + + return(NULL); + } + + flags = mach_read_from_4(ptr); + ptr += 4; + } + + if (end_ptr < ptr + 2) { + + return(NULL); + } + + name_len = mach_read_from_2(ptr); + + ptr += 2; + + if (end_ptr < ptr + name_len) { + + return(NULL); + } + + name = (const char*) ptr; + + ptr += name_len; + + if (type == MLOG_FILE_RENAME) { + if (end_ptr < ptr + 2) { + + return(NULL); + } + + new_name_len = mach_read_from_2(ptr); + + ptr += 2; + + if (end_ptr < ptr + new_name_len) { + + return(NULL); + } + + new_name = (const char*) ptr; + + ptr += new_name_len; + } + + /* We managed to parse a full log record body */ + /* + printf("Parsed log rec of type %lu space %lu\n" + "name %s\n", type, space_id, name); + + if (type == MLOG_FILE_RENAME) { + printf("new name %s\n", new_name); + } + */ + if (!space_id) { + return(ptr); + } else { + /* Only replay file ops during recovery. This is a + release-build assert to minimize any data loss risk by a + misapplied file operation. */ + ut_a(recv_recovery_is_on()); + } + + /* Let us try to perform the file operation, if sensible. Note that + ibbackup has at this stage already read in all space id info to the + fil0fil.cc data structures. + + NOTE that our algorithm is not guaranteed to work correctly if there + were renames of tables during the backup. See ibbackup code for more + on the problem. */ + + switch (type) { + case MLOG_FILE_DELETE: + if (fil_tablespace_exists_in_mem(space_id)) { + dberr_t err = fil_delete_tablespace( + space_id, BUF_REMOVE_FLUSH_NO_WRITE); + ut_a(err == DB_SUCCESS); + } + + break; + + case MLOG_FILE_RENAME: + /* In order to replay the rename, the following must hold: + * The new name is not already used. + * A tablespace is open in memory with the old name. + * The space ID for that tablepace matches this log entry. + This will prevent unintended renames during recovery. */ + + if (fil_get_space_id_for_table(new_name) == ULINT_UNDEFINED + && space_id == fil_get_space_id_for_table(name)) { + /* Create the database directory for the new name, if + it does not exist yet */ + fil_create_directory_for_tablename(new_name); + + if (!fil_rename_tablespace(name, space_id, + new_name, NULL)) { + ut_error; + } + } + + break; + + case MLOG_FILE_CREATE: + case MLOG_FILE_CREATE2: + if (fil_tablespace_exists_in_mem(space_id)) { + /* Do nothing */ + } else if (fil_get_space_id_for_table(name) + != ULINT_UNDEFINED) { + /* Do nothing */ + } else if (log_flags & MLOG_FILE_FLAG_TEMP) { + /* Temporary table, do nothing */ + } else { + const char* path = NULL; + + /* Create the database directory for name, if it does + not exist yet */ + fil_create_directory_for_tablename(name); + + if (fil_create_new_single_table_tablespace( + space_id, name, path, flags, + DICT_TF2_USE_TABLESPACE, + FIL_IBD_FILE_INITIAL_SIZE) != DB_SUCCESS) { + ut_error; + } + } + + break; + + default: + ut_error; + } + + return(ptr); +} + +/*******************************************************************//** +Allocates a file name for the EXPORT/IMPORT config file name. The +string must be freed by caller with mem_free(). +@return own: file name */ +static +char* +fil_make_cfg_name( +/*==============*/ + const char* filepath) /*!< in: .ibd file name */ +{ + char* cfg_name; + + /* Create a temporary file path by replacing the .ibd suffix + with .cfg. */ + + ut_ad(strlen(filepath) > 4); + + cfg_name = mem_strdup(filepath); + ut_snprintf(cfg_name + strlen(cfg_name) - 3, 4, "cfg"); + return(cfg_name); +} + +/*******************************************************************//** +Check for change buffer merges. +@return 0 if no merges else count + 1. */ +static +ulint +fil_ibuf_check_pending_ops( +/*=======================*/ + fil_space_t* space, /*!< in/out: Tablespace to check */ + ulint count) /*!< in: number of attempts so far */ +{ + ut_ad(mutex_own(&fil_system->mutex)); + + if (space != 0 && space->n_pending_ops != 0) { + + if (count > 5000) { + ib_logf(IB_LOG_LEVEL_WARN, + "Trying to close/delete tablespace " + "'%s' but there are %lu pending change " + "buffer merges on it.", + space->name, + (ulong) space->n_pending_ops); + } + + return(count + 1); + } + + return(0); +} + +/*******************************************************************//** +Check for pending IO. +@return 0 if no pending else count + 1. */ +static +ulint +fil_check_pending_io( +/*=================*/ + fil_space_t* space, /*!< in/out: Tablespace to check */ + fil_node_t** node, /*!< out: Node in space list */ + ulint count) /*!< in: number of attempts so far */ +{ + ut_ad(mutex_own(&fil_system->mutex)); + ut_a(space->n_pending_ops == 0); + + /* The following code must change when InnoDB supports + multiple datafiles per tablespace. */ + ut_a(UT_LIST_GET_LEN(space->chain) == 1); + + *node = UT_LIST_GET_FIRST(space->chain); + + if (space->n_pending_flushes > 0 || (*node)->n_pending > 0) { + + ut_a(!(*node)->being_extended); + + if (count > 1000) { + ib_logf(IB_LOG_LEVEL_WARN, + "Trying to close/delete tablespace '%s' " + "but there are %lu flushes " + " and %lu pending i/o's on it.", + space->name, + (ulong) space->n_pending_flushes, + (ulong) (*node)->n_pending); + } + + return(count + 1); + } + + return(0); +} + +/*******************************************************************//** +Check pending operations on a tablespace. +@return DB_SUCCESS or error failure. */ +static +dberr_t +fil_check_pending_operations( +/*=========================*/ + ulint id, /*!< in: space id */ + fil_space_t** space, /*!< out: tablespace instance in memory */ + char** path) /*!< out/own: tablespace path */ +{ + ulint count = 0; + + ut_a(id != TRX_SYS_SPACE); + ut_ad(space); + + *space = 0; + + mutex_enter(&fil_system->mutex); + fil_space_t* sp = fil_space_get_by_id(id); + if (sp) { + sp->stop_new_ops = TRUE; + } + mutex_exit(&fil_system->mutex); + + /* Check for pending change buffer merges. */ + + do { + mutex_enter(&fil_system->mutex); + + sp = fil_space_get_by_id(id); + + count = fil_ibuf_check_pending_ops(sp, count); + + mutex_exit(&fil_system->mutex); + + if (count > 0) { + os_thread_sleep(20000); + } + + } while (count > 0); + + /* Check for pending IO. */ + + *path = 0; + + do { + mutex_enter(&fil_system->mutex); + + sp = fil_space_get_by_id(id); + + if (sp == NULL) { + mutex_exit(&fil_system->mutex); + return(DB_TABLESPACE_NOT_FOUND); + } + + fil_node_t* node; + + count = fil_check_pending_io(sp, &node, count); + + if (count == 0) { + *path = mem_strdup(node->name); + } + + mutex_exit(&fil_system->mutex); + + if (count > 0) { + os_thread_sleep(20000); + } + + } while (count > 0); + + ut_ad(sp); + + *space = sp; + return(DB_SUCCESS); +} + +/*******************************************************************//** +Closes a single-table tablespace. The tablespace must be cached in the +memory cache. Free all pages used by the tablespace. +@return DB_SUCCESS or error */ +UNIV_INTERN +dberr_t +fil_close_tablespace( +/*=================*/ + trx_t* trx, /*!< in/out: Transaction covering the close */ + ulint id) /*!< in: space id */ +{ + char* path = 0; + fil_space_t* space = 0; + + ut_a(id != TRX_SYS_SPACE); + + dberr_t err = fil_check_pending_operations(id, &space, &path); + + if (err != DB_SUCCESS) { + return(err); + } + + ut_a(space); + ut_a(path != 0); + + rw_lock_x_lock(&space->latch); + +#ifndef UNIV_HOTBACKUP + /* Invalidate in the buffer pool all pages belonging to the + tablespace. Since we have set space->stop_new_ops = TRUE, readahead + or ibuf merge can no longer read more pages of this tablespace to the + buffer pool. Thus we can clean the tablespace out of the buffer pool + completely and permanently. The flag stop_new_ops also prevents + fil_flush() from being applied to this tablespace. */ + + buf_LRU_flush_or_remove_pages(id, BUF_REMOVE_FLUSH_WRITE, trx); +#endif + mutex_enter(&fil_system->mutex); + + /* If the free is successful, the X lock will be released before + the space memory data structure is freed. */ + + if (!fil_space_free(id, TRUE)) { + rw_lock_x_unlock(&space->latch); + err = DB_TABLESPACE_NOT_FOUND; + } else { + err = DB_SUCCESS; + } + + mutex_exit(&fil_system->mutex); + + /* If it is a delete then also delete any generated files, otherwise + when we drop the database the remove directory will fail. */ + + char* cfg_name = fil_make_cfg_name(path); + + os_file_delete_if_exists(innodb_file_data_key, cfg_name); + + mem_free(path); + mem_free(cfg_name); + + return(err); +} + +/*******************************************************************//** +Deletes a single-table tablespace. The tablespace must be cached in the +memory cache. +@return DB_SUCCESS or error */ +UNIV_INTERN +dberr_t +fil_delete_tablespace( +/*==================*/ + ulint id, /*!< in: space id */ + buf_remove_t buf_remove) /*!< in: specify the action to take + on the tables pages in the buffer + pool */ +{ + char* path = 0; + fil_space_t* space = 0; + + ut_a(id != TRX_SYS_SPACE); + + dberr_t err = fil_check_pending_operations(id, &space, &path); + + if (err != DB_SUCCESS) { + + ib_logf(IB_LOG_LEVEL_ERROR, + "Cannot delete tablespace %lu because it is not " + "found in the tablespace memory cache.", + (ulong) id); + + return(err); + } + + ut_a(space); + ut_a(path != 0); + + /* Important: We rely on the data dictionary mutex to ensure + that a race is not possible here. It should serialize the tablespace + drop/free. We acquire an X latch only to avoid a race condition + when accessing the tablespace instance via: + + fsp_get_available_space_in_free_extents(). + + There our main motivation is to reduce the contention on the + dictionary mutex. */ + + rw_lock_x_lock(&space->latch); + +#ifndef UNIV_HOTBACKUP + /* IMPORTANT: Because we have set space::stop_new_ops there + can't be any new ibuf merges, reads or flushes. We are here + because node::n_pending was zero above. However, it is still + possible to have pending read and write requests: + + A read request can happen because the reader thread has + gone through the ::stop_new_ops check in buf_page_init_for_read() + before the flag was set and has not yet incremented ::n_pending + when we checked it above. + + A write request can be issued any time because we don't check + the ::stop_new_ops flag when queueing a block for write. + + We deal with pending write requests in the following function + where we'd minimally evict all dirty pages belonging to this + space from the flush_list. Not that if a block is IO-fixed + we'll wait for IO to complete. + + To deal with potential read requests by checking the + ::stop_new_ops flag in fil_io() */ + + buf_LRU_flush_or_remove_pages(id, buf_remove, 0); + +#endif /* !UNIV_HOTBACKUP */ + + /* If it is a delete then also delete any generated files, otherwise + when we drop the database the remove directory will fail. */ + { + char* cfg_name = fil_make_cfg_name(path); + os_file_delete_if_exists(innodb_file_data_key, cfg_name); + mem_free(cfg_name); + } + + /* Delete the link file pointing to the ibd file we are deleting. */ + if (FSP_FLAGS_HAS_DATA_DIR(space->flags)) { + fil_delete_link_file(space->name); + } + + mutex_enter(&fil_system->mutex); + + /* Double check the sanity of pending ops after reacquiring + the fil_system::mutex. */ + if (fil_space_get_by_id(id)) { + ut_a(space->n_pending_ops == 0); + ut_a(UT_LIST_GET_LEN(space->chain) == 1); + fil_node_t* node = UT_LIST_GET_FIRST(space->chain); + ut_a(node->n_pending == 0); + } + + if (!fil_space_free(id, TRUE)) { + err = DB_TABLESPACE_NOT_FOUND; + } + + mutex_exit(&fil_system->mutex); + + if (err != DB_SUCCESS) { + rw_lock_x_unlock(&space->latch); + } else if (!os_file_delete(innodb_file_data_key, path) + && !os_file_delete_if_exists(innodb_file_data_key, path)) { + + /* Note: This is because we have removed the + tablespace instance from the cache. */ + + err = DB_IO_ERROR; + } + + if (err == DB_SUCCESS) { +#ifndef UNIV_HOTBACKUP + /* Write a log record about the deletion of the .ibd + file, so that ibbackup can replay it in the + --apply-log phase. We use a dummy mtr and the familiar + log write mechanism. */ + mtr_t mtr; + + /* When replaying the operation in ibbackup, do not try + to write any log record */ + mtr_start(&mtr); + + fil_op_write_log(MLOG_FILE_DELETE, id, 0, 0, path, NULL, &mtr); + mtr_commit(&mtr); +#endif + err = DB_SUCCESS; + } + + mem_free(path); + + return(err); +} + +/*******************************************************************//** +Returns TRUE if a single-table tablespace is being deleted. +@return TRUE if being deleted */ +UNIV_INTERN +ibool +fil_tablespace_is_being_deleted( +/*============================*/ + ulint id) /*!< in: space id */ +{ + fil_space_t* space; + ibool is_being_deleted; + + mutex_enter(&fil_system->mutex); + + space = fil_space_get_by_id(id); + + ut_a(space != NULL); + + is_being_deleted = space->stop_new_ops; + + mutex_exit(&fil_system->mutex); + + return(is_being_deleted); +} + +#ifndef UNIV_HOTBACKUP +/*******************************************************************//** +Discards a single-table tablespace. The tablespace must be cached in the +memory cache. Discarding is like deleting a tablespace, but + + 1. We do not drop the table from the data dictionary; + + 2. We remove all insert buffer entries for the tablespace immediately; + in DROP TABLE they are only removed gradually in the background; + + 3. Free all the pages in use by the tablespace. +@return DB_SUCCESS or error */ +UNIV_INTERN +dberr_t +fil_discard_tablespace( +/*===================*/ + ulint id) /*!< in: space id */ +{ + dberr_t err; + + switch (err = fil_delete_tablespace(id, BUF_REMOVE_ALL_NO_WRITE)) { + case DB_SUCCESS: + break; + + case DB_IO_ERROR: + ib_logf(IB_LOG_LEVEL_WARN, + "While deleting tablespace %lu in DISCARD TABLESPACE." + " File rename/delete failed: %s", + (ulong) id, ut_strerr(err)); + break; + + case DB_TABLESPACE_NOT_FOUND: + ib_logf(IB_LOG_LEVEL_WARN, + "Cannot delete tablespace %lu in DISCARD " + "TABLESPACE. %s", + (ulong) id, ut_strerr(err)); + break; + + default: + ut_error; + } + + /* Remove all insert buffer entries for the tablespace */ + + ibuf_delete_for_discarded_space(id); + + return(err); +} +#endif /* !UNIV_HOTBACKUP */ + +/*******************************************************************//** +Renames the memory cache structures of a single-table tablespace. +@return TRUE if success */ +static +ibool +fil_rename_tablespace_in_mem( +/*=========================*/ + fil_space_t* space, /*!< in: tablespace memory object */ + fil_node_t* node, /*!< in: file node of that tablespace */ + const char* new_name, /*!< in: new name */ + const char* new_path) /*!< in: new file path */ +{ + fil_space_t* space2; + const char* old_name = space->name; + + ut_ad(mutex_own(&fil_system->mutex)); + + space2 = fil_space_get_by_name(old_name); + if (space != space2) { + fputs("InnoDB: Error: cannot find ", stderr); + ut_print_filename(stderr, old_name); + fputs(" in tablespace memory cache\n", stderr); + + return(FALSE); + } + + space2 = fil_space_get_by_name(new_name); + if (space2 != NULL) { + fputs("InnoDB: Error: ", stderr); + ut_print_filename(stderr, new_name); + fputs(" is already in tablespace memory cache\n", stderr); + + return(FALSE); + } + + HASH_DELETE(fil_space_t, name_hash, fil_system->name_hash, + ut_fold_string(space->name), space); + mem_free(space->name); + mem_free(node->name); + + space->name = mem_strdup(new_name); + node->name = mem_strdup(new_path); + + HASH_INSERT(fil_space_t, name_hash, fil_system->name_hash, + ut_fold_string(new_name), space); + return(TRUE); +} + +/*******************************************************************//** +Allocates a file name for a single-table tablespace. The string must be freed +by caller with mem_free(). +@return own: file name */ +UNIV_INTERN +char* +fil_make_ibd_name( +/*==============*/ + const char* name, /*!< in: table name or a dir path */ + bool is_full_path) /*!< in: TRUE if it is a dir path */ +{ + char* filename; + ulint namelen = strlen(name); + ulint dirlen = strlen(fil_path_to_mysql_datadir); + ulint pathlen = dirlen + namelen + sizeof "/.ibd"; + + filename = static_cast(mem_alloc(pathlen)); + + if (is_full_path) { + memcpy(filename, name, namelen); + memcpy(filename + namelen, ".ibd", sizeof ".ibd"); + } else { + ut_snprintf(filename, pathlen, "%s/%s.ibd", + fil_path_to_mysql_datadir, name); + + } + + srv_normalize_path_for_win(filename); + + return(filename); +} + +/*******************************************************************//** +Allocates a file name for a tablespace ISL file (InnoDB Symbolic Link). +The string must be freed by caller with mem_free(). +@return own: file name */ +UNIV_INTERN +char* +fil_make_isl_name( +/*==============*/ + const char* name) /*!< in: table name */ +{ + char* filename; + ulint namelen = strlen(name); + ulint dirlen = strlen(fil_path_to_mysql_datadir); + ulint pathlen = dirlen + namelen + sizeof "/.isl"; + + filename = static_cast(mem_alloc(pathlen)); + + ut_snprintf(filename, pathlen, "%s/%s.isl", + fil_path_to_mysql_datadir, name); + + srv_normalize_path_for_win(filename); + + return(filename); +} + +/*******************************************************************//** +Renames a single-table tablespace. The tablespace must be cached in the +tablespace memory cache. +@return TRUE if success */ +UNIV_INTERN +ibool +fil_rename_tablespace( +/*==================*/ + const char* old_name_in, /*!< in: old table name in the + standard databasename/tablename + format of InnoDB, or NULL if we + do the rename based on the space + id only */ + ulint id, /*!< in: space id */ + const char* new_name, /*!< in: new table name in the + standard databasename/tablename + format of InnoDB */ + const char* new_path_in) /*!< in: new full datafile path + if the tablespace is remotely + located, or NULL if it is located + in the normal data directory. */ +{ + ibool success; + fil_space_t* space; + fil_node_t* node; + ulint count = 0; + char* new_path; + char* old_name; + char* old_path; + const char* not_given = "(name not specified)"; + + ut_a(id != 0); + +retry: + count++; + + if (!(count % 1000)) { + ut_print_timestamp(stderr); + fputs(" InnoDB: Warning: problems renaming ", stderr); + ut_print_filename(stderr, + old_name_in ? old_name_in : not_given); + fputs(" to ", stderr); + ut_print_filename(stderr, new_name); + fprintf(stderr, ", %lu iterations\n", (ulong) count); + } + + mutex_enter(&fil_system->mutex); + + space = fil_space_get_by_id(id); + + DBUG_EXECUTE_IF("fil_rename_tablespace_failure_1", space = NULL; ); + + if (space == NULL) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Cannot find space id %lu in the tablespace " + "memory cache, though the table '%s' in a " + "rename operation should have that id.", + (ulong) id, old_name_in ? old_name_in : not_given); + mutex_exit(&fil_system->mutex); + + return(FALSE); + } + + if (count > 25000) { + space->stop_ios = FALSE; + mutex_exit(&fil_system->mutex); + + return(FALSE); + } + + /* We temporarily close the .ibd file because we do not trust that + operating systems can rename an open file. For the closing we have to + wait until there are no pending i/o's or flushes on the file. */ + + space->stop_ios = TRUE; + + /* The following code must change when InnoDB supports + multiple datafiles per tablespace. */ + ut_a(UT_LIST_GET_LEN(space->chain) == 1); + node = UT_LIST_GET_FIRST(space->chain); + + if (node->n_pending > 0 + || node->n_pending_flushes > 0 + || node->being_extended) { + /* There are pending i/o's or flushes or the file is + currently being extended, sleep for a while and + retry */ + + mutex_exit(&fil_system->mutex); + + os_thread_sleep(20000); + + goto retry; + + } else if (node->modification_counter > node->flush_counter) { + /* Flush the space */ + + mutex_exit(&fil_system->mutex); + + os_thread_sleep(20000); + + fil_flush(id); + + goto retry; + + } else if (node->open) { + /* Close the file */ + + fil_node_close_file(node, fil_system); + } + + /* Check that the old name in the space is right */ + + if (old_name_in) { + old_name = mem_strdup(old_name_in); + ut_a(strcmp(space->name, old_name) == 0); + } else { + old_name = mem_strdup(space->name); + } + old_path = mem_strdup(node->name); + + /* Rename the tablespace and the node in the memory cache */ + new_path = new_path_in ? mem_strdup(new_path_in) + : fil_make_ibd_name(new_name, false); + + success = fil_rename_tablespace_in_mem( + space, node, new_name, new_path); + + if (success) { + + DBUG_EXECUTE_IF("fil_rename_tablespace_failure_2", + goto skip_second_rename; ); + + success = os_file_rename( + innodb_file_data_key, old_path, new_path); + + DBUG_EXECUTE_IF("fil_rename_tablespace_failure_2", +skip_second_rename: + success = FALSE; ); + + if (!success) { + /* We have to revert the changes we made + to the tablespace memory cache */ + + ut_a(fil_rename_tablespace_in_mem( + space, node, old_name, old_path)); + } + } + + space->stop_ios = FALSE; + + mutex_exit(&fil_system->mutex); + +#ifndef UNIV_HOTBACKUP + if (success && !recv_recovery_on) { + mtr_t mtr; + + mtr_start(&mtr); + + fil_op_write_log(MLOG_FILE_RENAME, id, 0, 0, old_name, new_name, + &mtr); + mtr_commit(&mtr); + } +#endif /* !UNIV_HOTBACKUP */ + + mem_free(new_path); + mem_free(old_path); + mem_free(old_name); + + return(success); +} + +/*******************************************************************//** +Creates a new InnoDB Symbolic Link (ISL) file. It is always created +under the 'datadir' of MySQL. The datadir is the directory of a +running mysqld program. We can refer to it by simply using the path '.'. +@return DB_SUCCESS or error code */ +UNIV_INTERN +dberr_t +fil_create_link_file( +/*=================*/ + const char* tablename, /*!< in: tablename */ + const char* filepath) /*!< in: pathname of tablespace */ +{ + os_file_t file; + ibool success; + dberr_t err = DB_SUCCESS; + char* link_filepath; + char* prev_filepath = fil_read_link_file(tablename); + + ut_ad(!srv_read_only_mode); + + if (prev_filepath) { + /* Truncate will call this with an existing + link file which contains the same filepath. */ + if (0 == strcmp(prev_filepath, filepath)) { + mem_free(prev_filepath); + return(DB_SUCCESS); + } + mem_free(prev_filepath); + } + + link_filepath = fil_make_isl_name(tablename); + + file = os_file_create_simple_no_error_handling( + innodb_file_data_key, link_filepath, + OS_FILE_CREATE, OS_FILE_READ_WRITE, &success, 0); + + if (!success) { + /* The following call will print an error message */ + ulint error = os_file_get_last_error(true); + + ut_print_timestamp(stderr); + fputs(" InnoDB: Cannot create file ", stderr); + ut_print_filename(stderr, link_filepath); + fputs(".\n", stderr); + + if (error == OS_FILE_ALREADY_EXISTS) { + fputs("InnoDB: The link file: ", stderr); + ut_print_filename(stderr, filepath); + fputs(" already exists.\n", stderr); + err = DB_TABLESPACE_EXISTS; + + } else if (error == OS_FILE_DISK_FULL) { + err = DB_OUT_OF_FILE_SPACE; + + } else if (error == OS_FILE_OPERATION_NOT_SUPPORTED) { + err = DB_UNSUPPORTED; + } else { + err = DB_ERROR; + } + + /* file is not open, no need to close it. */ + mem_free(link_filepath); + return(err); + } + + if (!os_file_write(link_filepath, file, filepath, 0, + strlen(filepath))) { + err = DB_ERROR; + } + + /* Close the file, we only need it at startup */ + os_file_close(file); + + mem_free(link_filepath); + + return(err); +} + +/*******************************************************************//** +Deletes an InnoDB Symbolic Link (ISL) file. */ +UNIV_INTERN +void +fil_delete_link_file( +/*=================*/ + const char* tablename) /*!< in: name of table */ +{ + char* link_filepath = fil_make_isl_name(tablename); + + os_file_delete_if_exists(innodb_file_data_key, link_filepath); + + mem_free(link_filepath); +} + +/*******************************************************************//** +Reads an InnoDB Symbolic Link (ISL) file. +It is always created under the 'datadir' of MySQL. The name is of the +form {databasename}/{tablename}. and the isl file is expected to be in a +'{databasename}' directory called '{tablename}.isl'. The caller must free +the memory of the null-terminated path returned if it is not null. +@return own: filepath found in link file, NULL if not found. */ +UNIV_INTERN +char* +fil_read_link_file( +/*===============*/ + const char* name) /*!< in: tablespace name */ +{ + char* filepath = NULL; + char* link_filepath; + FILE* file = NULL; + + /* The .isl file is in the 'normal' tablespace location. */ + link_filepath = fil_make_isl_name(name); + + file = fopen(link_filepath, "r+b"); + + mem_free(link_filepath); + + if (file) { + filepath = static_cast(mem_alloc(OS_FILE_MAX_PATH)); + + os_file_read_string(file, filepath, OS_FILE_MAX_PATH); + fclose(file); + + if (strlen(filepath)) { + /* Trim whitespace from end of filepath */ + ulint lastch = strlen(filepath) - 1; + while (lastch > 4 && filepath[lastch] <= 0x20) { + filepath[lastch--] = 0x00; + } + srv_normalize_path_for_win(filepath); + } + } + + return(filepath); +} + +/*******************************************************************//** +Opens a handle to the file linked to in an InnoDB Symbolic Link file. +@return TRUE if remote linked tablespace file is found and opened. */ +UNIV_INTERN +ibool +fil_open_linked_file( +/*===============*/ + const char* tablename, /*!< in: database/tablename */ + char** remote_filepath,/*!< out: remote filepath */ + os_file_t* remote_file, /*!< out: remote file handle */ + ulint atomic_writes) /*!< in: atomic writes table option + value */ +{ + ibool success; + + *remote_filepath = fil_read_link_file(tablename); + if (*remote_filepath == NULL) { + return(FALSE); + } + + /* The filepath provided is different from what was + found in the link file. */ + *remote_file = os_file_create_simple_no_error_handling( + innodb_file_data_key, *remote_filepath, + OS_FILE_OPEN, OS_FILE_READ_ONLY, + &success, atomic_writes); + + if (!success) { + char* link_filepath = fil_make_isl_name(tablename); + + /* The following call prints an error message */ + os_file_get_last_error(true); + + ib_logf(IB_LOG_LEVEL_ERROR, + "A link file was found named '%s' " + "but the linked tablespace '%s' " + "could not be opened.", + link_filepath, *remote_filepath); + + mem_free(link_filepath); + mem_free(*remote_filepath); + *remote_filepath = NULL; + } + + return(success); +} + +/*******************************************************************//** +Creates a new single-table tablespace to a database directory of MySQL. +Database directories are under the 'datadir' of MySQL. The datadir is the +directory of a running mysqld program. We can refer to it by simply the +path '.'. Tables created with CREATE TEMPORARY TABLE we place in the temp +dir of the mysqld server. + +@return DB_SUCCESS or error code */ +UNIV_INTERN +dberr_t +fil_create_new_single_table_tablespace( +/*===================================*/ + ulint space_id, /*!< in: space id */ + const char* tablename, /*!< in: the table name in the usual + databasename/tablename format + of InnoDB */ + const char* dir_path, /*!< in: NULL or a dir path */ + ulint flags, /*!< in: tablespace flags */ + ulint flags2, /*!< in: table flags2 */ + ulint size) /*!< in: the initial size of the + tablespace file in pages, + must be >= FIL_IBD_FILE_INITIAL_SIZE */ +{ + os_file_t file; + ibool ret; + dberr_t err; + byte* buf2; + byte* page; + char* path; + ibool success; + /* TRUE if a table is created with CREATE TEMPORARY TABLE */ + bool is_temp = !!(flags2 & DICT_TF2_TEMPORARY); + bool has_data_dir = FSP_FLAGS_HAS_DATA_DIR(flags); + ulint atomic_writes = FSP_FLAGS_GET_ATOMIC_WRITES(flags); + + ut_a(space_id > 0); + ut_ad(!srv_read_only_mode); + ut_a(space_id < SRV_LOG_SPACE_FIRST_ID); + ut_a(size >= FIL_IBD_FILE_INITIAL_SIZE); + ut_a(fsp_flags_is_valid(flags)); + + if (is_temp) { + /* Temporary table filepath */ + ut_ad(dir_path); + path = fil_make_ibd_name(dir_path, true); + } else if (has_data_dir) { + ut_ad(dir_path); + path = os_file_make_remote_pathname(dir_path, tablename, "ibd"); + + /* Since this tablespace file will be created in a + remote directory, let's create the subdirectories + in the path, if they are not there already. */ + success = os_file_create_subdirs_if_needed(path); + if (!success) { + err = DB_ERROR; + goto error_exit_3; + } + } else { + path = fil_make_ibd_name(tablename, false); + } + + file = os_file_create( + innodb_file_data_key, path, + OS_FILE_CREATE | OS_FILE_ON_ERROR_NO_EXIT, + OS_FILE_NORMAL, + OS_DATA_FILE, + &ret, + atomic_writes); + + if (ret == FALSE) { + /* The following call will print an error message */ + ulint error = os_file_get_last_error(true); + + ib_logf(IB_LOG_LEVEL_ERROR, + "Cannot create file '%s'\n", path); + + if (error == OS_FILE_ALREADY_EXISTS) { + ib_logf(IB_LOG_LEVEL_ERROR, + "The file '%s' already exists though the " + "corresponding table did not exist " + "in the InnoDB data dictionary. " + "Have you moved InnoDB .ibd files " + "around without using the SQL commands " + "DISCARD TABLESPACE and IMPORT TABLESPACE, " + "or did mysqld crash in the middle of " + "CREATE TABLE? " + "You can resolve the problem by removing " + "the file '%s' under the 'datadir' of MySQL.", + path, path); + + err = DB_TABLESPACE_EXISTS; + goto error_exit_3; + } + + if (error == OS_FILE_OPERATION_NOT_SUPPORTED) { + err = DB_UNSUPPORTED; + goto error_exit_3; + } + + if (error == OS_FILE_DISK_FULL) { + err = DB_OUT_OF_FILE_SPACE; + goto error_exit_3; + } + + err = DB_ERROR; + goto error_exit_3; + } + + ret = os_file_set_size(path, file, size * UNIV_PAGE_SIZE); + + if (!ret) { + err = DB_OUT_OF_FILE_SPACE; + goto error_exit_2; + } + + /* printf("Creating tablespace %s id %lu\n", path, space_id); */ + + /* We have to write the space id to the file immediately and flush the + file to disk. This is because in crash recovery we must be aware what + tablespaces exist and what are their space id's, so that we can apply + the log records to the right file. It may take quite a while until + buffer pool flush algorithms write anything to the file and flush it to + disk. If we would not write here anything, the file would be filled + with zeros from the call of os_file_set_size(), until a buffer pool + flush would write to it. */ + + buf2 = static_cast(ut_malloc(3 * UNIV_PAGE_SIZE)); + /* Align the memory for file i/o if we might have O_DIRECT set */ + page = static_cast(ut_align(buf2, UNIV_PAGE_SIZE)); + + memset(page, '\0', UNIV_PAGE_SIZE); + + /* Add the UNIV_PAGE_SIZE to the table flags and write them to the + tablespace header. */ + flags = fsp_flags_set_page_size(flags, UNIV_PAGE_SIZE); + fsp_header_init_fields(page, space_id, flags); + mach_write_to_4(page + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID, space_id); + ut_ad(fsp_flags_is_valid(flags)); + + if (!(fsp_flags_is_compressed(flags))) { + buf_flush_init_for_writing(page, NULL, 0); + ret = os_file_write(path, file, page, 0, UNIV_PAGE_SIZE); + } else { + page_zip_des_t page_zip; + ulint zip_size; + + zip_size = fsp_flags_get_zip_size(flags); + + page_zip_set_size(&page_zip, zip_size); + page_zip.data = page + UNIV_PAGE_SIZE; +#ifdef UNIV_DEBUG + page_zip.m_start = +#endif /* UNIV_DEBUG */ + page_zip.m_end = page_zip.m_nonempty = + page_zip.n_blobs = 0; + buf_flush_init_for_writing(page, &page_zip, 0); + ret = os_file_write(path, file, page_zip.data, 0, zip_size); + } + + ut_free(buf2); + + if (!ret) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Could not write the first page to tablespace " + "'%s'", path); + + err = DB_ERROR; + goto error_exit_2; + } + + ret = os_file_flush(file); + + if (!ret) { + ib_logf(IB_LOG_LEVEL_ERROR, + "File flush of tablespace '%s' failed", path); + err = DB_ERROR; + goto error_exit_2; + } + + if (has_data_dir) { + /* Now that the IBD file is created, make the ISL file. */ + err = fil_create_link_file(tablename, path); + if (err != DB_SUCCESS) { + goto error_exit_2; + } + } + + success = fil_space_create(tablename, space_id, flags, FIL_TABLESPACE); + if (!success || !fil_node_create(path, size, space_id, FALSE)) { + err = DB_ERROR; + goto error_exit_1; + } + +#ifndef UNIV_HOTBACKUP + { + mtr_t mtr; + ulint mlog_file_flag = 0; + + if (is_temp) { + mlog_file_flag |= MLOG_FILE_FLAG_TEMP; + } + + mtr_start(&mtr); + + fil_op_write_log(flags + ? MLOG_FILE_CREATE2 + : MLOG_FILE_CREATE, + space_id, mlog_file_flag, flags, + tablename, NULL, &mtr); + + mtr_commit(&mtr); + } +#endif + err = DB_SUCCESS; + + /* Error code is set. Cleanup the various variables used. + These labels reflect the order in which variables are assigned or + actions are done. */ +error_exit_1: + if (has_data_dir && err != DB_SUCCESS) { + fil_delete_link_file(tablename); + } +error_exit_2: + os_file_close(file); + if (err != DB_SUCCESS) { + os_file_delete(innodb_file_data_key, path); + } +error_exit_3: + mem_free(path); + + return(err); +} + +#ifndef UNIV_HOTBACKUP +/********************************************************************//** +Report information about a bad tablespace. */ +static +void +fil_report_bad_tablespace( +/*======================*/ + const char* filepath, /*!< in: filepath */ + const char* check_msg, /*!< in: fil_check_first_page() */ + ulint found_id, /*!< in: found space ID */ + ulint found_flags, /*!< in: found flags */ + ulint expected_id, /*!< in: expected space id */ + ulint expected_flags) /*!< in: expected flags */ +{ + if (check_msg) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Error %s in file '%s'," + "tablespace id=%lu, flags=%lu. " + "Please refer to " + REFMAN "innodb-troubleshooting-datadict.html " + "for how to resolve the issue.", + check_msg, filepath, + (ulong) expected_id, (ulong) expected_flags); + return; + } + + ib_logf(IB_LOG_LEVEL_ERROR, + "In file '%s', tablespace id and flags are %lu and %lu, " + "but in the InnoDB data dictionary they are %lu and %lu. " + "Have you moved InnoDB .ibd files around without using the " + "commands DISCARD TABLESPACE and IMPORT TABLESPACE? " + "Please refer to " + REFMAN "innodb-troubleshooting-datadict.html " + "for how to resolve the issue.", + filepath, (ulong) found_id, (ulong) found_flags, + (ulong) expected_id, (ulong) expected_flags); +} + +/********************************************************************//** +Tries to open a single-table tablespace and optionally checks that the +space id in it is correct. If this does not succeed, print an error message +to the .err log. This function is used to open a tablespace when we start +mysqld after the dictionary has been booted, and also in IMPORT TABLESPACE. + +NOTE that we assume this operation is used either at the database startup +or under the protection of the dictionary mutex, so that two users cannot +race here. This operation does not leave the file associated with the +tablespace open, but closes it after we have looked at the space id in it. + +If the validate boolean is set, we read the first page of the file and +check that the space id in the file is what we expect. We assume that +this function runs much faster if no check is made, since accessing the +file inode probably is much faster (the OS caches them) than accessing +the first page of the file. This boolean may be initially FALSE, but if +a remote tablespace is found it will be changed to true. + +If the fix_dict boolean is set, then it is safe to use an internal SQL +statement to update the dictionary tables if they are incorrect. + +@return DB_SUCCESS or error code */ +UNIV_INTERN +dberr_t +fil_open_single_table_tablespace( +/*=============================*/ + bool validate, /*!< in: Do we validate tablespace? */ + bool fix_dict, /*!< in: Can we fix the dictionary? */ + ulint id, /*!< in: space id */ + ulint flags, /*!< in: tablespace flags */ + const char* tablename, /*!< in: table name in the + databasename/tablename format */ + const char* path_in) /*!< in: tablespace filepath */ +{ + dberr_t err = DB_SUCCESS; + bool dict_filepath_same_as_default = false; + bool link_file_found = false; + bool link_file_is_bad = false; + fsp_open_info def; + fsp_open_info dict; + fsp_open_info remote; + ulint tablespaces_found = 0; + ulint valid_tablespaces_found = 0; + ulint atomic_writes = 0; + +#ifdef UNIV_SYNC_DEBUG + ut_ad(!fix_dict || rw_lock_own(&dict_operation_lock, RW_LOCK_EX)); +#endif /* UNIV_SYNC_DEBUG */ + ut_ad(!fix_dict || mutex_own(&(dict_sys->mutex))); + + /* Table flags can be ULINT_UNDEFINED if + dict_tf_to_fsp_flags_failure is set. */ + if (flags != ULINT_UNDEFINED) { + if (!fsp_flags_is_valid(flags)) { + return(DB_CORRUPTION); + } + } else { + return(DB_CORRUPTION); + } + + atomic_writes = fsp_flags_get_atomic_writes(flags); + + /* If the tablespace was relocated, we do not + compare the DATA_DIR flag */ + ulint mod_flags = flags & ~FSP_FLAGS_MASK_DATA_DIR; + + memset(&def, 0, sizeof(def)); + memset(&dict, 0, sizeof(dict)); + memset(&remote, 0, sizeof(remote)); + + /* Discover the correct filepath. We will always look for an ibd + in the default location. If it is remote, it should not be here. */ + def.filepath = fil_make_ibd_name(tablename, false); + + /* The path_in was read from SYS_DATAFILES. */ + if (path_in) { + if (strcmp(def.filepath, path_in)) { + dict.filepath = mem_strdup(path_in); + /* possibility of multiple files. */ + validate = true; + } else { + dict_filepath_same_as_default = true; + } + } + + link_file_found = fil_open_linked_file( + tablename, &remote.filepath, &remote.file, atomic_writes); + remote.success = link_file_found; + if (remote.success) { + /* possibility of multiple files. */ + validate = true; + tablespaces_found++; + + /* A link file was found. MySQL does not allow a DATA + DIRECTORY to be be the same as the default filepath. */ + ut_a(strcmp(def.filepath, remote.filepath)); + + /* If there was a filepath found in SYS_DATAFILES, + we hope it was the same as this remote.filepath found + in the ISL file. */ + if (dict.filepath + && (0 == strcmp(dict.filepath, remote.filepath))) { + remote.success = FALSE; + os_file_close(remote.file); + mem_free(remote.filepath); + remote.filepath = NULL; + tablespaces_found--; + } + } + + /* Attempt to open the tablespace at other possible filepaths. */ + if (dict.filepath) { + dict.file = os_file_create_simple_no_error_handling( + innodb_file_data_key, dict.filepath, OS_FILE_OPEN, + OS_FILE_READ_ONLY, &dict.success, atomic_writes); + if (dict.success) { + /* possibility of multiple files. */ + validate = true; + tablespaces_found++; + } + } + + /* Always look for a file at the default location. */ + ut_a(def.filepath); + def.file = os_file_create_simple_no_error_handling( + innodb_file_data_key, def.filepath, OS_FILE_OPEN, + OS_FILE_READ_ONLY, &def.success, atomic_writes); + if (def.success) { + tablespaces_found++; + } + + /* We have now checked all possible tablespace locations and + have a count of how many we found. If things are normal, we + only found 1. */ + if (!validate && tablespaces_found == 1) { + goto skip_validate; + } + + /* Read the first page of the datadir tablespace, if found. */ + if (def.success) { + def.check_msg = fil_read_first_page( + def.file, FALSE, &def.flags, &def.id, + &def.lsn, &def.lsn, id); + def.valid = !def.check_msg; + + /* Validate this single-table-tablespace with SYS_TABLES, + but do not compare the DATA_DIR flag, in case the + tablespace was relocated. */ + if (def.valid && def.id == id + && (def.flags & ~FSP_FLAGS_MASK_DATA_DIR) == mod_flags) { + valid_tablespaces_found++; + } else { + def.valid = false; + /* Do not use this tablespace. */ + fil_report_bad_tablespace( + def.filepath, def.check_msg, def.id, + def.flags, id, flags); + } + } + + /* Read the first page of the remote tablespace */ + if (remote.success) { + remote.check_msg = fil_read_first_page( + remote.file, FALSE, &remote.flags, &remote.id, + &remote.lsn, &remote.lsn, id); + remote.valid = !remote.check_msg; + + /* Validate this single-table-tablespace with SYS_TABLES, + but do not compare the DATA_DIR flag, in case the + tablespace was relocated. */ + if (remote.valid && remote.id == id + && (remote.flags & ~FSP_FLAGS_MASK_DATA_DIR) == mod_flags) { + valid_tablespaces_found++; + } else { + remote.valid = false; + /* Do not use this linked tablespace. */ + fil_report_bad_tablespace( + remote.filepath, remote.check_msg, remote.id, + remote.flags, id, flags); + link_file_is_bad = true; + } + } + + /* Read the first page of the datadir tablespace, if found. */ + if (dict.success) { + dict.check_msg = fil_read_first_page( + dict.file, FALSE, &dict.flags, &dict.id, + &dict.lsn, &dict.lsn, id); + dict.valid = !dict.check_msg; + + /* Validate this single-table-tablespace with SYS_TABLES, + but do not compare the DATA_DIR flag, in case the + tablespace was relocated. */ + if (dict.valid && dict.id == id + && (dict.flags & ~FSP_FLAGS_MASK_DATA_DIR) == mod_flags) { + valid_tablespaces_found++; + } else { + dict.valid = false; + /* Do not use this tablespace. */ + fil_report_bad_tablespace( + dict.filepath, dict.check_msg, dict.id, + dict.flags, id, flags); + } + } + + /* Make sense of these three possible locations. + First, bail out if no tablespace files were found. */ + if (valid_tablespaces_found == 0) { + /* The following call prints an error message */ + os_file_get_last_error(true); + + ib_logf(IB_LOG_LEVEL_ERROR, + "Could not find a valid tablespace file for '%s'. " + "See " REFMAN "innodb-troubleshooting-datadict.html " + "for how to resolve the issue.", + tablename); + + err = DB_CORRUPTION; + + goto cleanup_and_exit; + } + + /* Do not open any tablespaces if more than one tablespace with + the correct space ID and flags were found. */ + if (tablespaces_found > 1) { + ib_logf(IB_LOG_LEVEL_ERROR, + "A tablespace for %s has been found in " + "multiple places;", tablename); + if (def.success) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Default location; %s, LSN=" LSN_PF + ", Space ID=%lu, Flags=%lu", + def.filepath, def.lsn, + (ulong) def.id, (ulong) def.flags); + } + if (remote.success) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Remote location; %s, LSN=" LSN_PF + ", Space ID=%lu, Flags=%lu", + remote.filepath, remote.lsn, + (ulong) remote.id, (ulong) remote.flags); + } + if (dict.success) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Dictionary location; %s, LSN=" LSN_PF + ", Space ID=%lu, Flags=%lu", + dict.filepath, dict.lsn, + (ulong) dict.id, (ulong) dict.flags); + } + + /* Force-recovery will allow some tablespaces to be + skipped by REDO if there was more than one file found. + Unlike during the REDO phase of recovery, we now know + if the tablespace is valid according to the dictionary, + which was not available then. So if we did not force + recovery and there is only one good tablespace, ignore + any bad tablespaces. */ + if (valid_tablespaces_found > 1 || srv_force_recovery > 0) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Will not open the tablespace for '%s'", + tablename); + + if (def.success != def.valid + || dict.success != dict.valid + || remote.success != remote.valid) { + err = DB_CORRUPTION; + } else { + err = DB_ERROR; + } + goto cleanup_and_exit; + } + + /* There is only one valid tablespace found and we did + not use srv_force_recovery during REDO. Use this one + tablespace and clean up invalid tablespace pointers */ + if (def.success && !def.valid) { + def.success = false; + os_file_close(def.file); + tablespaces_found--; + } + if (dict.success && !dict.valid) { + dict.success = false; + os_file_close(dict.file); + /* Leave dict.filepath so that SYS_DATAFILES + can be corrected below. */ + tablespaces_found--; + } + if (remote.success && !remote.valid) { + remote.success = false; + os_file_close(remote.file); + mem_free(remote.filepath); + remote.filepath = NULL; + tablespaces_found--; + } + } + + /* At this point, there should be only one filepath. */ + ut_a(tablespaces_found == 1); + ut_a(valid_tablespaces_found == 1); + + /* Only fix the dictionary at startup when there is only one thread. + Calls to dict_load_table() can be done while holding other latches. */ + if (!fix_dict) { + goto skip_validate; + } + + /* We may need to change what is stored in SYS_DATAFILES or + SYS_TABLESPACES or adjust the link file. + Since a failure to update SYS_TABLESPACES or SYS_DATAFILES does + not prevent opening and using the single_table_tablespace either + this time or the next, we do not check the return code or fail + to open the tablespace. But dict_update_filepath() will issue a + warning to the log. */ + if (dict.filepath) { + if (remote.success) { + dict_update_filepath(id, remote.filepath); + } else if (def.success) { + dict_update_filepath(id, def.filepath); + if (link_file_is_bad) { + fil_delete_link_file(tablename); + } + } else if (!link_file_found || link_file_is_bad) { + ut_ad(dict.success); + /* Fix the link file if we got our filepath + from the dictionary but a link file did not + exist or it did not point to a valid file. */ + fil_delete_link_file(tablename); + fil_create_link_file(tablename, dict.filepath); + } + + } else if (remote.success && dict_filepath_same_as_default) { + dict_update_filepath(id, remote.filepath); + + } else if (remote.success && path_in == NULL) { + /* SYS_DATAFILES record for this space ID was not found. */ + dict_insert_tablespace_and_filepath( + id, tablename, remote.filepath, flags); + } + +skip_validate: + if (err != DB_SUCCESS) { + ; // Don't load the tablespace into the cache + } else if (!fil_space_create(tablename, id, flags, FIL_TABLESPACE)) { + err = DB_ERROR; + } else { + /* We do not measure the size of the file, that is why + we pass the 0 below */ + + if (!fil_node_create(remote.success ? remote.filepath : + dict.success ? dict.filepath : + def.filepath, 0, id, FALSE)) { + err = DB_ERROR; + } + } + +cleanup_and_exit: + if (remote.success) { + os_file_close(remote.file); + } + if (remote.filepath) { + mem_free(remote.filepath); + } + if (dict.success) { + os_file_close(dict.file); + } + if (dict.filepath) { + mem_free(dict.filepath); + } + if (def.success) { + os_file_close(def.file); + } + mem_free(def.filepath); + + return(err); +} +#endif /* !UNIV_HOTBACKUP */ + +#ifdef UNIV_HOTBACKUP +/*******************************************************************//** +Allocates a file name for an old version of a single-table tablespace. +The string must be freed by caller with mem_free()! +@return own: file name */ +static +char* +fil_make_ibbackup_old_name( +/*=======================*/ + const char* name) /*!< in: original file name */ +{ + static const char suffix[] = "_ibbackup_old_vers_"; + char* path; + ulint len = strlen(name); + + path = static_cast(mem_alloc(len + (15 + sizeof suffix))); + + memcpy(path, name, len); + memcpy(path + len, suffix, (sizeof suffix) - 1); + ut_sprintf_timestamp_without_extra_chars( + path + len + ((sizeof suffix) - 1)); + return(path); +} +#endif /* UNIV_HOTBACKUP */ + + +/*******************************************************************//** +Determine the space id of the given file descriptor by reading a few +pages from the beginning of the .ibd file. +@return true if space id was successfully identified, or false. */ +static +bool +fil_user_tablespace_find_space_id( +/*==============================*/ + fsp_open_info* fsp) /* in/out: contains file descriptor, which is + used as input. contains space_id, which is + the output */ +{ + bool st; + os_offset_t file_size; + + file_size = os_file_get_size(fsp->file); + + if (file_size == (os_offset_t) -1) { + ib_logf(IB_LOG_LEVEL_ERROR, "Could not get file size: %s", + fsp->filepath); + return(false); + } + + /* Assuming a page size, read the space_id from each page and store it + in a map. Find out which space_id is agreed on by majority of the + pages. Choose that space_id. */ + for (ulint page_size = UNIV_ZIP_SIZE_MIN; + page_size <= UNIV_PAGE_SIZE_MAX; page_size <<= 1) { + + /* map[space_id] = count of pages */ + std::map verify; + + ulint page_count = 64; + ulint valid_pages = 0; + + /* Adjust the number of pages to analyze based on file size */ + while ((page_count * page_size) > file_size) { + --page_count; + } + + ib_logf(IB_LOG_LEVEL_INFO, "Page size:%lu Pages to analyze:" + "%lu", page_size, page_count); + + byte* buf = static_cast(ut_malloc(2*page_size)); + byte* page = static_cast(ut_align(buf, page_size)); + + for (ulint j = 0; j < page_count; ++j) { + + st = os_file_read(fsp->file, page, (j* page_size), page_size, + fsp_flags_is_page_compressed(fsp->flags)); + + if (!st) { + ib_logf(IB_LOG_LEVEL_INFO, + "READ FAIL: page_no:%lu", j); + continue; + } + + bool uncompressed_ok = false; + + /* For uncompressed pages, the page size must be equal + to UNIV_PAGE_SIZE. */ + if (page_size == UNIV_PAGE_SIZE) { + uncompressed_ok = !buf_page_is_corrupted( + false, page, 0); + } + + bool compressed_ok = !buf_page_is_corrupted( + false, page, page_size); + + if (uncompressed_ok || compressed_ok) { + + ulint space_id = mach_read_from_4(page + + FIL_PAGE_SPACE_ID); + + if (space_id > 0) { + ib_logf(IB_LOG_LEVEL_INFO, + "VALID: space:%lu " + "page_no:%lu page_size:%lu", + space_id, j, page_size); + verify[space_id]++; + ++valid_pages; + } + } + } + + ut_free(buf); + + ib_logf(IB_LOG_LEVEL_INFO, "Page size: %lu, Possible space_id " + "count:%lu", page_size, (ulint) verify.size()); + + const ulint pages_corrupted = 3; + for (ulint missed = 0; missed <= pages_corrupted; ++missed) { + + for (std::map::iterator + m = verify.begin(); m != verify.end(); ++m ) { + + ib_logf(IB_LOG_LEVEL_INFO, "space_id:%lu, " + "Number of pages matched: %lu/%lu " + "(%lu)", m->first, m->second, + valid_pages, page_size); + + if (m->second == (valid_pages - missed)) { + + ib_logf(IB_LOG_LEVEL_INFO, + "Chosen space:%lu\n", m->first); + + fsp->id = m->first; + return(true); + } + } + + } + } + + return(false); +} + +/*******************************************************************//** +Finds the given page_no of the given space id from the double write buffer, +and copies it to the corresponding .ibd file. +@return true if copy was successful, or false. */ +bool +fil_user_tablespace_restore_page( +/*==============================*/ + fsp_open_info* fsp, /* in: contains space id and .ibd + file information */ + ulint page_no) /* in: page_no to obtain from double + write buffer */ +{ + bool err; + ulint flags; + ulint zip_size; + ulint page_size; + ulint buflen; + byte* page; + + ib_logf(IB_LOG_LEVEL_INFO, "Restoring page %lu of tablespace %lu", + page_no, fsp->id); + + // find if double write buffer has page_no of given space id + page = recv_sys->dblwr.find_page(fsp->id, page_no); + + if (!page) { + ib_logf(IB_LOG_LEVEL_WARN, "Doublewrite does not have " + "page_no=%lu of space: %lu", page_no, fsp->id); + err = false; + goto out; + } + + flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); + zip_size = fsp_flags_get_zip_size(flags); + page_size = fsp_flags_get_page_size(flags); + + ut_ad(page_no == page_get_page_no(page)); + + buflen = zip_size ? zip_size: page_size; + + ib_logf(IB_LOG_LEVEL_INFO, "Writing %lu bytes into file: %s", + buflen, fsp->filepath); + + err = os_file_write(fsp->filepath, fsp->file, page, + (zip_size ? zip_size : page_size) * page_no, + buflen); + + os_file_flush(fsp->file); +out: + return(err); +} + +/********************************************************************//** +Opens an .ibd file and adds the associated single-table tablespace to the +InnoDB fil0fil.cc data structures. +Set fsp->success to TRUE if tablespace is valid, FALSE if not. */ +static +void +fil_validate_single_table_tablespace( +/*=================================*/ + const char* tablename, /*!< in: database/tablename */ + fsp_open_info* fsp) /*!< in/out: tablespace info */ +{ + bool restore_attempted = false; + +check_first_page: + fsp->success = TRUE; + if (const char* check_msg = fil_read_first_page( + fsp->file, FALSE, &fsp->flags, &fsp->id, + &fsp->lsn, &fsp->lsn, ULINT_UNDEFINED)) { + ib_logf(IB_LOG_LEVEL_ERROR, + "%s in tablespace %s (table %s)", + check_msg, fsp->filepath, tablename); + fsp->success = FALSE; + } + + if (!fsp->success) { + if (!restore_attempted) { + if (!fil_user_tablespace_find_space_id(fsp)) { + return; + } + restore_attempted = true; + + if (fsp->id > 0 + && !fil_user_tablespace_restore_page(fsp, 0)) { + return; + } + goto check_first_page; + } + return; + } + + if (fsp->id == ULINT_UNDEFINED || fsp->id == 0) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Tablespace is not sensible;" + " Table: %s Space ID: %lu Filepath: %s\n", + tablename, (ulong) fsp->id, fsp->filepath); + fsp->success = FALSE; + return; + } + + mutex_enter(&fil_system->mutex); + fil_space_t* space = fil_space_get_by_id(fsp->id); + mutex_exit(&fil_system->mutex); + if (space != NULL) { + char* prev_filepath = fil_space_get_first_path(fsp->id); + + ib_logf(IB_LOG_LEVEL_ERROR, + "Attempted to open a previously opened tablespace. " + "Previous tablespace %s uses space ID: %lu at " + "filepath: %s. Cannot open tablespace %s which uses " + "space ID: %lu at filepath: %s", + space->name, (ulong) space->id, prev_filepath, + tablename, (ulong) fsp->id, fsp->filepath); + + mem_free(prev_filepath); + fsp->success = FALSE; + return; + } + + fsp->success = TRUE; +} + + +/********************************************************************//** +Opens an .ibd file and adds the associated single-table tablespace to the +InnoDB fil0fil.cc data structures. */ +static +void +fil_load_single_table_tablespace( +/*=============================*/ + const char* dbname, /*!< in: database name */ + const char* filename) /*!< in: file name (not a path), + including the .ibd or .isl extension */ +{ + char* tablename; + ulint tablename_len; + ulint dbname_len = strlen(dbname); + ulint filename_len = strlen(filename); + fsp_open_info def; + fsp_open_info remote; + os_offset_t size; + fil_space_t* space; + + memset(&def, 0, sizeof(def)); + memset(&remote, 0, sizeof(remote)); + + /* The caller assured that the extension is ".ibd" or ".isl". */ + ut_ad(0 == memcmp(filename + filename_len - 4, ".ibd", 4) + || 0 == memcmp(filename + filename_len - 4, ".isl", 4)); + + /* Build up the tablename in the standard form database/table. */ + tablename = static_cast( + mem_alloc(dbname_len + filename_len + 2)); + sprintf(tablename, "%s/%s", dbname, filename); + tablename_len = strlen(tablename) - strlen(".ibd"); + tablename[tablename_len] = '\0'; + + /* There may be both .ibd and .isl file in the directory. + And it is possible that the .isl file refers to a different + .ibd file. If so, we open and compare them the first time + one of them is sent to this function. So if this table has + already been loaded, there is nothing to do.*/ + mutex_enter(&fil_system->mutex); + space = fil_space_get_by_name(tablename); + if (space) { + mem_free(tablename); + mutex_exit(&fil_system->mutex); + return; + } + mutex_exit(&fil_system->mutex); + + /* Build up the filepath of the .ibd tablespace in the datadir. + This must be freed independent of def.success. */ + def.filepath = fil_make_ibd_name(tablename, false); + +#ifdef __WIN__ +# ifndef UNIV_HOTBACKUP + /* If lower_case_table_names is 0 or 2, then MySQL allows database + directory names with upper case letters. On Windows, all table and + database names in InnoDB are internally always in lower case. Put the + file path to lower case, so that we are consistent with InnoDB's + internal data dictionary. */ + + dict_casedn_str(def.filepath); +# endif /* !UNIV_HOTBACKUP */ +#endif + + /* Check for a link file which locates a remote tablespace. */ + remote.success = fil_open_linked_file( + tablename, &remote.filepath, &remote.file, FALSE); + + /* Read the first page of the remote tablespace */ + if (remote.success) { + fil_validate_single_table_tablespace(tablename, &remote); + if (!remote.success) { + os_file_close(remote.file); + mem_free(remote.filepath); + } + } + + + /* Try to open the tablespace in the datadir. */ + def.file = os_file_create_simple_no_error_handling( + innodb_file_data_key, def.filepath, OS_FILE_OPEN, + OS_FILE_READ_WRITE, &def.success, FALSE); + + /* Read the first page of the remote tablespace */ + if (def.success) { + fil_validate_single_table_tablespace(tablename, &def); + if (!def.success) { + os_file_close(def.file); + } + } + + if (!def.success && !remote.success) { + /* The following call prints an error message */ + os_file_get_last_error(true); + fprintf(stderr, + "InnoDB: Error: could not open single-table" + " tablespace file %s\n", def.filepath); + + if (!strncmp(filename, + tmp_file_prefix, tmp_file_prefix_length)) { + /* Ignore errors for #sql tablespaces. */ + mem_free(tablename); + if (remote.filepath) { + mem_free(remote.filepath); + } + if (def.filepath) { + mem_free(def.filepath); + } + return; + } +no_good_file: + fprintf(stderr, + "InnoDB: We do not continue the crash recovery," + " because the table may become\n" + "InnoDB: corrupt if we cannot apply the log" + " records in the InnoDB log to it.\n" + "InnoDB: To fix the problem and start mysqld:\n" + "InnoDB: 1) If there is a permission problem" + " in the file and mysqld cannot\n" + "InnoDB: open the file, you should" + " modify the permissions.\n" + "InnoDB: 2) If the table is not needed, or you" + " can restore it from a backup,\n" + "InnoDB: then you can remove the .ibd file," + " and InnoDB will do a normal\n" + "InnoDB: crash recovery and ignore that table.\n" + "InnoDB: 3) If the file system or the" + " disk is broken, and you cannot remove\n" + "InnoDB: the .ibd file, you can set" + " innodb_force_recovery > 0 in my.cnf\n" + "InnoDB: and force InnoDB to continue crash" + " recovery here.\n"); +will_not_choose: + mem_free(tablename); + if (remote.filepath) { + mem_free(remote.filepath); + } + if (def.filepath) { + mem_free(def.filepath); + } + + if (srv_force_recovery > 0) { + ib_logf(IB_LOG_LEVEL_INFO, + "innodb_force_recovery was set to %lu. " + "Continuing crash recovery even though we " + "cannot access the .ibd file of this table.", + srv_force_recovery); + return; + } + + exit(1); + } + + if (def.success && remote.success) { + ib_logf(IB_LOG_LEVEL_ERROR, + "Tablespaces for %s have been found in two places;\n" + "Location 1: SpaceID: %lu LSN: %lu File: %s\n" + "Location 2: SpaceID: %lu LSN: %lu File: %s\n" + "You must delete one of them.", + tablename, (ulong) def.id, (ulong) def.lsn, + def.filepath, (ulong) remote.id, (ulong) remote.lsn, + remote.filepath); + + def.success = FALSE; + os_file_close(def.file); + os_file_close(remote.file); + goto will_not_choose; + } + + /* At this point, only one tablespace is open */ + ut_a(def.success == !remote.success); + + fsp_open_info* fsp = def.success ? &def : &remote; + + /* Get and test the file size. */ + size = os_file_get_size(fsp->file); + + if (size == (os_offset_t) -1) { + /* The following call prints an error message */ + os_file_get_last_error(true); + + ib_logf(IB_LOG_LEVEL_ERROR, + "could not measure the size of single-table " + "tablespace file %s", fsp->filepath); + + os_file_close(fsp->file); + goto no_good_file; + } + + /* Every .ibd file is created >= 4 pages in size. Smaller files + cannot be ok. */ + ulong minimum_size = FIL_IBD_FILE_INITIAL_SIZE * UNIV_PAGE_SIZE; + if (size < minimum_size) { +#ifndef UNIV_HOTBACKUP + ib_logf(IB_LOG_LEVEL_ERROR, + "The size of single-table tablespace file %s " + "is only " UINT64PF ", should be at least %lu!", + fsp->filepath, size, minimum_size); + os_file_close(fsp->file); + goto no_good_file; +#else + fsp->id = ULINT_UNDEFINED; + fsp->flags = 0; +#endif /* !UNIV_HOTBACKUP */ + } + +#ifdef UNIV_HOTBACKUP + if (fsp->id == ULINT_UNDEFINED || fsp->id == 0) { + char* new_path; + + fprintf(stderr, + "InnoDB: Renaming tablespace %s of id %lu,\n" + "InnoDB: to %s_ibbackup_old_vers_\n" + "InnoDB: because its size %" PRId64 " is too small" + " (< 4 pages 16 kB each),\n" + "InnoDB: or the space id in the file header" + " is not sensible.\n" + "InnoDB: This can happen in an ibbackup run," + " and is not dangerous.\n", + fsp->filepath, fsp->id, fsp->filepath, size); + os_file_close(fsp->file); + + new_path = fil_make_ibbackup_old_name(fsp->filepath); + + bool success = os_file_rename( + innodb_file_data_key, fsp->filepath, new_path); + + ut_a(success); + + mem_free(new_path); + + goto func_exit_after_close; + } + + /* A backup may contain the same space several times, if the space got + renamed at a sensitive time. Since it is enough to have one version of + the space, we rename the file if a space with the same space id + already exists in the tablespace memory cache. We rather rename the + file than delete it, because if there is a bug, we do not want to + destroy valuable data. */ + + mutex_enter(&fil_system->mutex); + + space = fil_space_get_by_id(fsp->id); + + if (space) { + char* new_path; + + fprintf(stderr, + "InnoDB: Renaming tablespace %s of id %lu,\n" + "InnoDB: to %s_ibbackup_old_vers_\n" + "InnoDB: because space %s with the same id\n" + "InnoDB: was scanned earlier. This can happen" + " if you have renamed tables\n" + "InnoDB: during an ibbackup run.\n", + fsp->filepath, fsp->id, fsp->filepath, + space->name); + os_file_close(fsp->file); + + new_path = fil_make_ibbackup_old_name(fsp->filepath); + + mutex_exit(&fil_system->mutex); + + bool success = os_file_rename( + innodb_file_data_key, fsp->filepath, new_path); + + ut_a(success); + + mem_free(new_path); + + goto func_exit_after_close; + } + mutex_exit(&fil_system->mutex); +#endif /* UNIV_HOTBACKUP */ + ibool file_space_create_success = fil_space_create( + tablename, fsp->id, fsp->flags, FIL_TABLESPACE); + + if (!file_space_create_success) { + if (srv_force_recovery > 0) { + fprintf(stderr, + "InnoDB: innodb_force_recovery was set" + " to %lu. Continuing crash recovery\n" + "InnoDB: even though the tablespace" + " creation of this table failed.\n", + srv_force_recovery); + goto func_exit; + } + + /* Exit here with a core dump, stack, etc. */ + ut_a(file_space_create_success); + } + + /* We do not use the size information we have about the file, because + the rounding formula for extents and pages is somewhat complex; we + let fil_node_open() do that task. */ + + if (!fil_node_create(fsp->filepath, 0, fsp->id, FALSE)) { + ut_error; + } + +func_exit: + os_file_close(fsp->file); + +#ifdef UNIV_HOTBACKUP +func_exit_after_close: +#else + ut_ad(!mutex_own(&fil_system->mutex)); +#endif + mem_free(tablename); + if (remote.success) { + mem_free(remote.filepath); + } + mem_free(def.filepath); +} + +/***********************************************************************//** +A fault-tolerant function that tries to read the next file name in the +directory. We retry 100 times if os_file_readdir_next_file() returns -1. The +idea is to read as much good data as we can and jump over bad data. +@return 0 if ok, -1 if error even after the retries, 1 if at the end +of the directory */ +static +int +fil_file_readdir_next_file( +/*=======================*/ + dberr_t* err, /*!< out: this is set to DB_ERROR if an error + was encountered, otherwise not changed */ + const char* dirname,/*!< in: directory name or path */ + os_file_dir_t dir, /*!< in: directory stream */ + os_file_stat_t* info) /*!< in/out: buffer where the + info is returned */ +{ + for (ulint i = 0; i < 100; i++) { + int ret = os_file_readdir_next_file(dirname, dir, info); + + if (ret != -1) { + + return(ret); + } + + ib_logf(IB_LOG_LEVEL_ERROR, + "os_file_readdir_next_file() returned -1 in " + "directory %s, crash recovery may have failed " + "for some .ibd files!", dirname); + + *err = DB_ERROR; + } + + return(-1); +} + +#define CHECK_TIME_EVERY_N_FILES 10 +/********************************************************************//** +At the server startup, if we need crash recovery, scans the database +directories under the MySQL datadir, looking for .ibd files. Those files are +single-table tablespaces. We need to know the space id in each of them so that +we know into which file we should look to check the contents of a page stored +in the doublewrite buffer, also to know where to apply log records where the +space id is != 0. +@return DB_SUCCESS or error number */ +UNIV_INTERN +dberr_t +fil_load_single_table_tablespaces(void) +/*===================================*/ +{ + int ret; + char* dbpath = NULL; + ulint dbpath_len = 100; + ulint files_read = 0; + ulint files_read_at_last_check = 0; + ib_time_t prev_report_time = ut_time(); + os_file_dir_t dir; + os_file_dir_t dbdir; + os_file_stat_t dbinfo; + os_file_stat_t fileinfo; + dberr_t err = DB_SUCCESS; + + /* The datadir of MySQL is always the default directory of mysqld */ + + dir = os_file_opendir(fil_path_to_mysql_datadir, TRUE); + + if (dir == NULL) { + + return(DB_ERROR); + } + + dbpath = static_cast(mem_alloc(dbpath_len)); + + /* Scan all directories under the datadir. They are the database + directories of MySQL. */ + + ret = fil_file_readdir_next_file(&err, fil_path_to_mysql_datadir, dir, + &dbinfo); + while (ret == 0) { + ulint len; + /* printf("Looking at %s in datadir\n", dbinfo.name); */ + + if (dbinfo.type == OS_FILE_TYPE_FILE + || dbinfo.type == OS_FILE_TYPE_UNKNOWN) { + + goto next_datadir_item; + } + + /* We found a symlink or a directory; try opening it to see + if a symlink is a directory */ + + len = strlen(fil_path_to_mysql_datadir) + + strlen (dbinfo.name) + 2; + if (len > dbpath_len) { + dbpath_len = len; + + if (dbpath) { + mem_free(dbpath); + } + + dbpath = static_cast(mem_alloc(dbpath_len)); + } + ut_snprintf(dbpath, dbpath_len, + "%s/%s", fil_path_to_mysql_datadir, dbinfo.name); + srv_normalize_path_for_win(dbpath); + + dbdir = os_file_opendir(dbpath, FALSE); + + if (dbdir != NULL) { + + /* We found a database directory; loop through it, + looking for possible .ibd files in it */ + + ret = fil_file_readdir_next_file(&err, dbpath, dbdir, + &fileinfo); + while (ret == 0) { + + if (fileinfo.type == OS_FILE_TYPE_DIR) { + + goto next_file_item; + } + + /* We found a symlink or a file */ + if (strlen(fileinfo.name) > 4 + && (0 == strcmp(fileinfo.name + + strlen(fileinfo.name) - 4, + ".ibd") + || 0 == strcmp(fileinfo.name + + strlen(fileinfo.name) - 4, + ".isl"))) { + /* The name ends in .ibd or .isl; + try opening the file */ + fil_load_single_table_tablespace( + dbinfo.name, fileinfo.name); + files_read++; + if (files_read - files_read_at_last_check > + CHECK_TIME_EVERY_N_FILES) { + ib_time_t cur_time= ut_time(); + files_read_at_last_check= files_read; + double time_elapsed= ut_difftime(cur_time, + prev_report_time); + if (time_elapsed > 15) { + ib_logf(IB_LOG_LEVEL_INFO, + "Processed %ld .ibd/.isl files", + files_read); + prev_report_time= cur_time; + } + } + } +next_file_item: + ret = fil_file_readdir_next_file(&err, + dbpath, dbdir, + &fileinfo); + } + + if (0 != os_file_closedir(dbdir)) { + fputs("InnoDB: Warning: could not" + " close database directory ", stderr); + ut_print_filename(stderr, dbpath); + putc('\n', stderr); + + err = DB_ERROR; + } + } + +next_datadir_item: + ret = fil_file_readdir_next_file(&err, + fil_path_to_mysql_datadir, + dir, &dbinfo); + } + + mem_free(dbpath); + + if (0 != os_file_closedir(dir)) { + fprintf(stderr, + "InnoDB: Error: could not close MySQL datadir\n"); + + return(DB_ERROR); + } + + return(err); +} + +/*******************************************************************//** +Returns TRUE if a single-table tablespace does not exist in the memory cache, +or is being deleted there. +@return TRUE if does not exist or is being deleted */ +UNIV_INTERN +ibool +fil_tablespace_deleted_or_being_deleted_in_mem( +/*===========================================*/ + ulint id, /*!< in: space id */ + ib_int64_t version)/*!< in: tablespace_version should be this; if + you pass -1 as the value of this, then this + parameter is ignored */ +{ + fil_space_t* space; + + ut_ad(fil_system); + + mutex_enter(&fil_system->mutex); + + space = fil_space_get_by_id(id); + + if (space == NULL || space->stop_new_ops) { + mutex_exit(&fil_system->mutex); + + return(TRUE); + } + + if (version != ((ib_int64_t)-1) + && space->tablespace_version != version) { + mutex_exit(&fil_system->mutex); + + return(TRUE); + } + + mutex_exit(&fil_system->mutex); + + return(FALSE); +} + +/*******************************************************************//** +Returns TRUE if a single-table tablespace exists in the memory cache. +@return TRUE if exists */ +UNIV_INTERN +ibool +fil_tablespace_exists_in_mem( +/*=========================*/ + ulint id) /*!< in: space id */ +{ + fil_space_t* space; + + ut_ad(fil_system); + + mutex_enter(&fil_system->mutex); + + space = fil_space_get_by_id(id); + + mutex_exit(&fil_system->mutex); + + return(space != NULL); +} + +/*******************************************************************//** +Report that a tablespace for a table was not found. */ +static +void +fil_report_missing_tablespace( +/*===========================*/ + const char* name, /*!< in: table name */ + ulint space_id) /*!< in: table's space id */ +{ + char index_name[MAX_FULL_NAME_LEN + 1]; + + innobase_format_name(index_name, sizeof(index_name), name, TRUE); + + ib_logf(IB_LOG_LEVEL_ERROR, + "Table %s in the InnoDB data dictionary has tablespace id %lu, " + "but tablespace with that id or name does not exist. Have " + "you deleted or moved .ibd files? This may also be a table " + "created with CREATE TEMPORARY TABLE whose .ibd and .frm " + "files MySQL automatically removed, but the table still " + "exists in the InnoDB internal data dictionary.", + name, space_id); +} + +/*******************************************************************//** +Returns TRUE if a matching tablespace exists in the InnoDB tablespace memory +cache. Note that if we have not done a crash recovery at the database startup, +there may be many tablespaces which are not yet in the memory cache. +@return TRUE if a matching tablespace exists in the memory cache */ +UNIV_INTERN +ibool +fil_space_for_table_exists_in_mem( +/*==============================*/ + ulint id, /*!< in: space id */ + const char* name, /*!< in: table name used in + fil_space_create(). Either the + standard 'dbname/tablename' format + or table->dir_path_of_temp_table */ + ibool mark_space, /*!< in: in crash recovery, at database + startup we mark all spaces which have + an associated table in the InnoDB + data dictionary, so that + we can print a warning about orphaned + tablespaces */ + ibool print_error_if_does_not_exist, + /*!< in: print detailed error + information to the .err log if a + matching tablespace is not found from + memory */ + bool adjust_space, /*!< in: whether to adjust space id + when find table space mismatch */ + mem_heap_t* heap, /*!< in: heap memory */ + table_id_t table_id) /*!< in: table id */ +{ + fil_space_t* fnamespace; + fil_space_t* space; + + ut_ad(fil_system); + + mutex_enter(&fil_system->mutex); + + /* Look if there is a space with the same id */ + + space = fil_space_get_by_id(id); + + /* Look if there is a space with the same name; the name is the + directory path from the datadir to the file */ + + fnamespace = fil_space_get_by_name(name); + if (space && space == fnamespace) { + /* Found */ + + if (mark_space) { + space->mark = TRUE; + } + + mutex_exit(&fil_system->mutex); + + return(TRUE); + } + + /* Info from "fnamespace" comes from the ibd file itself, it can + be different from data obtained from System tables since it is + not transactional. If adjust_space is set, and the mismatching + space are between a user table and its temp table, we shall + adjust the ibd file name according to system table info */ + if (adjust_space + && space != NULL + && row_is_mysql_tmp_table_name(space->name) + && !row_is_mysql_tmp_table_name(name)) { + + mutex_exit(&fil_system->mutex); + + DBUG_EXECUTE_IF("ib_crash_before_adjust_fil_space", + DBUG_SUICIDE();); + + if (fnamespace) { + char* tmp_name; + + tmp_name = dict_mem_create_temporary_tablename( + heap, name, table_id); + + fil_rename_tablespace(fnamespace->name, fnamespace->id, + tmp_name, NULL); + } + + DBUG_EXECUTE_IF("ib_crash_after_adjust_one_fil_space", + DBUG_SUICIDE();); + + fil_rename_tablespace(space->name, id, name, NULL); + + DBUG_EXECUTE_IF("ib_crash_after_adjust_fil_space", + DBUG_SUICIDE();); + + mutex_enter(&fil_system->mutex); + fnamespace = fil_space_get_by_name(name); + ut_ad(space == fnamespace); + mutex_exit(&fil_system->mutex); + + return(TRUE); + } + + if (!print_error_if_does_not_exist) { + + mutex_exit(&fil_system->mutex); + + return(FALSE); + } + + if (space == NULL) { + if (fnamespace == NULL) { + if (print_error_if_does_not_exist) { + fil_report_missing_tablespace(name, id); + } + } else { + ut_print_timestamp(stderr); + fputs(" InnoDB: Error: table ", stderr); + ut_print_filename(stderr, name); + fprintf(stderr, "\n" + "InnoDB: in InnoDB data dictionary has" + " tablespace id %lu,\n" + "InnoDB: but a tablespace with that id" + " does not exist. There is\n" + "InnoDB: a tablespace of name %s and id %lu," + " though. Have\n" + "InnoDB: you deleted or moved .ibd files?\n", + (ulong) id, fnamespace->name, + (ulong) fnamespace->id); + } +error_exit: + fputs("InnoDB: Please refer to\n" + "InnoDB: " REFMAN "innodb-troubleshooting-datadict.html\n" + "InnoDB: for how to resolve the issue.\n", stderr); + + mutex_exit(&fil_system->mutex); + + return(FALSE); + } + + if (0 != strcmp(space->name, name)) { + ut_print_timestamp(stderr); + fputs(" InnoDB: Error: table ", stderr); + ut_print_filename(stderr, name); + fprintf(stderr, "\n" + "InnoDB: in InnoDB data dictionary has" + " tablespace id %lu,\n" + "InnoDB: but the tablespace with that id" + " has name %s.\n" + "InnoDB: Have you deleted or moved .ibd files?\n", + (ulong) id, space->name); + + if (fnamespace != NULL) { + fputs("InnoDB: There is a tablespace" + " with the right name\n" + "InnoDB: ", stderr); + ut_print_filename(stderr, fnamespace->name); + fprintf(stderr, ", but its id is %lu.\n", + (ulong) fnamespace->id); + } + + goto error_exit; + } + + mutex_exit(&fil_system->mutex); + + return(FALSE); +} + +/*******************************************************************//** +Checks if a single-table tablespace for a given table name exists in the +tablespace memory cache. +@return space id, ULINT_UNDEFINED if not found */ +UNIV_INTERN +ulint +fil_get_space_id_for_table( +/*=======================*/ + const char* tablename) /*!< in: table name in the standard + 'databasename/tablename' format */ +{ + fil_space_t* fnamespace; + ulint id = ULINT_UNDEFINED; + + ut_ad(fil_system); + + mutex_enter(&fil_system->mutex); + + /* Look if there is a space with the same name. */ + + fnamespace = fil_space_get_by_name(tablename); + + if (fnamespace) { + id = fnamespace->id; + } + + mutex_exit(&fil_system->mutex); + + return(id); +} + +/**********************************************************************//** +Tries to extend a data file so that it would accommodate the number of pages +given. The tablespace must be cached in the memory cache. If the space is big +enough already, does nothing. +@return TRUE if success */ +UNIV_INTERN +ibool +fil_extend_space_to_desired_size( +/*=============================*/ + ulint* actual_size, /*!< out: size of the space after extension; + if we ran out of disk space this may be lower + than the desired size */ + ulint space_id, /*!< in: space id */ + ulint size_after_extend)/*!< in: desired size in pages after the + extension; if the current space size is bigger + than this already, the function does nothing */ +{ + fil_node_t* node; + fil_space_t* space; + byte* buf2; + byte* buf; + ulint buf_size; + ulint start_page_no; + ulint file_start_page_no; + ulint page_size; + ulint pages_added; + ibool success; + + ut_ad(!srv_read_only_mode); + +retry: + pages_added = 0; + success = TRUE; + + fil_mutex_enter_and_prepare_for_io(space_id); + + space = fil_space_get_by_id(space_id); + ut_a(space); + + if (space->size >= size_after_extend) { + /* Space already big enough */ + + *actual_size = space->size; + + mutex_exit(&fil_system->mutex); + + return(TRUE); + } + + page_size = fsp_flags_get_zip_size(space->flags); + if (!page_size) { + page_size = UNIV_PAGE_SIZE; + } + + node = UT_LIST_GET_LAST(space->chain); + + if (!node->being_extended) { + /* Mark this node as undergoing extension. This flag + is used by other threads to wait for the extension + opereation to finish. */ + node->being_extended = TRUE; + } else { + /* Another thread is currently extending the file. Wait + for it to finish. + It'd have been better to use event driven mechanism but + the entire module is peppered with polling stuff. */ + mutex_exit(&fil_system->mutex); + os_thread_sleep(100000); + goto retry; + } + + if (!fil_node_prepare_for_io(node, fil_system, space)) { + /* The tablespace data file, such as .ibd file, is missing */ + node->being_extended = false; + mutex_exit(&fil_system->mutex); + + return(false); + } + + /* At this point it is safe to release fil_system mutex. No + other thread can rename, delete or close the file because + we have set the node->being_extended flag. */ + mutex_exit(&fil_system->mutex); + + start_page_no = space->size; + file_start_page_no = space->size - node->size; + +#ifdef HAVE_POSIX_FALLOCATE + if (srv_use_posix_fallocate) { + os_offset_t start_offset = start_page_no * page_size; + os_offset_t n_pages = (size_after_extend - start_page_no); + os_offset_t len = n_pages * page_size; + + if (posix_fallocate(node->handle, start_offset, len) == -1) { + ib_logf(IB_LOG_LEVEL_ERROR, "preallocating file " + "space for file \'%s\' failed. Current size " + INT64PF ", desired size " INT64PF "\n", + node->name, start_offset, len+start_offset); + os_file_handle_error_no_exit(node->name, "posix_fallocate", FALSE, __FILE__, __LINE__); + success = FALSE; + } else { + success = TRUE; + } + + mutex_enter(&fil_system->mutex); + + if (success) { + node->size += n_pages; + space->size += n_pages; + os_has_said_disk_full = FALSE; + } + + /* If posix_fallocate was used to extent the file space + we need to complete the io. Because no actual writes were + dispatched read operation is enough here. Without this + there will be assertion at shutdown indicating that + all IO is not completed. */ + fil_node_complete_io(node, fil_system, OS_FILE_READ); + goto file_extended; + } +#endif + + /* Extend at most 64 pages at a time */ + buf_size = ut_min(64, size_after_extend - start_page_no) * page_size; + buf2 = static_cast(mem_alloc(buf_size + page_size)); + buf = static_cast(ut_align(buf2, page_size)); + + memset(buf, 0, buf_size); + + while (start_page_no < size_after_extend) { + ulint n_pages + = ut_min(buf_size / page_size, + size_after_extend - start_page_no); + + os_offset_t offset + = ((os_offset_t) (start_page_no - file_start_page_no)) + * page_size; +#ifdef UNIV_HOTBACKUP + success = os_file_write(node->name, node->handle, buf, + offset, page_size * n_pages); +#else + success = os_aio(OS_FILE_WRITE, OS_AIO_SYNC, + node->name, node->handle, buf, + offset, page_size * n_pages, + NULL, NULL, space_id, NULL, 0, 0, 0, 0, 0); +#endif /* UNIV_HOTBACKUP */ + if (success) { + os_has_said_disk_full = FALSE; + } else { + /* Let us measure the size of the file to determine + how much we were able to extend it */ + os_offset_t size; + + size = os_file_get_size(node->handle); + ut_a(size != (os_offset_t) -1); + + n_pages = ((ulint) (size / page_size)) + - node->size - pages_added; + + pages_added += n_pages; + break; + } + + start_page_no += n_pages; + pages_added += n_pages; + } + + mem_free(buf2); + + mutex_enter(&fil_system->mutex); + + ut_a(node->being_extended); + + space->size += pages_added; + node->size += pages_added; + + fil_node_complete_io(node, fil_system, OS_FILE_WRITE); + + /* At this point file has been extended */ +file_extended: + + node->being_extended = FALSE; + *actual_size = space->size; + +#ifndef UNIV_HOTBACKUP + if (space_id == 0) { + ulint pages_per_mb = (1024 * 1024) / page_size; + + /* Keep the last data file size info up to date, rounded to + full megabytes */ + + srv_data_file_sizes[srv_n_data_files - 1] + = (node->size / pages_per_mb) * pages_per_mb; + } +#endif /* !UNIV_HOTBACKUP */ + + /* + printf("Extended %s to %lu, actual size %lu pages\n", space->name, + size_after_extend, *actual_size); */ + mutex_exit(&fil_system->mutex); + + fil_flush(space_id); + + return(success); +} + +#ifdef UNIV_HOTBACKUP +/********************************************************************//** +Extends all tablespaces to the size stored in the space header. During the +ibbackup --apply-log phase we extended the spaces on-demand so that log records +could be applied, but that may have left spaces still too small compared to +the size stored in the space header. */ +UNIV_INTERN +void +fil_extend_tablespaces_to_stored_len(void) +/*======================================*/ +{ + fil_space_t* space; + byte* buf; + ulint actual_size; + ulint size_in_header; + dberr_t error; + ibool success; + + buf = mem_alloc(UNIV_PAGE_SIZE); + + mutex_enter(&fil_system->mutex); + + space = UT_LIST_GET_FIRST(fil_system->space_list); + + while (space) { + ut_a(space->purpose == FIL_TABLESPACE); + + mutex_exit(&fil_system->mutex); /* no need to protect with a + mutex, because this is a + single-threaded operation */ + error = fil_read(TRUE, space->id, + fsp_flags_get_zip_size(space->flags), + 0, 0, UNIV_PAGE_SIZE, buf, NULL); + ut_a(error == DB_SUCCESS); + + size_in_header = fsp_get_size_low(buf); + + success = fil_extend_space_to_desired_size( + &actual_size, space->id, size_in_header); + if (!success) { + fprintf(stderr, + "InnoDB: Error: could not extend the" + " tablespace of %s\n" + "InnoDB: to the size stored in header," + " %lu pages;\n" + "InnoDB: size after extension %lu pages\n" + "InnoDB: Check that you have free disk space" + " and retry!\n", + space->name, size_in_header, actual_size); + ut_a(success); + } + + mutex_enter(&fil_system->mutex); + + space = UT_LIST_GET_NEXT(space_list, space); + } + + mutex_exit(&fil_system->mutex); + + mem_free(buf); +} +#endif + +/*========== RESERVE FREE EXTENTS (for a B-tree split, for example) ===*/ + +/*******************************************************************//** +Tries to reserve free extents in a file space. +@return TRUE if succeed */ +UNIV_INTERN +ibool +fil_space_reserve_free_extents( +/*===========================*/ + ulint id, /*!< in: space id */ + ulint n_free_now, /*!< in: number of free extents now */ + ulint n_to_reserve) /*!< in: how many one wants to reserve */ +{ + fil_space_t* space; + ibool success; + + ut_ad(fil_system); + + mutex_enter(&fil_system->mutex); + + space = fil_space_get_by_id(id); + + ut_a(space); + + if (space->n_reserved_extents + n_to_reserve > n_free_now) { + success = FALSE; + } else { + space->n_reserved_extents += n_to_reserve; + success = TRUE; + } + + mutex_exit(&fil_system->mutex); + + return(success); +} + +/*******************************************************************//** +Releases free extents in a file space. */ +UNIV_INTERN +void +fil_space_release_free_extents( +/*===========================*/ + ulint id, /*!< in: space id */ + ulint n_reserved) /*!< in: how many one reserved */ +{ + fil_space_t* space; + + ut_ad(fil_system); + + mutex_enter(&fil_system->mutex); + + space = fil_space_get_by_id(id); + + ut_a(space); + ut_a(space->n_reserved_extents >= n_reserved); + + space->n_reserved_extents -= n_reserved; + + mutex_exit(&fil_system->mutex); +} + +/*******************************************************************//** +Gets the number of reserved extents. If the database is silent, this number +should be zero. */ +UNIV_INTERN +ulint +fil_space_get_n_reserved_extents( +/*=============================*/ + ulint id) /*!< in: space id */ +{ + fil_space_t* space; + ulint n; + + ut_ad(fil_system); + + mutex_enter(&fil_system->mutex); + + space = fil_space_get_by_id(id); + + ut_a(space); + + n = space->n_reserved_extents; + + mutex_exit(&fil_system->mutex); + + return(n); +} + +/*============================ FILE I/O ================================*/ + +/********************************************************************//** +NOTE: you must call fil_mutex_enter_and_prepare_for_io() first! + +Prepares a file node for i/o. Opens the file if it is closed. Updates the +pending i/o's field in the node and the system appropriately. Takes the node +off the LRU list if it is in the LRU list. The caller must hold the fil_sys +mutex. +@return false if the file can't be opened, otherwise true */ +static +bool +fil_node_prepare_for_io( +/*====================*/ + fil_node_t* node, /*!< in: file node */ + fil_system_t* system, /*!< in: tablespace memory cache */ + fil_space_t* space) /*!< in: space */ +{ + ut_ad(node && system && space); + ut_ad(mutex_own(&(system->mutex))); + + if (system->n_open > system->max_n_open + 5) { + ut_print_timestamp(stderr); + fprintf(stderr, + " InnoDB: Warning: open files %lu" + " exceeds the limit %lu\n", + (ulong) system->n_open, + (ulong) system->max_n_open); + } + + if (node->open == FALSE) { + /* File is closed: open it */ + ut_a(node->n_pending == 0); + + if (!fil_node_open_file(node, system, space)) { + return(false); + } + } + + if (node->n_pending == 0 && fil_space_belongs_in_lru(space)) { + /* The node is in the LRU list, remove it */ + + ut_a(UT_LIST_GET_LEN(system->LRU) > 0); + + UT_LIST_REMOVE(LRU, system->LRU, node); + } + + node->n_pending++; + + return(true); +} + +/********************************************************************//** +Updates the data structures when an i/o operation finishes. Updates the +pending i/o's field in the node appropriately. */ +static +void +fil_node_complete_io( +/*=================*/ + fil_node_t* node, /*!< in: file node */ + fil_system_t* system, /*!< in: tablespace memory cache */ + ulint type) /*!< in: OS_FILE_WRITE or OS_FILE_READ; marks + the node as modified if + type == OS_FILE_WRITE */ +{ + ut_ad(node); + ut_ad(system); + ut_ad(mutex_own(&(system->mutex))); + + ut_a(node->n_pending > 0); + + node->n_pending--; + + if (type == OS_FILE_WRITE) { + ut_ad(!srv_read_only_mode); + system->modification_counter++; + node->modification_counter = system->modification_counter; + + if (fil_buffering_disabled(node->space)) { + + /* We don't need to keep track of unflushed + changes as user has explicitly disabled + buffering. */ + ut_ad(!node->space->is_in_unflushed_spaces); + node->flush_counter = node->modification_counter; + + } else if (!node->space->is_in_unflushed_spaces) { + + node->space->is_in_unflushed_spaces = true; + UT_LIST_ADD_FIRST(unflushed_spaces, + system->unflushed_spaces, + node->space); + } + } + + if (node->n_pending == 0 && fil_space_belongs_in_lru(node->space)) { + + /* The node must be put back to the LRU list */ + UT_LIST_ADD_FIRST(LRU, system->LRU, node); + } +} + +/********************************************************************//** +Report information about an invalid page access. */ +static +void +fil_report_invalid_page_access( +/*===========================*/ + ulint block_offset, /*!< in: block offset */ + ulint space_id, /*!< in: space id */ + const char* space_name, /*!< in: space name */ + ulint byte_offset, /*!< in: byte offset */ + ulint len, /*!< in: I/O length */ + ulint type) /*!< in: I/O type */ +{ + fprintf(stderr, + "InnoDB: Error: trying to access page number %lu" + " in space %lu,\n" + "InnoDB: space name %s,\n" + "InnoDB: which is outside the tablespace bounds.\n" + "InnoDB: Byte offset %lu, len %lu, i/o type %lu.\n" + "InnoDB: If you get this error at mysqld startup," + " please check that\n" + "InnoDB: your my.cnf matches the ibdata files" + " that you have in the\n" + "InnoDB: MySQL server.\n", + (ulong) block_offset, (ulong) space_id, space_name, + (ulong) byte_offset, (ulong) len, (ulong) type); +} + +/********************************************************************//** +Reads or writes data. This operation is asynchronous (aio). +@return DB_SUCCESS, or DB_TABLESPACE_DELETED if we are trying to do +i/o on a tablespace which does not exist */ +UNIV_INTERN +dberr_t +_fil_io( +/*===*/ + ulint type, /*!< in: OS_FILE_READ or OS_FILE_WRITE, + ORed to OS_FILE_LOG, if a log i/o + and ORed to OS_AIO_SIMULATED_WAKE_LATER + if simulated aio and we want to post a + batch of i/os; NOTE that a simulated batch + may introduce hidden chances of deadlocks, + because i/os are not actually handled until + all have been posted: use with great + caution! */ + bool sync, /*!< in: true if synchronous aio is desired */ + ulint space_id, /*!< in: space id */ + ulint zip_size, /*!< in: compressed page size in bytes; + 0 for uncompressed pages */ + ulint block_offset, /*!< in: offset in number of blocks */ + ulint byte_offset, /*!< in: remainder of offset in bytes; in + aio this must be divisible by the OS block + size */ + ulint len, /*!< in: how many bytes to read or write; this + must not cross a file boundary; in aio this + must be a block size multiple */ + void* buf, /*!< in/out: buffer where to store read data + or from where to write; in aio this must be + appropriately aligned */ + void* message, /*!< in: message for aio handler if non-sync + aio used, else ignored */ + ulint* write_size, /*!< in/out: Actual write size initialized + after fist successfull trim + operation for this page and if + initialized we do not trim again if + actual page size does not decrease. */ + trx_t* trx) +{ + ulint mode; + fil_space_t* space; + fil_node_t* node; + ibool ret; + ulint is_log; + ulint wake_later; + os_offset_t offset; + ibool ignore_nonexistent_pages; + ibool page_compressed = FALSE; + ulint page_compression_level = 0; + ibool page_encrypted = FALSE; + ulint page_encryption_key = 0; + + + is_log = type & OS_FILE_LOG; + type = type & ~OS_FILE_LOG; + + wake_later = type & OS_AIO_SIMULATED_WAKE_LATER; + type = type & ~OS_AIO_SIMULATED_WAKE_LATER; + + ignore_nonexistent_pages = type & BUF_READ_IGNORE_NONEXISTENT_PAGES; + type &= ~BUF_READ_IGNORE_NONEXISTENT_PAGES; + + ut_ad(byte_offset < UNIV_PAGE_SIZE); + ut_ad(!zip_size || !byte_offset); + ut_ad(ut_is_2pow(zip_size)); + ut_ad(buf); + ut_ad(len > 0); + ut_ad(UNIV_PAGE_SIZE == (ulong)(1 << UNIV_PAGE_SIZE_SHIFT)); +#if (1 << UNIV_PAGE_SIZE_SHIFT_MAX) != UNIV_PAGE_SIZE_MAX +# error "(1 << UNIV_PAGE_SIZE_SHIFT_MAX) != UNIV_PAGE_SIZE_MAX" +#endif +#if (1 << UNIV_PAGE_SIZE_SHIFT_MIN) != UNIV_PAGE_SIZE_MIN +# error "(1 << UNIV_PAGE_SIZE_SHIFT_MIN) != UNIV_PAGE_SIZE_MIN" +#endif + ut_ad(fil_validate_skip()); +#ifndef UNIV_HOTBACKUP +# ifndef UNIV_LOG_DEBUG + /* ibuf bitmap pages must be read in the sync aio mode: */ + ut_ad(recv_no_ibuf_operations + || type == OS_FILE_WRITE + || !ibuf_bitmap_page(zip_size, block_offset) + || sync + || is_log); +# endif /* UNIV_LOG_DEBUG */ + if (sync) { + mode = OS_AIO_SYNC; + } else if (is_log) { + mode = OS_AIO_LOG; + } else if (type == OS_FILE_READ + && !recv_no_ibuf_operations + && ibuf_page(space_id, zip_size, block_offset, NULL)) { + mode = OS_AIO_IBUF; + } else { + mode = OS_AIO_NORMAL; + } +#else /* !UNIV_HOTBACKUP */ + ut_a(sync); + mode = OS_AIO_SYNC; +#endif /* !UNIV_HOTBACKUP */ + + if (type == OS_FILE_READ) { + srv_stats.data_read.add(len); + } else if (type == OS_FILE_WRITE) { + ut_ad(!srv_read_only_mode); + srv_stats.data_written.add(len); + if (fil_page_is_index_page((byte *)buf)) { + srv_stats.index_pages_written.inc(); + } else { + srv_stats.non_index_pages_written.inc(); + } + } + + /* Reserve the fil_system mutex and make sure that we can open at + least one file while holding it, if the file is not already open */ + + fil_mutex_enter_and_prepare_for_io(space_id); + + space = fil_space_get_by_id(space_id); + + page_compressed = fsp_flags_is_page_compressed(space->flags); + page_compression_level = fsp_flags_get_page_compression_level(space->flags); + + page_encrypted = fsp_flags_is_page_encrypted(space->flags); + page_encryption_key = fsp_flags_get_page_encryption_key(space->flags); + + + /* If we are deleting a tablespace we don't allow any read + operations on that. However, we do allow write operations. */ + if (space == 0 || (type == OS_FILE_READ && space->stop_new_ops)) { + mutex_exit(&fil_system->mutex); + + ib_logf(IB_LOG_LEVEL_ERROR, + "Trying to do i/o to a tablespace which does " + "not exist. i/o type %lu, space id %lu, " + "page no. %lu, i/o length %lu bytes", + (ulong) type, (ulong) space_id, (ulong) block_offset, + (ulong) len); + + return(DB_TABLESPACE_DELETED); + } + + ut_ad(mode != OS_AIO_IBUF || space->purpose == FIL_TABLESPACE); + + node = UT_LIST_GET_FIRST(space->chain); + + for (;;) { + if (node == NULL) { + if (ignore_nonexistent_pages) { + mutex_exit(&fil_system->mutex); + return(DB_ERROR); + } + + fil_report_invalid_page_access( + block_offset, space_id, space->name, + byte_offset, len, type); + + ut_error; + + } else if (fil_is_user_tablespace_id(space->id) + && node->size == 0) { + + /* We do not know the size of a single-table tablespace + before we open the file */ + break; + } else if (node->size > block_offset) { + /* Found! */ + break; + } else { + block_offset -= node->size; + node = UT_LIST_GET_NEXT(chain, node); + } + } + + /* Open file if closed */ + if (!fil_node_prepare_for_io(node, fil_system, space)) { + if (space->purpose == FIL_TABLESPACE + && fil_is_user_tablespace_id(space->id)) { + mutex_exit(&fil_system->mutex); + + ib_logf(IB_LOG_LEVEL_ERROR, + "Trying to do i/o to a tablespace which " + "exists without .ibd data file. " + "i/o type %lu, space id %lu, page no %lu, " + "i/o length %lu bytes", + (ulong) type, (ulong) space_id, + (ulong) block_offset, (ulong) len); + + return(DB_TABLESPACE_DELETED); + } + + /* The tablespace is for log. Currently, we just assert here + to prevent handling errors along the way fil_io returns. + Also, if the log files are missing, it would be hard to + promise the server can continue running. */ + ut_a(0); + } + + /* Check that at least the start offset is within the bounds of a + single-table tablespace, including rollback tablespaces. */ + if (UNIV_UNLIKELY(node->size <= block_offset) + && space->id != 0 && space->purpose == FIL_TABLESPACE) { + + fil_report_invalid_page_access( + block_offset, space_id, space->name, byte_offset, + len, type); + + ut_error; + } + + /* Now we have made the changes in the data structures of fil_system */ + mutex_exit(&fil_system->mutex); + + /* Calculate the low 32 bits and the high 32 bits of the file offset */ + + if (!zip_size) { + offset = ((os_offset_t) block_offset << UNIV_PAGE_SIZE_SHIFT) + + byte_offset; + + ut_a(node->size - block_offset + >= ((byte_offset + len + (UNIV_PAGE_SIZE - 1)) + / UNIV_PAGE_SIZE)); + } else { + ulint zip_size_shift; + switch (zip_size) { + case 1024: zip_size_shift = 10; break; + case 2048: zip_size_shift = 11; break; + case 4096: zip_size_shift = 12; break; + case 8192: zip_size_shift = 13; break; + case 16384: zip_size_shift = 14; break; + default: ut_error; + } + offset = ((os_offset_t) block_offset << zip_size_shift) + + byte_offset; + ut_a(node->size - block_offset + >= (len + (zip_size - 1)) / zip_size); + } + + /* Do aio */ + + ut_a(byte_offset % OS_MIN_LOG_BLOCK_SIZE == 0); + ut_a((len % OS_MIN_LOG_BLOCK_SIZE) == 0); + +#ifndef UNIV_HOTBACKUP + if (UNIV_UNLIKELY(space->is_corrupt && srv_pass_corrupt_table)) { + + /* should ignore i/o for the crashed space */ + if (srv_pass_corrupt_table == 1 || + type == OS_FILE_WRITE) { + + mutex_enter(&fil_system->mutex); + fil_node_complete_io(node, fil_system, type); + mutex_exit(&fil_system->mutex); + if (mode == OS_AIO_NORMAL) { + ut_a(space->purpose == FIL_TABLESPACE); + buf_page_io_complete(static_cast + (message)); + } + } + + if (srv_pass_corrupt_table == 1 && type == OS_FILE_READ) { + + return(DB_TABLESPACE_DELETED); + + } else if (type == OS_FILE_WRITE) { + + return(DB_SUCCESS); + } + } + + /* Queue the aio request */ + ret = os_aio(type, mode | wake_later, node->name, node->handle, buf, + offset, len, node, message, space_id, trx, page_compressed, page_compression_level, write_size, page_encrypted, page_encryption_key); + +#else + /* In ibbackup do normal i/o, not aio */ + if (type == OS_FILE_READ) { + ret = os_file_read(node->handle, buf, offset, len); + } else { + ut_ad(!srv_read_only_mode); + ret = os_file_write(node->name, node->handle, buf, + offset, len); + } +#endif /* !UNIV_HOTBACKUP */ + ut_a(ret); + + if (mode == OS_AIO_SYNC) { + /* The i/o operation is already completed when we return from + os_aio: */ + + mutex_enter(&fil_system->mutex); + + fil_node_complete_io(node, fil_system, type); + + mutex_exit(&fil_system->mutex); + + ut_ad(fil_validate_skip()); + } + + return(DB_SUCCESS); +} + +#ifndef UNIV_HOTBACKUP +/**********************************************************************//** +Waits for an aio operation to complete. This function is used to write the +handler for completed requests. The aio array of pending requests is divided +into segments (see os0file.cc for more info). The thread specifies which +segment it wants to wait for. */ +UNIV_INTERN +void +fil_aio_wait( +/*=========*/ + ulint segment) /*!< in: the number of the segment in the aio + array to wait for */ +{ + ibool ret; + fil_node_t* fil_node; + void* message; + ulint type; + ulint space_id = 0; + + ut_ad(fil_validate_skip()); + + if (srv_use_native_aio) { + srv_set_io_thread_op_info(segment, "native aio handle"); +#ifdef WIN_ASYNC_IO + ret = os_aio_windows_handle( + segment, 0, &fil_node, &message, &type, &space_id); +#elif defined(LINUX_NATIVE_AIO) + ret = os_aio_linux_handle( + segment, &fil_node, &message, &type, &space_id); +#else + ut_error; + ret = 0; /* Eliminate compiler warning */ +#endif /* WIN_ASYNC_IO */ + } else { + srv_set_io_thread_op_info(segment, "simulated aio handle"); + + ret = os_aio_simulated_handle( + segment, &fil_node, &message, &type, &space_id); + } + + ut_a(ret); + if (fil_node == NULL) { + ut_ad(srv_shutdown_state == SRV_SHUTDOWN_EXIT_THREADS); + return; + } + + srv_set_io_thread_op_info(segment, "complete io for fil node"); + + mutex_enter(&fil_system->mutex); + + fil_node_complete_io(fil_node, fil_system, type); + + mutex_exit(&fil_system->mutex); + + ut_ad(fil_validate_skip()); + + /* Do the i/o handling */ + /* IMPORTANT: since i/o handling for reads will read also the insert + buffer in tablespace 0, you have to be very careful not to introduce + deadlocks in the i/o system. We keep tablespace 0 data files always + open, and use a special i/o thread to serve insert buffer requests. */ + + if (fil_node->space->purpose == FIL_TABLESPACE) { + srv_set_io_thread_op_info(segment, "complete io for buf page"); + buf_page_io_complete(static_cast(message)); + } else { + srv_set_io_thread_op_info(segment, "complete io for log"); + log_io_complete(static_cast(message)); + } +} +#endif /* UNIV_HOTBACKUP */ + +/**********************************************************************//** +Flushes to disk possible writes cached by the OS. If the space does not exist +or is being dropped, does not do anything. */ +UNIV_INTERN +void +fil_flush( +/*======*/ + ulint space_id) /*!< in: file space id (this can be a group of + log files or a tablespace of the database) */ +{ + fil_space_t* space; + fil_node_t* node; + os_file_t file; + + + mutex_enter(&fil_system->mutex); + + space = fil_space_get_by_id(space_id); + + if (!space || space->stop_new_ops) { + mutex_exit(&fil_system->mutex); + + return; + } + + if (fil_buffering_disabled(space)) { + + /* No need to flush. User has explicitly disabled + buffering. */ + ut_ad(!space->is_in_unflushed_spaces); + ut_ad(fil_space_is_flushed(space)); + ut_ad(space->n_pending_flushes == 0); + +#ifdef UNIV_DEBUG + for (node = UT_LIST_GET_FIRST(space->chain); + node != NULL; + node = UT_LIST_GET_NEXT(chain, node)) { + ut_ad(node->modification_counter + == node->flush_counter); + ut_ad(node->n_pending_flushes == 0); + } +#endif /* UNIV_DEBUG */ + + mutex_exit(&fil_system->mutex); + return; + } + + space->n_pending_flushes++; /*!< prevent dropping of the space while + we are flushing */ + for (node = UT_LIST_GET_FIRST(space->chain); + node != NULL; + node = UT_LIST_GET_NEXT(chain, node)) { + + ib_int64_t old_mod_counter = node->modification_counter;; + + if (old_mod_counter <= node->flush_counter) { + continue; + } + + ut_a(node->open); + + if (space->purpose == FIL_TABLESPACE) { + fil_n_pending_tablespace_flushes++; + } else { + fil_n_pending_log_flushes++; + fil_n_log_flushes++; + } +#ifdef __WIN__ + if (node->is_raw_disk) { + + goto skip_flush; + } +#endif /* __WIN__ */ +retry: + if (node->n_pending_flushes > 0) { + /* We want to avoid calling os_file_flush() on + the file twice at the same time, because we do + not know what bugs OS's may contain in file + i/o */ + + ib_int64_t sig_count = + os_event_reset(node->sync_event); + + mutex_exit(&fil_system->mutex); + + os_event_wait_low(node->sync_event, sig_count); + + mutex_enter(&fil_system->mutex); + + if (node->flush_counter >= old_mod_counter) { + + goto skip_flush; + } + + goto retry; + } + + ut_a(node->open); + file = node->handle; + node->n_pending_flushes++; + + mutex_exit(&fil_system->mutex); + + os_file_flush(file); + + mutex_enter(&fil_system->mutex); + + os_event_set(node->sync_event); + + node->n_pending_flushes--; +skip_flush: + if (node->flush_counter < old_mod_counter) { + node->flush_counter = old_mod_counter; + + if (space->is_in_unflushed_spaces + && fil_space_is_flushed(space)) { + + space->is_in_unflushed_spaces = false; + + UT_LIST_REMOVE( + unflushed_spaces, + fil_system->unflushed_spaces, + space); + } + } + + if (space->purpose == FIL_TABLESPACE) { + fil_n_pending_tablespace_flushes--; + } else { + fil_n_pending_log_flushes--; + } + } + + space->n_pending_flushes--; + + mutex_exit(&fil_system->mutex); +} + +/**********************************************************************//** +Flushes to disk the writes in file spaces of the given type possibly cached by +the OS. */ +UNIV_INTERN +void +fil_flush_file_spaces( +/*==================*/ + ulint purpose) /*!< in: FIL_TABLESPACE, FIL_LOG */ +{ + fil_space_t* space; + ulint* space_ids; + ulint n_space_ids; + ulint i; + + mutex_enter(&fil_system->mutex); + + n_space_ids = UT_LIST_GET_LEN(fil_system->unflushed_spaces); + if (n_space_ids == 0) { + + mutex_exit(&fil_system->mutex); + return; + } + + /* Assemble a list of space ids to flush. Previously, we + traversed fil_system->unflushed_spaces and called UT_LIST_GET_NEXT() + on a space that was just removed from the list by fil_flush(). + Thus, the space could be dropped and the memory overwritten. */ + space_ids = static_cast( + mem_alloc(n_space_ids * sizeof *space_ids)); + + n_space_ids = 0; + + for (space = UT_LIST_GET_FIRST(fil_system->unflushed_spaces); + space; + space = UT_LIST_GET_NEXT(unflushed_spaces, space)) { + + if (space->purpose == purpose && !space->stop_new_ops) { + + space_ids[n_space_ids++] = space->id; + } + } + + mutex_exit(&fil_system->mutex); + + /* Flush the spaces. It will not hurt to call fil_flush() on + a non-existing space id. */ + for (i = 0; i < n_space_ids; i++) { + + fil_flush(space_ids[i]); + } + + mem_free(space_ids); +} + +/** Functor to validate the space list. */ +struct Check { + void operator()(const fil_node_t* elem) + { + ut_a(elem->open || !elem->n_pending); + } +}; + +/******************************************************************//** +Checks the consistency of the tablespace cache. +@return TRUE if ok */ +UNIV_INTERN +ibool +fil_validate(void) +/*==============*/ +{ + fil_space_t* space; + fil_node_t* fil_node; + ulint n_open = 0; + ulint i; + + mutex_enter(&fil_system->mutex); + + /* Look for spaces in the hash table */ + + for (i = 0; i < hash_get_n_cells(fil_system->spaces); i++) { + + for (space = static_cast( + HASH_GET_FIRST(fil_system->spaces, i)); + space != 0; + space = static_cast( + HASH_GET_NEXT(hash, space))) { + + UT_LIST_VALIDATE( + chain, fil_node_t, space->chain, Check()); + + for (fil_node = UT_LIST_GET_FIRST(space->chain); + fil_node != 0; + fil_node = UT_LIST_GET_NEXT(chain, fil_node)) { + + if (fil_node->n_pending > 0) { + ut_a(fil_node->open); + } + + if (fil_node->open) { + n_open++; + } + } + } + } + + ut_a(fil_system->n_open == n_open); + + UT_LIST_CHECK(LRU, fil_node_t, fil_system->LRU); + + for (fil_node = UT_LIST_GET_FIRST(fil_system->LRU); + fil_node != 0; + fil_node = UT_LIST_GET_NEXT(LRU, fil_node)) { + + ut_a(fil_node->n_pending == 0); + ut_a(!fil_node->being_extended); + ut_a(fil_node->open); + ut_a(fil_space_belongs_in_lru(fil_node->space)); + } + + mutex_exit(&fil_system->mutex); + + return(TRUE); +} + +/********************************************************************//** +Returns TRUE if file address is undefined. +@return TRUE if undefined */ +UNIV_INTERN +ibool +fil_addr_is_null( +/*=============*/ + fil_addr_t addr) /*!< in: address */ +{ + return(addr.page == FIL_NULL); +} + +/********************************************************************//** +Get the predecessor of a file page. +@return FIL_PAGE_PREV */ +UNIV_INTERN +ulint +fil_page_get_prev( +/*==============*/ + const byte* page) /*!< in: file page */ +{ + return(mach_read_from_4(page + FIL_PAGE_PREV)); +} + +/********************************************************************//** +Get the successor of a file page. +@return FIL_PAGE_NEXT */ +UNIV_INTERN +ulint +fil_page_get_next( +/*==============*/ + const byte* page) /*!< in: file page */ +{ + return(mach_read_from_4(page + FIL_PAGE_NEXT)); +} + +/*********************************************************************//** +Sets the file page type. */ +UNIV_INTERN +void +fil_page_set_type( +/*==============*/ + byte* page, /*!< in/out: file page */ + ulint type) /*!< in: type */ +{ + ut_ad(page); + + mach_write_to_2(page + FIL_PAGE_TYPE, type); +} + +/*********************************************************************//** +Gets the file page type. +@return type; NOTE that if the type has not been written to page, the +return value not defined */ +UNIV_INTERN +ulint +fil_page_get_type( +/*==============*/ + const byte* page) /*!< in: file page */ +{ + ut_ad(page); + + return(mach_read_from_2(page + FIL_PAGE_TYPE)); +} + +/****************************************************************//** +Closes the tablespace memory cache. */ +UNIV_INTERN +void +fil_close(void) +/*===========*/ +{ +#ifndef UNIV_HOTBACKUP + /* The mutex should already have been freed. */ + ut_ad(fil_system->mutex.magic_n == 0); +#endif /* !UNIV_HOTBACKUP */ + + hash_table_free(fil_system->spaces); + + hash_table_free(fil_system->name_hash); + + ut_a(UT_LIST_GET_LEN(fil_system->LRU) == 0); + ut_a(UT_LIST_GET_LEN(fil_system->unflushed_spaces) == 0); + ut_a(UT_LIST_GET_LEN(fil_system->space_list) == 0); + + mem_free(fil_system); + + fil_system = NULL; +} + +/********************************************************************//** +Initializes a buffer control block when the buf_pool is created. */ +static +void +fil_buf_block_init( +/*===============*/ + buf_block_t* block, /*!< in: pointer to control block */ + byte* frame) /*!< in: pointer to buffer frame */ +{ + UNIV_MEM_DESC(frame, UNIV_PAGE_SIZE); + + block->frame = frame; + + block->page.io_fix = BUF_IO_NONE; + /* There are assertions that check for this. */ + block->page.buf_fix_count = 1; + block->page.state = BUF_BLOCK_READY_FOR_USE; + + page_zip_des_init(&block->page.zip); +} + +struct fil_iterator_t { + os_file_t file; /*!< File handle */ + const char* filepath; /*!< File path name */ + os_offset_t start; /*!< From where to start */ + os_offset_t end; /*!< Where to stop */ + os_offset_t file_size; /*!< File size in bytes */ + ulint page_size; /*!< Page size */ + ulint n_io_buffers; /*!< Number of pages to use + for IO */ + byte* io_buffer; /*!< Buffer to use for IO */ +}; + +/********************************************************************//** +TODO: This can be made parallel trivially by chunking up the file and creating +a callback per thread. . Main benefit will be to use multiple CPUs for +checksums and compressed tables. We have to do compressed tables block by +block right now. Secondly we need to decompress/compress and copy too much +of data. These are CPU intensive. + +Iterate over all the pages in the tablespace. +@param iter - Tablespace iterator +@param block - block to use for IO +@param callback - Callback to inspect and update page contents +@retval DB_SUCCESS or error code */ +static +dberr_t +fil_iterate( +/*========*/ + const fil_iterator_t& iter, + buf_block_t* block, + PageCallback& callback) +{ + os_offset_t offset; + ulint page_no = 0; + ulint space_id = callback.get_space_id(); + ulint n_bytes = iter.n_io_buffers * iter.page_size; + + ut_ad(!srv_read_only_mode); + + /* TODO: For compressed tables we do a lot of useless + copying for non-index pages. Unfortunately, it is + required by buf_zip_decompress() */ + + for (offset = iter.start; offset < iter.end; offset += n_bytes) { + + byte* io_buffer = iter.io_buffer; + + block->frame = io_buffer; + + if (callback.get_zip_size() > 0) { + page_zip_des_init(&block->page.zip); + page_zip_set_size(&block->page.zip, iter.page_size); + block->page.zip.data = block->frame + UNIV_PAGE_SIZE; + ut_d(block->page.zip.m_external = true); + ut_ad(iter.page_size == callback.get_zip_size()); + + /* Zip IO is done in the compressed page buffer. */ + io_buffer = block->page.zip.data; + } else { + io_buffer = iter.io_buffer; + } + + /* We have to read the exact number of bytes. Otherwise the + InnoDB IO functions croak on failed reads. */ + + n_bytes = static_cast( + ut_min(static_cast(n_bytes), + iter.end - offset)); + + ut_ad(n_bytes > 0); + ut_ad(!(n_bytes % iter.page_size)); + + if (!os_file_read(iter.file, io_buffer, offset, + (ulint) n_bytes, + fil_space_is_page_compressed(space_id))) { + + ib_logf(IB_LOG_LEVEL_ERROR, "os_file_read() failed"); + + return(DB_IO_ERROR); + } + + bool updated = false; + os_offset_t page_off = offset; + ulint n_pages_read = (ulint) n_bytes / iter.page_size; + + for (ulint i = 0; i < n_pages_read; ++i) { + + buf_block_set_file_page(block, space_id, page_no++); + + dberr_t err; + + if ((err = callback(page_off, block)) != DB_SUCCESS) { + + return(err); + + } else if (!updated) { + updated = buf_block_get_state(block) + == BUF_BLOCK_FILE_PAGE; + } + + buf_block_set_state(block, BUF_BLOCK_NOT_USED); + buf_block_set_state(block, BUF_BLOCK_READY_FOR_USE); + + page_off += iter.page_size; + block->frame += iter.page_size; + } + + /* A page was updated in the set, write back to disk. */ + if (updated + && !os_file_write( + iter.filepath, iter.file, io_buffer, + offset, (ulint) n_bytes)) { + + ib_logf(IB_LOG_LEVEL_ERROR, "os_file_write() failed"); + + return(DB_IO_ERROR); + } + } + + return(DB_SUCCESS); +} + +/********************************************************************//** +Iterate over all the pages in the tablespace. +@param table - the table definiton in the server +@param n_io_buffers - number of blocks to read and write together +@param callback - functor that will do the page updates +@return DB_SUCCESS or error code */ +UNIV_INTERN +dberr_t +fil_tablespace_iterate( +/*===================*/ + dict_table_t* table, + ulint n_io_buffers, + PageCallback& callback) +{ + dberr_t err; + os_file_t file; + char* filepath; + + ut_a(n_io_buffers > 0); + ut_ad(!srv_read_only_mode); + + DBUG_EXECUTE_IF("ib_import_trigger_corruption_1", + return(DB_CORRUPTION);); + + if (DICT_TF_HAS_DATA_DIR(table->flags)) { + dict_get_and_save_data_dir_path(table, false); + ut_a(table->data_dir_path); + + filepath = os_file_make_remote_pathname( + table->data_dir_path, table->name, "ibd"); + } else { + filepath = fil_make_ibd_name(table->name, false); + } + + { + ibool success; + + file = os_file_create_simple_no_error_handling( + innodb_file_data_key, filepath, + OS_FILE_OPEN, OS_FILE_READ_WRITE, &success, FALSE); + + DBUG_EXECUTE_IF("fil_tablespace_iterate_failure", + { + static bool once; + + if (!once || ut_rnd_interval(0, 10) == 5) { + once = true; + success = FALSE; + os_file_close(file); + } + }); + + if (!success) { + /* The following call prints an error message */ + os_file_get_last_error(true); + + ib_logf(IB_LOG_LEVEL_ERROR, + "Trying to import a tablespace, but could not " + "open the tablespace file %s", filepath); + + mem_free(filepath); + + return(DB_TABLESPACE_NOT_FOUND); + + } else { + err = DB_SUCCESS; + } + } + + callback.set_file(filepath, file); + + os_offset_t file_size = os_file_get_size(file); + ut_a(file_size != (os_offset_t) -1); + + /* The block we will use for every physical page */ + buf_block_t block; + + memset(&block, 0x0, sizeof(block)); + + /* Allocate a page to read in the tablespace header, so that we + can determine the page size and zip_size (if it is compressed). + We allocate an extra page in case it is a compressed table. One + page is to ensure alignement. */ + + void* page_ptr = mem_alloc(3 * UNIV_PAGE_SIZE); + byte* page = static_cast(ut_align(page_ptr, UNIV_PAGE_SIZE)); + + fil_buf_block_init(&block, page); + + /* Read the first page and determine the page and zip size. */ + + if (!os_file_read(file, page, 0, UNIV_PAGE_SIZE, + dict_tf_get_page_compression(table->flags))) { + + err = DB_IO_ERROR; + + } else if ((err = callback.init(file_size, &block)) == DB_SUCCESS) { + fil_iterator_t iter; + + iter.file = file; + iter.start = 0; + iter.end = file_size; + iter.filepath = filepath; + iter.file_size = file_size; + iter.n_io_buffers = n_io_buffers; + iter.page_size = callback.get_page_size(); + + /* Compressed pages can't be optimised for block IO for now. + We do the IMPORT page by page. */ + + if (callback.get_zip_size() > 0) { + iter.n_io_buffers = 1; + ut_a(iter.page_size == callback.get_zip_size()); + } + + /** Add an extra page for compressed page scratch area. */ + + void* io_buffer = mem_alloc( + (2 + iter.n_io_buffers) * UNIV_PAGE_SIZE); + + iter.io_buffer = static_cast( + ut_align(io_buffer, UNIV_PAGE_SIZE)); + + err = fil_iterate(iter, &block, callback); + + mem_free(io_buffer); + } + + if (err == DB_SUCCESS) { + + ib_logf(IB_LOG_LEVEL_INFO, "Sync to disk"); + + if (!os_file_flush(file)) { + ib_logf(IB_LOG_LEVEL_INFO, "os_file_flush() failed!"); + err = DB_IO_ERROR; + } else { + ib_logf(IB_LOG_LEVEL_INFO, "Sync to disk - done!"); + } + } + + os_file_close(file); + + mem_free(page_ptr); + mem_free(filepath); + + return(err); +} + +/** +Set the tablespace compressed table size. +@return DB_SUCCESS if it is valie or DB_CORRUPTION if not */ +dberr_t +PageCallback::set_zip_size(const buf_frame_t* page) UNIV_NOTHROW +{ + m_zip_size = fsp_header_get_zip_size(page); + + if (!ut_is_2pow(m_zip_size) || m_zip_size > UNIV_ZIP_SIZE_MAX) { + return(DB_CORRUPTION); + } + + return(DB_SUCCESS); +} + +/********************************************************************//** +Delete the tablespace file and any related files like .cfg. +This should not be called for temporary tables. */ +UNIV_INTERN +void +fil_delete_file( +/*============*/ + const char* ibd_name) /*!< in: filepath of the ibd + tablespace */ +{ + /* Force a delete of any stale .ibd files that are lying around. */ + + ib_logf(IB_LOG_LEVEL_INFO, "Deleting %s", ibd_name); + + os_file_delete_if_exists(innodb_file_data_key, ibd_name); + + char* cfg_name = fil_make_cfg_name(ibd_name); + + os_file_delete_if_exists(innodb_file_data_key, cfg_name); + + mem_free(cfg_name); +} + +/************************************************************************* +Return local hash table informations. */ + +ulint +fil_system_hash_cells(void) +/*=======================*/ +{ + if (fil_system) { + return (fil_system->spaces->n_cells + + fil_system->name_hash->n_cells); + } else { + return 0; + } +} + +ulint +fil_system_hash_nodes(void) +/*=======================*/ +{ + if (fil_system) { + return (UT_LIST_GET_LEN(fil_system->space_list) + * (sizeof(fil_space_t) + MEM_BLOCK_HEADER_SIZE)); + } else { + return 0; + } +} + +/** +Iterate over all the spaces in the space list and fetch the +tablespace names. It will return a copy of the name that must be +freed by the caller using: delete[]. +@return DB_SUCCESS if all OK. */ +UNIV_INTERN +dberr_t +fil_get_space_names( +/*================*/ + space_name_list_t& space_name_list) + /*!< in/out: List to append to */ +{ + fil_space_t* space; + dberr_t err = DB_SUCCESS; + + mutex_enter(&fil_system->mutex); + + for (space = UT_LIST_GET_FIRST(fil_system->space_list); + space != NULL; + space = UT_LIST_GET_NEXT(space_list, space)) { + + if (space->purpose == FIL_TABLESPACE) { + ulint len; + char* name; + + len = strlen(space->name); + name = new(std::nothrow) char[len + 1]; + + if (name == 0) { + /* Caller to free elements allocated so far. */ + err = DB_OUT_OF_MEMORY; + break; + } + + memcpy(name, space->name, len); + name[len] = 0; + + space_name_list.push_back(name); + } + } + + mutex_exit(&fil_system->mutex); + + return(err); +} + +/****************************************************************//** +Generate redo logs for swapping two .ibd files */ +UNIV_INTERN +void +fil_mtr_rename_log( +/*===============*/ + ulint old_space_id, /*!< in: tablespace id of the old + table. */ + const char* old_name, /*!< in: old table name */ + ulint new_space_id, /*!< in: tablespace id of the new + table */ + const char* new_name, /*!< in: new table name */ + const char* tmp_name, /*!< in: temp table name used while + swapping */ + mtr_t* mtr) /*!< in/out: mini-transaction */ +{ + if (old_space_id != TRX_SYS_SPACE) { + fil_op_write_log(MLOG_FILE_RENAME, old_space_id, + 0, 0, old_name, tmp_name, mtr); + } + + if (new_space_id != TRX_SYS_SPACE) { + fil_op_write_log(MLOG_FILE_RENAME, new_space_id, + 0, 0, new_name, old_name, mtr); + } +} + +/************************************************************************* +functions to access is_corrupt flag of fil_space_t*/ + +ibool +fil_space_is_corrupt( +/*=================*/ + ulint space_id) +{ + fil_space_t* space; + ibool ret = FALSE; + + mutex_enter(&fil_system->mutex); + + space = fil_space_get_by_id(space_id); + + if (UNIV_UNLIKELY(space && space->is_corrupt)) { + ret = TRUE; + } + + mutex_exit(&fil_system->mutex); + + return(ret); +} + +void +fil_space_set_corrupt( +/*==================*/ + ulint space_id) +{ + fil_space_t* space; + + mutex_enter(&fil_system->mutex); + + space = fil_space_get_by_id(space_id); + + if (space) { + space->is_corrupt = TRUE; + } + + mutex_exit(&fil_system->mutex); +} + +/****************************************************************//** +Acquire fil_system mutex */ +void +fil_system_enter(void) +/*==================*/ +{ + ut_ad(!mutex_own(&fil_system->mutex)); + mutex_enter(&fil_system->mutex); +} + +/****************************************************************//** +Release fil_system mutex */ +void +fil_system_exit(void) +/*=================*/ +{ + ut_ad(mutex_own(&fil_system->mutex)); + mutex_exit(&fil_system->mutex); +} + +/*******************************************************************//** +Return space name */ +char* +fil_space_name( +/*===========*/ + fil_space_t* space) /*!< in: space */ +{ + return (space->name); +} + +/*******************************************************************//** +Return page type name */ +const char* +fil_get_page_type_name( +/*===================*/ + ulint page_type) /*!< in: FIL_PAGE_TYPE */ +{ + switch(page_type) { + case FIL_PAGE_PAGE_COMPRESSED: + return "PAGE_COMPRESSED"; + case FIL_PAGE_INDEX: + return "INDEX"; + case FIL_PAGE_UNDO_LOG: + return "UNDO LOG"; + case FIL_PAGE_INODE: + return "INODE"; + case FIL_PAGE_IBUF_FREE_LIST: + return "IBUF_FREE_LIST"; + case FIL_PAGE_TYPE_ALLOCATED: + return "ALLOCATED"; + case FIL_PAGE_IBUF_BITMAP: + return "IBUF_BITMAP"; + case FIL_PAGE_TYPE_SYS: + return "SYS"; + case FIL_PAGE_TYPE_TRX_SYS: + return "TRX_SYS"; + case FIL_PAGE_TYPE_FSP_HDR: + return "FSP_HDR"; + case FIL_PAGE_TYPE_XDES: + return "XDES"; + case FIL_PAGE_TYPE_BLOB: + return "BLOB"; + case FIL_PAGE_TYPE_ZBLOB: + return "ZBLOB"; + case FIL_PAGE_TYPE_ZBLOB2: + return "ZBLOB2"; + case FIL_PAGE_TYPE_COMPRESSED: + return "ORACLE PAGE COMPRESSED"; + default: + return "PAGE TYPE CORRUPTED"; + } +} diff --git a/storage/xtradb/fil/fil0pageencryption.cc b/storage/xtradb/fil/fil0pageencryption.cc new file mode 100644 index 0000000000000..b062098afa94f --- /dev/null +++ b/storage/xtradb/fil/fil0pageencryption.cc @@ -0,0 +1,537 @@ +/***************************************************************************** + +Copyright (C) 2013, 2014, SkySQL Ab. All Rights Reserved. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*****************************************************************************/ + +/******************************************************************//** +@file fil/fil0pageencryption.cc +Implementation for page encryption file spaces. + +Created 08/25/2014 Florin Fugaciu +***********************************************************************/ + +#include "fil0fil.h" +#include "fil0pageencryption.h" +#include "my_dbug.h" + +#include "buf0checksum.h" + +#include +#include + + +//#define UNIV_PAGEENCRIPTION_DEBUG +//#define CRYPT_FF + +/* calculate a checksum to verify decryption */ +ulint fil_page_encryption_calc_checksum( + unsigned char* buf, size_t size) { + ulint checksum = 0; + checksum = ut_fold_binary(buf, + size); + checksum = checksum & 0xFFFFFFFFUL; + return checksum; +} + +/****************************************************************//** +For page encrypted pages encrypt the page before actual write +operation. +@return encrypted page to be written*/ +byte* +fil_encrypt_page( +/*==============*/ + ulint space_id, /*!< in: tablespace id of the table. */ + byte* buf, /*!< in: buffer from which to write; in aio + this must be appropriately aligned */ + byte* out_buf, /*!< out: encrypted buffer */ + ulint len, /*!< in: length of input buffer.*/ + ulint encryption_key,/*!< in: encryption key */ + ulint* out_len, /*!< out: actual length of encrypted page */ + ulint unit_test + ) +{ + + int err = AES_OK; + int key = 0; + ulint header_len = FIL_PAGE_DATA; + ulint remainder = 0; + ulint data_size = 0; + ulint page_size = UNIV_PAGE_SIZE; + ulint orig_page_type=0; + ulint write_size = 0; + ib_uint64_t flush_lsn = 0; + ib_uint32_t checksum = 0; + ulint page_compressed = 0; + ulint offset_ctrl_data = 0; + fil_space_t* space = NULL; + byte* tmp_buf; + + + ut_ad(buf); ut_ad(out_buf); + key = encryption_key; + + //TODO encryption default key + /* If no encryption key was provided to this table, use system + default key + if (key == 0) { + key = 0; + }*/ + + + if (!unit_test) { + ut_ad(fil_space_is_page_encrypted(space_id)); + fil_system_enter(); + space = fil_space_get_by_id(space_id); + fil_system_exit(); + + +#ifdef UNIV_DEBUG + fprintf(stderr, + "InnoDB: Note: Preparing for encryption for space %lu name %s len %lu\n", + space_id, fil_space_name(space), len); +#endif /* UNIV_DEBUG */ + + } + + /* data_size -1 bytes will be encrypted at first. + * data_size is the length of the cipher text.*/ + data_size = ((page_size - header_len - FIL_PAGE_DATA_END) / 16) * 16; + + /* following number of bytes are not encrypted at first */ + remainder = (page_size - header_len - FIL_PAGE_DATA_END) - (data_size - 1); + + /* read bytes 1..5 of FIL_PAGE_FILE_FLUSH_LSN */ + flush_lsn = ut_ull_create(mach_read_from_3(buf + FIL_PAGE_FILE_FLUSH_LSN), mach_read_from_2(buf + FIL_PAGE_FILE_FLUSH_LSN + 3)); + + /* read original page type */ + orig_page_type = mach_read_from_2(buf + FIL_PAGE_TYPE); + + if (flush_lsn!=0x0) { + /** currently no support because FIL_PAGE_FILE_FLUSH_LSN field is used to store control data */ + fprintf(stderr, + "InnoDB: Warning: Encryption failed for space %lu name %s. Unsupported or unknown page type.\n", + space_id, fil_space_name(space), len, err, write_size); + + srv_stats.pages_page_encryption_error.inc(); + *out_len = len; + + return (buf); + } + + /* calculate a checksum, can be used to verify decryption */ + checksum = fil_page_encryption_calc_checksum(buf + header_len, page_size - (FIL_PAGE_DATA_END + header_len)); + + + char *stringkey = "BDE472A295675CA92E0467EADBC0E023"; + uint8 key_len = 16; + char *stringiv = "2D1AF8D3974E0BD3EFED5A6F82594F5E"; + uint8 iv_len = 16; + + + //AES_KEY aeskey; + //AES_set_encrypt_key(hexkey, key_len*8, &aeskey); + //AES_cbc_encrypt((uchar*)buf +header_len + offset, (uchar *)out_buf + header_len, data_size-offset, &aeskey, iv, AES_ENCRYPT); + write_size = data_size; + /* + err = my_aes_encrypt_cbc((char*)buf + header_len + offset, + data_size - offset, + (char *) out_buf + header_len, + &write_size, + stringkey, + key_len, + stringiv, + iv_len); + +*/ + + /* 1st encryption: data_size -1 bytes starting from FIL_PAGE_DATA */ + err = my_aes_encrypt_cbc((char*)buf +header_len, + data_size-1, + (char *)out_buf + header_len, + &write_size, + stringkey, + key_len, + stringiv, + iv_len);; + + ut_ad(write_size == data_size); + + + /* copy remaining bytes to output buffer */ + memcpy(out_buf + header_len + data_size, buf + header_len + data_size -1 , remainder); + + + /* create temporary buffer for 2nd encryption */ + tmp_buf = static_cast(ut_malloc(64)); + + /* 2nd encryption: 63 bytes from out_buf, result length is 64 bytes */ + err = my_aes_encrypt_cbc((char*)out_buf + page_size -FIL_PAGE_DATA_END -62, + 63, + (char*)tmp_buf, + &write_size, + stringkey, + key_len, + stringiv, + iv_len); + ut_ad(write_size == 64); + //AES_cbc_encrypt((uchar*)out_buf + page_size -FIL_PAGE_DATA_END -62, tmp_buf, 63, &aeskey, iv, AES_ENCRYPT); + /* copy 62 bytes from 2nd encryption to out_buf, last 2 bytes are copied later to a header field*/ + memcpy(out_buf + page_size - FIL_PAGE_DATA_END -62, tmp_buf, 62); + + /* error handling */ + if (err != AES_OK) { + /* If error we leave the actual page as it was */ + + fprintf(stderr, + "InnoDB: Warning: Encryption failed for space %lu name %s len %lu rt %d write %lu\n", + space_id, fil_space_name(space), len, err, data_size); + fflush(stderr); + srv_stats.pages_page_encryption_error.inc(); + *out_len = len; + + /* free temporary buffer */ + ut_free(tmp_buf); + + + return (buf); + } + + + /* Set up the page header. Copied from input buffer*/ + memcpy(out_buf, buf, FIL_PAGE_DATA); + + + /* Set up the checksum. This is only usable to verify decryption */ + mach_write_to_4(out_buf + FIL_PAGE_SPACE_OR_CHKSUM, checksum); + + + + /* Set up the correct page type */ + mach_write_to_2(out_buf + FIL_PAGE_TYPE, FIL_PAGE_PAGE_ENCRYPTED); + + + /* set up the trailer. The old style checksum is not used because + * in case of a compressed page, this data may be compressed or unused! */ + memcpy(out_buf + (page_size -FIL_PAGE_DATA_END), buf + (page_size - FIL_PAGE_DATA_END), FIL_PAGE_DATA_END); + + + offset_ctrl_data = page_size - FIL_PAGE_DATA_END - FIL_PAGE_FILE_FLUSH_LSN; + + + /* Set up the encryption key. Written to the 1st byte of FIL_PAGE_FILE_FLUSH_LSN */ + mach_write_to_1(out_buf+ page_size-FIL_PAGE_DATA_END - offset_ctrl_data, key); + + + /* store original page type. Written to 2nd and 3rd byte of the FIL_PAGE_FILE_FLUSH_LSN */ + mach_write_to_2(out_buf+page_size-FIL_PAGE_DATA_END +1 - offset_ctrl_data, orig_page_type); + + /* write remaining bytes to FIL_PAGE_FILE_FLUSH_LSN byte 4 and 5 */ + memcpy(out_buf+ page_size -FIL_PAGE_DATA_END + 3 - offset_ctrl_data, tmp_buf+ 62, 2); + + + + +#ifdef UNIV_DEBUG + /* Verify */ + ut_ad(fil_page_is_encrypted(out_buf)); + + //ut_ad(mach_read_from_8(out_buf+FIL_PAGE_FILE_FLUSH_LSN) == FIL_PAGE_COMPRESSION_ZLIB); + +#endif /* UNIV_DEBUG */ + + + srv_stats.pages_page_encrypted.inc(); + *out_len = page_size; + + /* free temporary buffer */ + ut_free(tmp_buf); + + return (out_buf); +} + + + +/****************************************************************//** +For page encrypted pages decrypt the page after actual read +operation. +@return decrypted page */ +void +fil_decrypt_page( +/*================*/ + byte* page_buf, /*!< in: preallocated buffer or NULL */ + byte* buf, /*!< in/out: buffer from which to read; in aio + this must be appropriately aligned */ + ulint len, /*!< in: length of output buffer.*/ + ulint* write_size, /*!< in/out: Actual payload size of the decrypted data. */ + ulint unit_test) +{ + int err = AES_OK; + ulint page_encryption_key; + + ulint data_size = 0; + ulint page_size = UNIV_PAGE_SIZE; + ulint orig_page_type=0; + ulint remaining_bytes = 0; + + ulint header_len = FIL_PAGE_DATA; + ulint remainder = 0; + ulint offset = 1; + ulint offset_ctrl_data = 0; + ulint flush_lsn = 0; + ulint page_compressed = 0; + ulint checksum = 0; + ulint stored_checksum = 0; + ulint tmp_write_size = 0; + + byte * in_buf; + byte * tmp_buf; + byte * tmp_page_buf; + + ut_ad(buf); ut_ad(len); + + /* Before actual decrypt, make sure that page type is correct */ + + if (mach_read_from_2(buf+FIL_PAGE_TYPE) != FIL_PAGE_PAGE_ENCRYPTED) { + fprintf(stderr, + "InnoDB: Corruption: We try to decrypt corrupted page\n" + "InnoDB: CRC %lu type %lu.\n" + "InnoDB: len %lu\n", + mach_read_from_4(buf + FIL_PAGE_SPACE_OR_CHKSUM), + mach_read_from_2(buf + FIL_PAGE_TYPE), len); + + fflush(stderr); + ut_error + ; + } + + /* Get flush lsn bytes 1..3 to determine, whether the page is PAGE_COMPRESSED and PAGE_ENCRYPTION */ + + + + offset_ctrl_data = page_size - FIL_PAGE_DATA_END - FIL_PAGE_FILE_FLUSH_LSN; + + /* Get encryption key */ + page_encryption_key = mach_read_from_1(buf + page_size -FIL_PAGE_DATA_END - offset_ctrl_data); + + /* Get the page type */ + orig_page_type = mach_read_from_2(buf + page_size -FIL_PAGE_DATA_END + 1 - offset_ctrl_data); + + + if (FIL_PAGE_PAGE_COMPRESSED==orig_page_type) { + page_compressed = 1; + } + +// If no buffer was given, we need to allocate temporal buffer + if (NULL == page_buf) { +#ifdef UNIV_PAGEENCRIPTION_DEBUG + fprintf(stderr, + "InnoDB: FIL: Note: Decryption buffer not given, allocating...\n"); + fflush(stderr); +#endif /* UNIV_PAGEENCRIPTION_DEBUG */ + in_buf = static_cast(ut_malloc(len+16)); + //in_buf = static_cast(ut_malloc(UNIV_PAGE_SIZE)); + } else { + in_buf = page_buf; + } + + + data_size = ((page_size - header_len - FIL_PAGE_DATA_END) / 16) * 16; + remainder = (page_size - header_len - FIL_PAGE_DATA_END) - (data_size - 1); + + + + +#ifdef UNIV_PAGEENCRIPTION_DEBUG + fprintf(stderr, + "InnoDB: Note: Preparing for decrypt for len %lu\n", actual_size); + fflush(stderr); +#endif /* UNIV_PAGEENCRIPTION_DEBUG */ + + + + tmp_buf= static_cast(ut_malloc(64)); + tmp_page_buf = static_cast(ut_malloc(page_size)); + memset(tmp_page_buf,0, page_size); + char *stringkey = "BDE472A295675CA92E0467EADBC0E023"; + uint8 key_len = 16; + + char *stringiv = "2D1AF8D3974E0BD3EFED5A6F82594F5E"; + uint8 iv_len = 16; + + /* 1st decryption: 64 bytes */ + /* 62 bytes from data area and 2 bytes from header are copied to temporary buffer */ + memcpy(tmp_buf, buf + page_size - FIL_PAGE_DATA_END -62, 62); + memcpy(tmp_buf + 62, buf + FIL_PAGE_FILE_FLUSH_LSN +3, 2); + err = my_aes_decrypt_cbc((const char*) tmp_buf, + 64, + (char *) tmp_page_buf + page_size -FIL_PAGE_DATA_END -62, + &tmp_write_size, + stringkey, + key_len, + stringiv, + iv_len + ); + ut_ad(tmp_write_size == 63); + + /* copy 1st part from buf to tmp_page_buf */ + /* do not override result of 1st decryption */ + memcpy(tmp_page_buf + FIL_PAGE_DATA, buf + FIL_PAGE_DATA, data_size - 60); + memset(in_buf, 0, page_size); + + + + err = my_aes_decrypt_cbc((char*) tmp_page_buf + FIL_PAGE_DATA, + data_size, + (char *) in_buf + FIL_PAGE_DATA, + &tmp_write_size, + stringkey, + key_len, + stringiv, + iv_len + ); + ut_ad(tmp_write_size = data_size-1); + memcpy(in_buf + FIL_PAGE_DATA + data_size -1, tmp_page_buf + page_size - FIL_PAGE_DATA_END - 2, remainder); + + + /* calculate a checksum to verify decryption*/ + checksum = fil_page_encryption_calc_checksum(in_buf + header_len, page_size - (FIL_PAGE_DATA_END + header_len) ); + /* compare with stored checksum */ + stored_checksum = mach_read_from_4(buf); + + ut_free(tmp_page_buf); + ut_free(tmp_buf); + + if (checksum != stored_checksum) { + err = PAGE_ENCRYPTION_WRONG_KEY; + } + + /* If decrypt fails it means that page is corrupted or has an unknown key */ + if (err != AES_OK) { + fprintf(stderr, "InnoDB: Corruption: Page is marked as encrypted\n" + "InnoDB: but decrypt failed with error %d.\n" + "InnoDB: size %lu len %lu, key%d\n", err, data_size, + len, page_encryption_key); + fflush(stderr); + ut_error; + } + +#ifdef UNIV_PAGEENCRIPTION_DEBUG + fprintf(stderr, "InnoDB: Note: Decryption succeeded for len %lu\n", len); + fflush(stderr); +#endif /* UNIV_PAGEENCRIPTION_DEBUG */ + + + + /* copy header */ + memcpy(in_buf, buf, FIL_PAGE_DATA); + + /* copy trailer */ + memcpy(in_buf + page_size - FIL_PAGE_DATA_END, buf + page_size - FIL_PAGE_DATA_END, FIL_PAGE_DATA_END); + + /* Copy the decrypted page to the buffer pool*/ + memcpy(buf , in_buf, page_size); + + /* setting original page type */ + + mach_write_to_2(buf + FIL_PAGE_TYPE , orig_page_type); + + + /* fill flush lsn bytes 1..5 with 0 because this header field was used to store control data + */ + mach_write_to_4(buf+FIL_PAGE_FILE_FLUSH_LSN, 0); + mach_write_to_1(buf+FIL_PAGE_FILE_FLUSH_LSN + 4, 0); + + + /* calc check sums and write to the buffer, if page was not compressed */ + if (!page_compressed) { + do_check_sum(page_size, buf); + } else { + /* page_compression uses BUF_NO_CHECKSUM_MAGIC as checksum */ + mach_write_to_4(buf+FIL_PAGE_SPACE_OR_CHKSUM, BUF_NO_CHECKSUM_MAGIC); + + } + + + // Need to free temporal buffer if no buffer was given + if (NULL == page_buf) { + ut_free(in_buf); + } + + srv_stats.pages_page_decrypted.inc(); + +} + + + + + +int summef(int a, int b) +{ + return a + b; +} + +int summef2(int a, int b) +{ + int s = (a + b)^2; + s = (a + b)*2; + return s; +} + +int multiplikation(int a, int b) +{ + return a * b; +} + + + + +void do_check_sum( ulint page_size, byte* buf) { + ib_uint32_t checksum; + /* recalculate check sum - from buf0flu.cc*/ + switch ((srv_checksum_algorithm_t) srv_checksum_algorithm) { + case SRV_CHECKSUM_ALGORITHM_CRC32: + case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32: + checksum = buf_calc_page_crc32(buf); + break; + case SRV_CHECKSUM_ALGORITHM_INNODB: + case SRV_CHECKSUM_ALGORITHM_STRICT_INNODB: + checksum = (ib_uint32_t) buf_calc_page_new_checksum(buf); + break; + case SRV_CHECKSUM_ALGORITHM_NONE: + case SRV_CHECKSUM_ALGORITHM_STRICT_NONE: + checksum = BUF_NO_CHECKSUM_MAGIC; + break; + /* no default so the compiler will emit a warning if new enum + is added and not handled here */ + } + mach_write_to_4(buf + FIL_PAGE_SPACE_OR_CHKSUM, checksum); + /* We overwrite the first 4 bytes of the end lsn field to store + the old formula checksum. Since it depends also on the field + FIL_PAGE_SPACE_OR_CHKSUM, it has to be calculated after storing the + new formula checksum. */ + if (srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB + || srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_INNODB) { + checksum = (ib_uint32_t) (buf_calc_page_old_checksum(buf)); + /* In other cases we use the value assigned from above. + If CRC32 is used then it is faster to use that checksum + (calculated above) instead of calculating another one. + We can afford to store something other than + buf_calc_page_old_checksum() or BUF_NO_CHECKSUM_MAGIC in + this field because the file will not be readable by old + versions of MySQL/InnoDB anyway (older than MySQL 5.6.3) */ + } + mach_write_to_4(buf + page_size - FIL_PAGE_END_LSN_OLD_CHKSUM, checksum); +} diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 2a551be0284da..7fc525717a2fd 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -620,6 +620,12 @@ ha_create_table_option innodb_table_option_list[]= HA_TOPTION_NUMBER("PAGE_COMPRESSION_LEVEL", page_compression_level, ULINT_UNDEFINED, 0, 9, 1), /* With this option user can enable atomic writes feature for this table */ HA_TOPTION_ENUM("ATOMIC_WRITES", atomic_writes, "DEFAULT,ON,OFF", 0), + /* With this option the user can enable page encryption for the table */ + HA_TOPTION_BOOL("PAGE_ENCRYPTION", page_encryption, 0), + + /* With this option the user defines the key identifier using for the encryption */ + HA_TOPTION_NUMBER("PAGE_ENCRYPTION_KEY", page_encryption_key, ULINT_UNDEFINED, 1, 255, 1), + HA_TOPTION_END }; @@ -976,6 +982,14 @@ static SHOW_VAR innodb_status_variables[]= { (char*) &export_vars.innodb_page_compressed_trim_op_saved, SHOW_LONGLONG}, {"num_pages_page_decompressed", (char*) &export_vars.innodb_pages_page_decompressed, SHOW_LONGLONG}, + {"num_pages_page_compression_error", + (char*) &export_vars.innodb_pages_page_compression_error, SHOW_LONGLONG}, + {"num_pages_page_encrypted", + (char*) &export_vars.innodb_pages_page_encrypted, SHOW_LONGLONG}, + {"num_pages_page_decrypted", + (char*) &export_vars.innodb_pages_page_decrypted, SHOW_LONGLONG}, + {"num_pages_page_encryption_error", + (char*) &export_vars.innodb_pages_page_encryption_error, SHOW_LONGLONG}, {"have_lz4", (char*) &innodb_have_lz4, SHOW_BOOL}, {"have_lzo", @@ -11360,6 +11374,8 @@ innobase_table_flags( modified by another thread while the table is being created. */ const ulint default_compression_level = page_zip_level; + const ulint default_encryption_key = 1; + *flags = 0; *flags2 = 0; @@ -11558,9 +11574,11 @@ innobase_table_flags( options->page_compressed, (ulint)options->page_compression_level == ULINT_UNDEFINED ? default_compression_level : options->page_compression_level, - options->atomic_writes); - - if (create_info->options & HA_LEX_CREATE_TMP_TABLE) { + options->atomic_writes, + options->page_encryption, + (ulint)options->page_encryption_key == ULINT_UNDEFINED ? + default_encryption_key : options->page_encryption_key); + if (create_info->options & HA_LEX_CREATE_TMP_TABLE) { *flags2 |= DICT_TF2_TEMPORARY; } @@ -11594,6 +11612,24 @@ ha_innobase::check_table_options( enum row_type row_format = table->s->row_type;; ha_table_option_struct *options= table->s->option_struct; atomic_writes_t awrites = (atomic_writes_t)options->atomic_writes; + if (options->page_encryption) { + if (!use_tablespace) { + push_warning( + thd, Sql_condition::WARN_LEVEL_WARN, + HA_WRONG_CREATE_OPTION, + "InnoDB: PAGE_ENCRYPTION requires" + " innodb_file_per_table."); + return "PAGE_ENCRYPTION"; + } + if (create_info->key_block_size) { + push_warning( + thd, Sql_condition::WARN_LEVEL_WARN, + HA_WRONG_CREATE_OPTION, + "InnoDB: PAGE_ENCRYPTION table can't have" + " key_block_size"); + return "PAGE_ENCRYPTION"; + } + } /* Check page compression requirements */ if (options->page_compressed) { @@ -11658,6 +11694,28 @@ ha_innobase::check_table_options( } } + if ((ulint)options->page_encryption_key != ULINT_UNDEFINED) { + if (options->page_encryption == false) { + push_warning( + thd, Sql_condition::WARN_LEVEL_WARN, + HA_WRONG_CREATE_OPTION, + "InnoDB: PAGE_ENCRYPTION_KEY requires" + " PAGE_ENCRYPTION"); + return "PAGE_ENCRYPTION_KEY"; + } + + if (options->page_encryption_key < 1 || options->page_encryption_key > 255) { + push_warning_printf( + thd, Sql_condition::WARN_LEVEL_WARN, + HA_WRONG_CREATE_OPTION, + "InnoDB: invalid PAGE_ENCRYPTION_KEY = %lu." + " Valid values are [1..255]", + options->page_encryption_key); + return "PAGE_ENCRYPTION_KEY"; + } + } + + /* Check atomic writes requirements */ if (awrites == ATOMIC_WRITES_ON || (awrites == ATOMIC_WRITES_DEFAULT && srv_use_atomic_writes)) { diff --git a/storage/xtradb/handler/ha_innodb.h b/storage/xtradb/handler/ha_innodb.h index d6a1b738fe777..ea752b312d831 100644 --- a/storage/xtradb/handler/ha_innodb.h +++ b/storage/xtradb/handler/ha_innodb.h @@ -71,6 +71,8 @@ struct ha_table_option_struct srv_use_atomic_writes=1. Atomic writes are not used if value OFF.*/ + bool page_encryption; /*!< Flag for an encrypted table */ + int page_encryption_key; /*!< ID of the encryption key */ }; /** The class defining a handle to an Innodb table */ diff --git a/storage/xtradb/include/dict0dict.h b/storage/xtradb/include/dict0dict.h index 52ac5eee86b27..84d6aa4fb9b94 100644 --- a/storage/xtradb/include/dict0dict.h +++ b/storage/xtradb/include/dict0dict.h @@ -915,8 +915,10 @@ dict_tf_set( pages */ ulint page_compression_level, /*!< in: table page compression level */ - ulint atomic_writes) /*!< in: table atomic + ulint atomic_writes, /*!< in: table atomic writes option value*/ + bool page_encrypted,/*!< in: table uses page encryption */ + ulint page_encryption_key) /*!< in: page encryption key */ __attribute__((nonnull)); /********************************************************************//** Convert a 32 bit integer table flags to the 32 bit integer that is diff --git a/storage/xtradb/include/dict0dict.ic b/storage/xtradb/include/dict0dict.ic index 2b698dd721848..6a54cf5719063 100644 --- a/storage/xtradb/include/dict0dict.ic +++ b/storage/xtradb/include/dict0dict.ic @@ -543,6 +543,10 @@ dict_tf_is_valid( ulint data_dir = DICT_TF_HAS_DATA_DIR(flags); ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(flags); + ulint page_encryption = DICT_TF_GET_PAGE_ENCRYPTION(flags); + ulint page_encryption_key = DICT_TF_GET_PAGE_ENCRYPTION_KEY(flags); + + /* Make sure there are no bits that we do not know about. */ if (unused != 0) { @@ -554,9 +558,11 @@ dict_tf_is_valid( "InnoDB: unused %ld data_dir %ld zip_ssize %ld\n" "InnoDB: page_compression %ld page_compression_level %ld\n" "InnoDB: atomic_writes %ld\n", + "InnoDB: page_encryption %ld page_encryption_key %ld\n", unused, compact, atomic_blobs, unused, data_dir, zip_ssize, - page_compression, page_compression_level, atomic_writes + page_compression, page_compression_level, atomic_writes, + page_encryption, page_encryption_key ); return(false); @@ -693,7 +699,10 @@ dict_sys_tables_type_validate( ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL(type); ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(type); - ut_a(atomic_writes <= ATOMIC_WRITES_OFF); + ulint page_encryption = DICT_TF_GET_PAGE_ENCRYPTION(type); + ulint page_encryption_key = DICT_TF_GET_PAGE_ENCRYPTION_KEY(type); + + ut_a(atomic_writes >= 0 && atomic_writes <= ATOMIC_WRITES_OFF); /* The low order bit of SYS_TABLES.TYPE is always set to 1. If the format is UNIV_FORMAT_B or higher, this field is the same @@ -856,7 +865,9 @@ dict_tf_set( pages */ ulint page_compression_level, /*!< in: table page compression level */ - ulint atomic_writes) /*!< in: table atomic writes setup */ + ulint atomic_writes, /*!< in: table atomic writes setup */ + bool page_encrypted, /*!< in: table uses page encryption */ + ulint page_encryption_key /*!< in: page encryption key */) { atomic_writes_t awrites = (atomic_writes_t)atomic_writes; @@ -897,6 +908,11 @@ dict_tf_set( *flags |= (atomic_writes << DICT_TF_POS_ATOMIC_WRITES); ut_a(dict_tf_get_atomic_writes(*flags) == awrites); + + if (page_encrypted) { + *flags |= (1 << DICT_TF_POS_PAGE_ENCRYPTION) + | (page_encryption_key << DICT_TF_POS_PAGE_ENCRYPTION_KEY); + } } /********************************************************************//** @@ -919,6 +935,10 @@ dict_tf_to_fsp_flags( ulint fsp_flags; ulint page_compression = DICT_TF_GET_PAGE_COMPRESSION(table_flags); ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL(table_flags); + + ulint page_encryption = DICT_TF_GET_PAGE_ENCRYPTION(table_flags); + ulint page_encryption_key = DICT_TF_GET_PAGE_ENCRYPTION_KEY(table_flags); + ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(table_flags); DBUG_EXECUTE_IF("dict_tf_to_fsp_flags_failure", @@ -946,6 +966,14 @@ dict_tf_to_fsp_flags( if page compression is used for this table. */ fsp_flags |= FSP_FLAGS_SET_PAGE_COMPRESSION_LEVEL(fsp_flags, page_compression_level); + /* In addition, tablespace flags also contain if the page + encryption is used for this table. */ + fsp_flags |= FSP_FLAGS_SET_PAGE_ENCRYPTION(fsp_flags, page_encryption); + + /* In addition, tablespace flags also contain page encryption key if the page + encryption is used for this table. */ + fsp_flags |= FSP_FLAGS_SET_PAGE_ENCRYPTION_KEY(fsp_flags, page_encryption_key); + /* In addition, tablespace flags also contain flag if atomic writes is used for this table */ fsp_flags |= FSP_FLAGS_SET_ATOMIC_WRITES(fsp_flags, atomic_writes); @@ -987,6 +1015,9 @@ dict_sys_tables_type_to_tf( | DICT_TF_MASK_PAGE_COMPRESSION | DICT_TF_MASK_PAGE_COMPRESSION_LEVEL | DICT_TF_MASK_ATOMIC_WRITES + | DICT_TF_MASK_PAGE_ENCRYPTION + | DICT_TF_MASK_PAGE_ENCRYPTION_KEY + ); return(flags); @@ -1022,7 +1053,9 @@ dict_tf_to_sys_tables_type( | DICT_TF_MASK_DATA_DIR | DICT_TF_MASK_PAGE_COMPRESSION | DICT_TF_MASK_PAGE_COMPRESSION_LEVEL - | DICT_TF_MASK_ATOMIC_WRITES); + | DICT_TF_MASK_ATOMIC_WRITES + | DICT_TF_MASK_PAGE_ENCRYPTION + | DICT_TF_MASK_PAGE_ENCRYPTION_KEY); return(type); } diff --git a/storage/xtradb/include/dict0mem.h b/storage/xtradb/include/dict0mem.h index 8de9206cb81c8..e32012dd7dc60 100644 --- a/storage/xtradb/include/dict0mem.h +++ b/storage/xtradb/include/dict0mem.h @@ -132,6 +132,12 @@ Width of the page compression flag #define DICT_TF_WIDTH_PAGE_COMPRESSION 1 #define DICT_TF_WIDTH_PAGE_COMPRESSION_LEVEL 4 +/** +Width of the page encryption flag +*/ +#define DICT_TF_WIDTH_PAGE_ENCRYPTION 1 +#define DICT_TF_WIDTH_PAGE_ENCRYPTION_KEY 8 + /** Width of atomic writes flag DEFAULT=0, ON = 1, OFF = 2 @@ -145,7 +151,7 @@ DEFAULT=0, ON = 1, OFF = 2 + DICT_TF_WIDTH_DATA_DIR \ + DICT_TF_WIDTH_PAGE_COMPRESSION \ + DICT_TF_WIDTH_PAGE_COMPRESSION_LEVEL \ - + DICT_TF_WIDTH_ATOMIC_WRITES) + + DICT_TF_WIDTH_ATOMIC_WRITES + DICT_TF_WIDTH_PAGE_ENCRYPTION + DICT_TF_WIDTH_PAGE_ENCRYPTION_KEY) /** A mask of all the known/used bits in table flags */ #define DICT_TF_BIT_MASK (~(~0 << DICT_TF_BITS)) @@ -170,9 +176,16 @@ DEFAULT=0, ON = 1, OFF = 2 /** Zero relative shift position of the ATOMIC_WRITES field */ #define DICT_TF_POS_ATOMIC_WRITES (DICT_TF_POS_PAGE_COMPRESSION_LEVEL \ + DICT_TF_WIDTH_PAGE_COMPRESSION_LEVEL) + +/** Zero relative shift position of the PAGE_ENCRYPTION field */ +#define DICT_TF_POS_PAGE_ENCRYPTION (DICT_TF_POS_ATOMIC_WRITES \ + + DICT_TF_WIDTH_ATOMIC_WRITES) +/** Zero relative shift position of the PAGE_ENCRYPTION_KEY field */ +#define DICT_TF_POS_PAGE_ENCRYPTION_KEY (DICT_TF_POS_PAGE_ENCRYPTION \ + + DICT_TF_WIDTH_PAGE_ENCRYPTION) /** Zero relative shift position of the start of the UNUSED bits */ -#define DICT_TF_POS_UNUSED (DICT_TF_POS_ATOMIC_WRITES \ - + DICT_TF_WIDTH_ATOMIC_WRITES) +#define DICT_TF_POS_UNUSED (DICT_TF_POS_PAGE_ENCRYPTION_KEY \ + + DICT_TF_WIDTH_PAGE_ENCRYPTION_KEY) /** Bit mask of the COMPACT field */ #define DICT_TF_MASK_COMPACT \ @@ -202,6 +215,14 @@ DEFAULT=0, ON = 1, OFF = 2 #define DICT_TF_MASK_ATOMIC_WRITES \ ((~(~0 << DICT_TF_WIDTH_ATOMIC_WRITES)) \ << DICT_TF_POS_ATOMIC_WRITES) +/** Bit mask of the PAGE_ENCRYPTION field */ +#define DICT_TF_MASK_PAGE_ENCRYPTION \ + ((~(~0 << DICT_TF_WIDTH_PAGE_ENCRYPTION)) \ + << DICT_TF_POS_PAGE_ENCRYPTION) +/** Bit mask of the PAGE_ENCRYPTION_KEY field */ +#define DICT_TF_MASK_PAGE_ENCRYPTION_KEY \ + ((~(~0 << DICT_TF_WIDTH_PAGE_ENCRYPTION_KEY)) \ + << DICT_TF_POS_PAGE_ENCRYPTION_KEY) /** Return the value of the COMPACT field */ #define DICT_TF_GET_COMPACT(flags) \ @@ -219,6 +240,17 @@ DEFAULT=0, ON = 1, OFF = 2 #define DICT_TF_HAS_DATA_DIR(flags) \ ((flags & DICT_TF_MASK_DATA_DIR) \ >> DICT_TF_POS_DATA_DIR) + +/** Return the contents of the PAGE_ENCRYPTION field */ +#define DICT_TF_GET_PAGE_ENCRYPTION(flags) \ + ((flags & DICT_TF_MASK_PAGE_ENCRYPTION) \ + >> DICT_TF_POS_PAGE_ENCRYPTION) +/** Return the contents of the PAGE_ENCRYPTION KEY field */ +#define DICT_TF_GET_PAGE_ENCRYPTION_KEY(flags) \ + ((flags & DICT_TF_MASK_PAGE_ENCRYPTION_KEY) \ + >> DICT_TF_POS_PAGE_ENCRYPTION_KEY) + + /** Return the contents of the UNUSED bits */ #define DICT_TF_GET_UNUSED(flags) \ (flags >> DICT_TF_POS_UNUSED) diff --git a/storage/xtradb/include/dict0pagecompress.ic b/storage/xtradb/include/dict0pagecompress.ic index 811976434a83b..3ada655d601a8 100644 --- a/storage/xtradb/include/dict0pagecompress.ic +++ b/storage/xtradb/include/dict0pagecompress.ic @@ -42,6 +42,8 @@ dict_tf_verify_flags( ulint page_compression = DICT_TF_GET_PAGE_COMPRESSION(table_flags); ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL(table_flags); ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(table_flags); + ulint page_encryption = DICT_TF_GET_PAGE_ENCRYPTION(table_flags); + ulint page_encryption_key = DICT_TF_GET_PAGE_ENCRYPTION_KEY(table_flags); ulint post_antelope = FSP_FLAGS_GET_POST_ANTELOPE(fsp_flags); ulint zip_ssize = FSP_FLAGS_GET_ZIP_SSIZE(fsp_flags); ulint fsp_atomic_blobs = FSP_FLAGS_HAS_ATOMIC_BLOBS(fsp_flags); @@ -50,6 +52,9 @@ dict_tf_verify_flags( ulint fsp_page_compression = FSP_FLAGS_GET_PAGE_COMPRESSION(fsp_flags); ulint fsp_page_compression_level = FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(fsp_flags); ulint fsp_atomic_writes = FSP_FLAGS_GET_ATOMIC_WRITES(fsp_flags); + ulint fsp_page_encryption = FSP_FLAGS_GET_PAGE_ENCRYPTION(fsp_flags); + ulint fsp_page_encryption_key = FSP_FLAGS_GET_PAGE_ENCRYPTION_KEY(fsp_flags); + DBUG_EXECUTE_IF("dict_tf_verify_flags_failure", return(ULINT_UNDEFINED);); @@ -107,6 +112,26 @@ dict_tf_verify_flags( return (FALSE); } + if (page_encryption != fsp_page_encryption) { + fprintf(stderr, + "InnoDB: Error: table flags has page_encryption %ld" + " in the data dictionary\n" + "InnoDB: but the flags in file has page_encryption %ld\n", + page_encryption, fsp_page_encryption); + + return (FALSE); + } + if (page_encryption_key != fsp_page_encryption_key) { + fprintf(stderr, + "InnoDB: Error: table flags has page_encryption_key %ld" + " in the data dictionary\n" + "InnoDB: but the flags in file has page_encryption_key %ld\n", + page_encryption_key, fsp_page_encryption_key); + + return (FALSE); + } + + return(TRUE); } diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index 3960eef5d7e09..688e25957f8ff 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -157,6 +157,14 @@ static const ulint FIL_PAGE_COMPRESS_SIZE_V1 = FIL_PAGE_ORIGINAL_SIZE_V1 + 2; #define FIL_PAGE_COMPRESSION_ZLIB 1 /*!< Compressin algorithm ZLIB. */ #define FIL_PAGE_COMPRESSION_LZ4 2 /*!< Compressin algorithm LZ4. */ +#define FIL_PAGE_ENCRYPTION_AES_128 16 /*!< Encryption algorithm AES-128. */ +#define FIL_PAGE_ENCRYPTION_AES_196 24 /*!< Encryption algorithm AES-196. */ +#define FIL_PAGE_ENCRYPTION_AES_256 32 /*!< Encryption algorithm AES-256. */ + +#define FIL_PAGE_ENCRYPTED_SIZE 2 /*!< Number of bytes used to store + actual payload data size on encrypted pages. */ + + /* @} */ /** File page trailer @{ */ #define FIL_PAGE_END_LSN_OLD_CHKSUM 8 /*!< the low 4 bytes of this are used @@ -168,6 +176,7 @@ static const ulint FIL_PAGE_COMPRESS_SIZE_V1 = FIL_PAGE_ORIGINAL_SIZE_V1 + 2; /** File page types (values of FIL_PAGE_TYPE) @{ */ #define FIL_PAGE_PAGE_COMPRESSED 34354 /*!< Page compressed page */ +#define FIL_PAGE_PAGE_ENCRYPTED 34355 /*!< Page encrypted page */ #define FIL_PAGE_INDEX 17855 /*!< B-tree node */ #define FIL_PAGE_UNDO_LOG 2 /*!< Undo log page */ #define FIL_PAGE_INODE 3 /*!< Index node */ diff --git a/storage/xtradb/include/fil0pageencryption.h b/storage/xtradb/include/fil0pageencryption.h new file mode 100644 index 0000000000000..c192d40fda812 --- /dev/null +++ b/storage/xtradb/include/fil0pageencryption.h @@ -0,0 +1,97 @@ +/***************************************************************************** + +Copyright (C) 2013 SkySQL Ab. All Rights Reserved. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*****************************************************************************/ + +#ifndef fil0pageencryption_h +#define fil0pageencryption_h + +#ifndef EP_UNIT_TEST +#include "fsp0fsp.h" +#include "fsp0pageencryption.h" +#endif + +#define PAGE_ENCRYPTION_WRONG_KEY 1 + +/******************************************************************//** +@file include/fil0pageencryption.h +Helper functions for encryption/decryption page data on to table space. + +Created 08/25/2014 Florin Fugaciu +***********************************************************************/ + + +/*******************************************************************//** +Returns the page encryption flag of the space, or false if the space +is not encrypted. The tablespace must be cached in the memory cache. +@return true if page encrypted, false if not or space not found */ +ibool +fil_space_is_page_encrypted( +/*=========================*/ + ulint id); /*!< in: space id */ + + +/*******************************************************************//** +Find out whether the page is page encrypted +@return true if page is page encrypted, false if not */ +UNIV_INLINE +ibool +fil_page_is_encrypted( +/*===================*/ + const byte *buf); /*!< in: page */ + + +/****************************************************************//** +For page encrypted pages encrypt the page before actual write +operation. +@return encrypted page to be written*/ +byte* +fil_encrypt_page( +/*==============*/ + ulint space_id, /*!< in: tablespace id of the + table. */ + byte* buf, /*!< in: buffer from which to write; in aio + this must be appropriately aligned */ + byte* out_buf, /*!< out: compressed buffer */ + ulint len, /*!< in: length of input buffer.*/ + ulint compression_level, /*!< in: compression level */ + ulint* out_len, /*!< out: actual length of compressed page */ + ulint unit_test); + +/****************************************************************//** +For page encrypted pages decrypt the page after actual read +operation. +@return decrypted page */ +void +fil_decrypt_page( +/*================*/ + byte* page_buf, /*!< in: preallocated buffer or NULL */ + byte* buf, /*!< out: buffer from which to read; in aio + this must be appropriately aligned */ + ulint len, /*!< in: length of output buffer.*/ + ulint* write_size, /*!< in/out: Actual payload size of the decrypted data. */ + ulint unit_test); + + + + +void do_check_sum( ulint page_size, byte* buf); + +int summef(int a, int b); +int summef2(int a, int b); +int multiplikation(int a, int b); + +#endif // fil0pageencryption_h diff --git a/storage/xtradb/include/fsp0fsp.h b/storage/xtradb/include/fsp0fsp.h index 6fe44a0ef1642..2749ffde4393e 100644 --- a/storage/xtradb/include/fsp0fsp.h +++ b/storage/xtradb/include/fsp0fsp.h @@ -57,6 +57,11 @@ is found in a remote location, not the default data directory. */ /** Number of flag bits used to indicate the page compression and compression level */ #define FSP_FLAGS_WIDTH_PAGE_COMPRESSION 1 #define FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL 4 + +/** Number of flag bits used to indicate the page compression and compression level */ +#define FSP_FLAGS_WIDTH_PAGE_ENCRYPTION 1 +#define FSP_FLAGS_WIDTH_PAGE_ENCRYPTION_KEY 8 + /** Number of flag bits used to indicate atomic writes for this tablespace */ #define FSP_FLAGS_WIDTH_ATOMIC_WRITES 2 @@ -68,7 +73,9 @@ is found in a remote location, not the default data directory. */ + FSP_FLAGS_WIDTH_DATA_DIR \ + FSP_FLAGS_WIDTH_PAGE_COMPRESSION \ + FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL \ - + FSP_FLAGS_WIDTH_ATOMIC_WRITES) + + FSP_FLAGS_WIDTH_ATOMIC_WRITES \ + + FSP_FLAGS_WIDTH_PAGE_ENCRYPTION \ + + FSP_FLAGS_WIDTH_PAGE_ENCRYPTION_KEY) /** A mask of all the known/used bits in tablespace flags */ #define FSP_FLAGS_MASK (~(~0 << FSP_FLAGS_WIDTH)) @@ -92,15 +99,24 @@ dictionary */ /** Zero relative shift position of the ATOMIC_WRITES field */ #define FSP_FLAGS_POS_ATOMIC_WRITES (FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL \ + FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL) + +#define FSP_FLAGS_POS_PAGE_ENCRYPTION (FSP_FLAGS_POS_ATOMIC_WRITES \ + + FSP_FLAGS_WIDTH_ATOMIC_WRITES) +/** Zero relative shift position of the PAGE_ENCRYPTION_KEY field */ +#define FSP_FLAGS_POS_PAGE_ENCRYPTION_KEY (FSP_FLAGS_POS_PAGE_ENCRYPTION \ + + FSP_FLAGS_WIDTH_PAGE_ENCRYPTION) + + + /** Zero relative shift position of the PAGE_SSIZE field */ -#define FSP_FLAGS_POS_PAGE_SSIZE (FSP_FLAGS_POS_ATOMIC_WRITES \ - + FSP_FLAGS_WIDTH_ATOMIC_WRITES) +#define FSP_FLAGS_POS_PAGE_SSIZE (FSP_FLAGS_POS_PAGE_ENCRYPTION_KEY \ + + FSP_FLAGS_WIDTH_PAGE_ENCRYPTION_KEY) /** Zero relative shift position of the start of the DATA DIR bits */ #define FSP_FLAGS_POS_DATA_DIR (FSP_FLAGS_POS_PAGE_SSIZE \ + FSP_FLAGS_WIDTH_PAGE_SSIZE) /** Zero relative shift position of the start of the UNUSED bits */ -#define FSP_FLAGS_POS_UNUSED (FSP_FLAGS_POS_DATA_DIR \ - + FSP_FLAGS_WIDTH_DATA_DIR) +#define FSP_FLAGS_POS_UNUSED (FSP_FLAGS_POS_DATA_DIR\ + + FSP_FLAGS_WIDTH_DATA_DIR) /** Bit mask of the POST_ANTELOPE field */ #define FSP_FLAGS_MASK_POST_ANTELOPE \ @@ -130,6 +146,17 @@ dictionary */ #define FSP_FLAGS_MASK_PAGE_COMPRESSION_LEVEL \ ((~(~0 << FSP_FLAGS_WIDTH_PAGE_COMPRESSION_LEVEL)) \ << FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL) + +#define FSP_FLAGS_MASK_PAGE_ENCRYPTION \ + ((~(~0 << FSP_FLAGS_WIDTH_PAGE_ENCRYPTION)) \ + << FSP_FLAGS_POS_PAGE_ENCRYPTION) +/** Bit mask of the PAGE_ENCRYPTION_KEY field */ +#define FSP_FLAGS_MASK_PAGE_ENCRYPTION_KEY \ + ((~(~0 << FSP_FLAGS_WIDTH_PAGE_ENCRYPTION_KEY)) \ + << FSP_FLAGS_POS_PAGE_ENCRYPTION_KEY) + + + /** Bit mask of the ATOMIC_WRITES field */ #define FSP_FLAGS_MASK_ATOMIC_WRITES \ ((~(~0 << FSP_FLAGS_WIDTH_ATOMIC_WRITES)) \ @@ -172,6 +199,16 @@ dictionary */ ((flags & FSP_FLAGS_MASK_ATOMIC_WRITES) \ >> FSP_FLAGS_POS_ATOMIC_WRITES) + +#define FSP_FLAGS_GET_PAGE_ENCRYPTION(flags) \ + ((flags & FSP_FLAGS_MASK_PAGE_ENCRYPTION) \ + >> FSP_FLAGS_POS_PAGE_ENCRYPTION) +/** Return the value of the PAGE_ENCRYPTION_KEY field */ +#define FSP_FLAGS_GET_PAGE_ENCRYPTION_KEY(flags) \ + ((flags & FSP_FLAGS_MASK_PAGE_ENCRYPTION_KEY) \ + >> FSP_FLAGS_POS_PAGE_ENCRYPTION_KEY) + + /** Set a PAGE_SSIZE into the correct bits in a given tablespace flags. */ #define FSP_FLAGS_SET_PAGE_SSIZE(flags, ssize) \ @@ -186,6 +223,14 @@ tablespace flags. */ tablespace flags. */ #define FSP_FLAGS_SET_PAGE_COMPRESSION_LEVEL(flags, level) \ (flags | (level << FSP_FLAGS_POS_PAGE_COMPRESSION_LEVEL)) + +/** Set a PAGE_ENCRYPTION into the correct bits in a given tablespace flags. */ +#define FSP_FLAGS_SET_PAGE_ENCRYPTION(flags, encryption) \ + (flags | (encryption << FSP_FLAGS_POS_PAGE_ENCRYPTION)) +/** Set a PAGE_ENCRYPTION_KEY into the correct bits in a given tablespace flags. */ +#define FSP_FLAGS_SET_PAGE_ENCRYPTION_KEY(flags, encryption_key) \ + (flags | (encryption_key << FSP_FLAGS_POS_PAGE_ENCRYPTION_KEY)) + /** Set a ATOMIC_WRITES into the correct bits in a given tablespace flags. */ #define FSP_FLAGS_SET_ATOMIC_WRITES(flags, atomics) \ diff --git a/storage/xtradb/include/fsp0fsp.ic b/storage/xtradb/include/fsp0fsp.ic index ddcb87b0e570d..368b2d73404c3 100644 --- a/storage/xtradb/include/fsp0fsp.ic +++ b/storage/xtradb/include/fsp0fsp.ic @@ -66,6 +66,8 @@ fsp_flags_is_valid( ulint unused = FSP_FLAGS_GET_UNUSED(flags); ulint page_compression = FSP_FLAGS_GET_PAGE_COMPRESSION(flags); ulint page_compression_level = FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(flags); + ulint page_encryption = FSP_FLAGS_GET_PAGE_ENCRYPTION(flags); + ulint page_encryption_key = FSP_FLAGS_GET_PAGE_ENCRYPTION_KEY(flags); ulint atomic_writes = FSP_FLAGS_GET_ATOMIC_WRITES(flags); DBUG_EXECUTE_IF("fsp_flags_is_valid_failure", return(false);); diff --git a/storage/xtradb/include/fsp0pageencryption.h b/storage/xtradb/include/fsp0pageencryption.h new file mode 100644 index 0000000000000..fdf484ea64485 --- /dev/null +++ b/storage/xtradb/include/fsp0pageencryption.h @@ -0,0 +1,59 @@ +/***************************************************************************** + +Copyright (C) 2013 SkySQL Ab. All Rights Reserved. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*****************************************************************************/ + +/******************************************************************//** +@file include/fsp0pageencryption.h +Helper functions for extracting/storing page encryption information to file space. + +Created 08/28/2014 Florin Fugaciu +***********************************************************************/ + +#ifndef FSP0PAGEENCRYPTION_H_ +#define FSP0PAGEENCRYPTION_H_ + + + +/********************************************************************//** +Determine if the tablespace is page encrypted from dict_table_t::flags. +@return TRUE if page encrypted, FALSE if not page encrypted */ +UNIV_INLINE +ibool +fsp_flags_is_page_encrypted( +/*=========================*/ + ulint flags); /*!< in: tablespace flags */ + + +/********************************************************************//** +Extract the page encryption key from tablespace flags. +A tablespace has only one physical page encryption key +whether that page is encrypted or not. +@return page encryption key of the file-per-table tablespace, +or zero if the table is not encrypted. */ +UNIV_INLINE +ulint +fsp_flags_get_page_encryption_key( +/*=================================*/ + ulint flags); /*!< in: tablespace flags */ + + +#ifndef UNIV_NONINL +#include "fsp0pageencryption.ic" +#endif + + +#endif /* FSP0PAGEENCRYPTION_H_ */ diff --git a/storage/xtradb/include/fsp0pageencryption.ic b/storage/xtradb/include/fsp0pageencryption.ic new file mode 100644 index 0000000000000..c86f1c371b004 --- /dev/null +++ b/storage/xtradb/include/fsp0pageencryption.ic @@ -0,0 +1,120 @@ +/***************************************************************************** + +Copyright (C) 2013,2014 SkySQL Ab. All Rights Reserved. + +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*****************************************************************************/ + +/******************************************************************//** +@file include/fsp0pageencryption.ic +Implementation for helper functions for encrypting/decrypting pages +and atomic writes information to file space. + +Created 08/28/2014 Florin Fugaciu +***********************************************************************/ + +#include "fsp0fsp.h" + + + + +/********************************************************************//** +Determine if the tablespace is page encrypted from dict_table_t::flags. +@return TRUE if page encrypted, FALSE if not page encrypted */ +UNIV_INLINE +ibool +fsp_flags_is_page_encrypted( +/*=========================*/ + ulint flags) /*!< in: tablespace flags */ +{ + return(FSP_FLAGS_GET_PAGE_ENCRYPTION(flags)); +} + +/********************************************************************//** +Extract the page encryption key from tablespace flags. +A tablespace has only one physical page encryption key +whether that page is encrypted or not. +@return page encryption key of the file-per-table tablespace, +or zero if the table is not encrypted. */ +UNIV_INLINE +ulint +fsp_flags_get_page_encryption_key( +/*=================================*/ + ulint flags) /*!< in: tablespace flags */ +{ + return(FSP_FLAGS_GET_PAGE_ENCRYPTION_KEY(flags)); +} + + +/*******************************************************************//** +Returns the page encryption flag of the space, or false if the space +is not encrypted. The tablespace must be cached in the memory cache. +@return true if page encrypted, false if not or space not found */ +UNIV_INLINE +ibool +fil_space_is_page_encrypted( +/*=========================*/ + ulint id) /*!< in: space id */ +{ + ulint flags; + + flags = fil_space_get_flags(id); + + if (flags && flags != ULINT_UNDEFINED) { + + return(fsp_flags_is_page_encrypted(flags)); + } + + return(flags); +} + + +/********************************************************************//** +Determine the tablespace is using atomic writes from dict_table_t::flags. +@return true if atomic writes is used, false if not * /read_ +UNIV_INLINE +atomic_writes_t +fsp_flags_get_atomic_writes( +/*========================* / + ulint flags) /*!< in: tablespace flags * / +{ + return((atomic_writes_t)FSP_FLAGS_GET_ATOMIC_WRITES(flags)); +}*/ + +/*******************************************************************//** +Find out wheather the page is index page or not +@return true if page type index page, false if not * / +UNIV_INLINE +ibool +fil_page_is_index_page( +/*===================* / + byte *buf) /*!< in: page * / +{ + return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_INDEX); +}*/ + +/*******************************************************************//** +Find out whether the page is page encrypted +@return true if page is page encrypted, false if not */ +UNIV_INLINE +ibool +fil_page_is_encrypted( +/*===================*/ + const byte *buf) /*!< in: page */ +{ + //ibool result = FALSE; + ibool result = TRUE; + return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_ENCRYPTED); + //return(result); +} diff --git a/storage/xtradb/include/os0file.h b/storage/xtradb/include/os0file.h index 76e77799b4372..841e1c447bded 100644 --- a/storage/xtradb/include/os0file.h +++ b/storage/xtradb/include/os0file.h @@ -322,10 +322,10 @@ The wrapper functions have the prefix of "innodb_". */ # define os_aio(type, mode, name, file, buf, offset, \ n, message1, message2, space_id, \ - trx, page_compressed, page_compression_level, write_size) \ + trx, page_compressed, page_compression_level, write_size, page_encryption, page_encryption_key) \ pfs_os_aio_func(type, mode, name, file, buf, offset, \ n, message1, message2, space_id, trx, \ - page_compressed, page_compression_level, write_size, \ + page_compressed, page_compression_level, write_size, page_encryption, page_encryption_key, \ __FILE__, __LINE__) # define os_file_read(file, buf, offset, n, compressed) \ @@ -374,10 +374,10 @@ to original un-instrumented file I/O APIs */ # define os_aio(type, mode, name, file, buf, offset, n, message1, \ message2, space_id, trx, \ - page_compressed, page_compression_level, write_size) \ + page_compressed, page_compression_level, write_size, page_encryption, page_encryption_key) \ os_aio_func(type, mode, name, file, buf, offset, n, \ message1, message2, space_id, trx, \ - page_compressed, page_compression_level, write_size) + page_compressed, page_compression_level, write_size, page_encryption, page_encryption_key) # define os_file_read(file, buf, offset, n, compressed) \ os_file_read_func(file, buf, offset, n, NULL, compressed) @@ -805,6 +805,10 @@ pfs_os_aio_func( operation for this page and if initialized we do not trim again if actual page size does not decrease. */ + ibool page_encryption, /*!< in: is page encryption used + on this file space */ + ulint page_encryption_key, /*!< page encryption + key to be used */ const char* src_file,/*!< in: file name where func invoked */ ulint src_line);/*!< in: line where the func invoked */ /*******************************************************************//** @@ -1187,6 +1191,10 @@ os_aio_func( on this file space */ ulint page_compression_level, /*!< page compression level to be used */ + ibool page_encryption, /*!< in: is page encryption used + on this file space */ + ulint page_encryption_key, /*!< page encryption key + to be used */ ulint* write_size);/*!< in/out: Actual write size initialized after fist successfull trim operation for this page and if diff --git a/storage/xtradb/include/os0file.ic b/storage/xtradb/include/os0file.ic index 61300387e1bf9..f2ff5c79351d5 100644 --- a/storage/xtradb/include/os0file.ic +++ b/storage/xtradb/include/os0file.ic @@ -229,6 +229,11 @@ pfs_os_aio_func( operation for this page and if initialized we do not trim again if actual page size does not decrease. */ + ibool page_encryption, /*!< in: is page encryption used + on this file space */ + ulint page_encryption_key, /*!< page encryption + key to be used */ + const char* src_file,/*!< in: file name where func invoked */ ulint src_line)/*!< in: line where the func invoked */ { @@ -245,7 +250,8 @@ pfs_os_aio_func( result = os_aio_func(type, mode, name, file, buf, offset, n, message1, message2, space_id, trx, - page_compression, page_compression_level, write_size); + page_compression, page_compression_level, + page_encryption, page_encryption_key, write_size); register_pfs_file_io_end(locker, n); diff --git a/storage/xtradb/include/srv0mon.h b/storage/xtradb/include/srv0mon.h index 8e6975ed68fe4..8f8c7317cba23 100644 --- a/storage/xtradb/include/srv0mon.h +++ b/storage/xtradb/include/srv0mon.h @@ -318,6 +318,11 @@ enum monitor_id_t { MONITOR_OVLD_PAGES_PAGE_DECOMPRESSED, MONITOR_OVLD_PAGES_PAGE_COMPRESSION_ERROR, + /* New monitor variables for page encryption */ + MONITOR_OVLD_PAGES_PAGE_ENCRYPTED, + MONITOR_OVLD_PAGES_PAGE_DECRYPTED, + MONITOR_OVLD_PAGES_PAGE_ENCRYPTION_ERROR, + /* Index related counters */ MONITOR_MODULE_INDEX, MONITOR_INDEX_SPLIT, diff --git a/storage/xtradb/include/srv0srv.h b/storage/xtradb/include/srv0srv.h index 48cad8c36c09f..3b78a8ccb8b5a 100644 --- a/storage/xtradb/include/srv0srv.h +++ b/storage/xtradb/include/srv0srv.h @@ -124,6 +124,13 @@ struct srv_stats_t { /* Number of page compression errors */ ulint_ctr_64_t pages_page_compression_error; + /* Number of pages encrypted with page encryption */ + ulint_ctr_64_t pages_page_encrypted; + /* Number of pages decrypted with page encryption */ + ulint_ctr_64_t pages_page_decrypted; + /* Number of page encryption errors */ + ulint_ctr_64_t pages_page_encryption_error; + /** Number of data read in total (in bytes) */ ulint_ctr_1_t data_read; @@ -1143,6 +1150,12 @@ struct export_var_t{ compression */ ib_int64_t innodb_pages_page_compression_error;/*!< Number of page compression errors */ + ib_int64_t innodb_pages_page_encrypted;/*!< Number of pages + encrypted by page encryption */ + ib_int64_t innodb_pages_page_decrypted;/*!< Number of pages + decrypted by page encryption */ + ib_int64_t innodb_pages_page_encryption_error;/*!< Number of page + encryption errors */ }; /** Thread slot in the thread table. */ diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index d67573d14aa09..3f34a6c94ac31 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -43,7 +43,10 @@ Created 10/21/1995 Heikki Tuuri #include "srv0srv.h" #include "srv0start.h" #include "fil0fil.h" +#include "fsp0fsp.h" #include "fil0pagecompress.h" +#include "fsp0pageencryption.h" +#include "fil0pageencryption.h" #include "buf0buf.h" #include "btr0types.h" #include "trx0trx.h" @@ -219,9 +222,18 @@ struct os_aio_slot_t{ freed after the write has been completed */ + byte* page_encryption_page; /*!< Memory allocated for + page encrypted page and + freed after the write + has been completed */ + + ibool page_compression; ulint page_compression_level; + ibool page_encryption; + ulint page_encryption_key; + ulint* write_size; /*!< Actual write size initialized after fist successfull trim operation for this page and if @@ -232,7 +244,13 @@ struct os_aio_slot_t{ page compressed pages, do not free this */ + byte* page_buf2; /*!< Actual page buffer for + page encrypted pages, do not + free this */ + + ibool page_compress_success; + ibool page_encryption_success; #ifdef LINUX_NATIVE_AIO struct iocb control; /* Linux control block for aio */ @@ -378,6 +396,13 @@ os_slot_alloc_lzo_mem( os_aio_slot_t* slot); /*!< in: slot structure */ #endif +/**********************************************************************//** +Allocate memory for temporal buffer used for page encryption. This +buffer is freed later. */ +UNIV_INTERN +void +os_slot_alloc_page_buf2( +os_aio_slot_t* slot); /*!< in: slot structure */ /****************************************************************//** Does error handling when a file operation fails. @return TRUE if we should retry the operation */ @@ -488,19 +513,19 @@ os_get_os_version(void) /* Windows : Handling synchronous IO on files opened asynchronously. -If file is opened for asynchronous IO (FILE_FLAG_OVERLAPPED) and also bound to +If file is opened for asynchronous IO (FILE_FLAG_OVERLAPPED) and also bound to a completion port, then every IO on this file would normally be enqueued to the completion port. Sometimes however we would like to do a synchronous IO. This is possible if we initialitze have overlapped.hEvent with a valid event and set its lowest order bit to 1 (see MSDN ReadFile and WriteFile description for more info) -We'll create this special event once for each thread and store in thread local +We'll create this special event once for each thread and store in thread local storage. */ /***********************************************************************//** -Initialize tls index.for event handle used for synchronized IO on files that +Initialize tls index.for event handle used for synchronized IO on files that might be opened with FILE_FLAG_OVERLAPPED. */ static void win_init_syncio_event() @@ -3122,7 +3147,9 @@ os_file_read_func( ret = os_file_pread(file, buf, n, offset, trx); if ((ulint) ret == n) { - + if (fil_page_is_encrypted((byte *)buf)) { + fil_decrypt_page(NULL, (byte *)buf, n, NULL, 0); + } /* Note that InnoDB writes files that are not formated as file spaces and they do not have FIL_PAGE_TYPE field, thus we must use here information is the actual @@ -3131,6 +3158,7 @@ os_file_read_func( fil_decompress_page(NULL, (byte *)buf, n, NULL); } + return(TRUE); } @@ -3250,6 +3278,12 @@ os_file_read_no_error_handling_func( fil_decompress_page(NULL, (byte *)buf, n, NULL); } + if (fil_page_is_encrypted((byte *)buf)) { + fil_decrypt_page(NULL, (byte *)buf, n, NULL, 0); + } + + + return(TRUE); } #endif /* __WIN__ */ @@ -4262,6 +4296,15 @@ os_aio_array_free( } } + for (i = 0; i < array->n_slots; i++) { + os_aio_slot_t* slot = os_aio_array_get_nth_slot(array, i); + if (slot->page_encryption_page) { + ut_free(slot->page_encryption_page); + slot->page_encryption_page = NULL; + } + } + + ut_free(array->slots); ut_free(array); @@ -4611,6 +4654,10 @@ os_aio_array_reserve_slot( on this file space */ ulint page_compression_level, /*!< page compression level to be used */ + ibool page_encryption, /*!< in: is page encryption used + on this file space */ + ulint page_encryption_key, /*!< page encryption key + to be used */ ulint* write_size)/*!< in/out: Actual write size initialized after fist successfull trim operation for this page and if @@ -4708,9 +4755,13 @@ os_aio_array_reserve_slot( slot->space_id = space_id; slot->page_compress_success = FALSE; + slot->page_encryption_success = FALSE; + slot->write_size = write_size; slot->page_compression_level = page_compression_level; slot->page_compression = page_compression; + slot->page_encryption_key = page_encryption_key; + slot->page_encryption = page_encryption; /* If the space is page compressed and this is write operation then we compress the page */ @@ -4756,6 +4807,37 @@ os_aio_array_reserve_slot( /* Take array mutex back */ os_mutex_enter(array->mutex); + } //CMD + /* If the space is page encryption and this is write operation + then we encrypt the page */ + if (message1 && type == OS_FILE_WRITE && page_encryption ) { + ulint real_len = len; + byte* tmp = NULL; + + /* Release the array mutex while encrypting */ + os_mutex_exit(array->mutex); + + // We allocate memory for page encrypted buffer if and only + // if it is not yet allocated. + if (slot->page_buf2 == NULL) { + os_slot_alloc_page_buf2(slot); + } + + ut_ad(slot->page_buf2); + tmp = fil_encrypt_page(fil_node_get_space_id(slot->message1), (byte *)buf, slot->page_buf2, len, page_encryption_key, &real_len, 0); + + /* If encryption succeeded, set up the length and buffer */ + if (tmp != buf) { + len = real_len; + buf = slot->page_buf2; + slot->len = real_len; + slot->page_encryption_success = TRUE; + } else { + slot->page_encryption_success = FALSE; + } + + /* Take array mutex back */ + os_mutex_enter(array->mutex); } #ifdef WIN_ASYNC_IO @@ -5038,6 +5120,10 @@ os_aio_func( on this file space */ ulint page_compression_level, /*!< page compression level to be used */ + ibool page_encryption, /*!< in: is page encryption used + on this file space */ + ulint page_encryption_key, /*!< page encryption key + to be used */ ulint* write_size)/*!< in/out: Actual write size initialized after fist successfull trim operation for this page and if @@ -5134,7 +5220,8 @@ os_aio_func( } slot = os_aio_array_reserve_slot(type, array, message1, message2, file, name, buf, offset, n, space_id, - page_compression, page_compression_level, write_size); + page_compression, page_compression_level, + page_encryption, page_encryption_key, write_size); if (type == OS_FILE_READ) { if (srv_use_native_aio) { os_n_file_reads++; @@ -5245,7 +5332,7 @@ os_aio_windows_handle( HANDLE port = READ_SEGMENT(segment)? read_completion_port : completion_port; for(;;) { - ret = GetQueuedCompletionStatus(port, &len, &key, + ret = GetQueuedCompletionStatus(port, &len, &key, (OVERLAPPED **)&slot, INFINITE); /* If shutdown key was received, repost the shutdown message and exit */ @@ -5260,19 +5347,19 @@ os_aio_windows_handle( if(WRITE_SEGMENT(segment)&& slot->type == OS_FILE_READ) { /* - Redirect read completions to the dedicated completion port + Redirect read completions to the dedicated completion port and thread. We need to split read and write threads. If we do not - do that, and just allow all io threads process all IO, it is possible + do that, and just allow all io threads process all IO, it is possible to get stuck in a deadlock in buffer pool code, - Currently, the problem is solved this way - "write io" threads + Currently, the problem is solved this way - "write io" threads always get all completion notifications, from both async reads and writes. Write completion is handled in the same thread that gets it. Read completion is forwarded via PostQueueCompletionStatus()) to the second completion port dedicated solely to reads. One of the "read io" threads waiting on this port will finally handle the IO. - Forwarding IO completion this way costs a context switch , and this + Forwarding IO completion this way costs a context switch , and this seems tolerable since asynchronous reads are by far less frequent. */ ut_a(PostQueuedCompletionStatus(read_completion_port, len, key, @@ -5498,6 +5585,30 @@ os_aio_linux_collect( } } + /* page encryption */ + if (slot->message1 && slot->page_encryption) { + if (slot->page_buf2==NULL) { + os_slot_alloc_page_buf2(slot); + } + + ut_ad(slot->page_buf2); + + if (slot->type == OS_FILE_READ) { + if (fil_page_is_encrypted(slot->buf)) { + fil_decrypt_page(slot->page_buf2, slot->buf, slot->len, slot->write_size, 0); + } + } else { + if (slot->page_encryption_success && + fil_page_is_encrypted(slot->page_buf2)) { + ut_ad(slot->page_encryption_page); + if (srv_use_trim && os_fallocate_failed == FALSE) { + // Deallocate unused blocks from file system + os_file_trim(slot->file, slot, slot->len); + } + } + } + } + /* Mark this request as completed. The error handling will be done in the calling function. */ os_mutex_enter(array->mutex); @@ -6515,6 +6626,24 @@ os_file_trim( } +/**********************************************************************//** +Allocate memory for temporal buffer used for page encryption. This +buffer is freed later. */ +UNIV_INTERN +void +os_slot_alloc_page_buf2( +/*===================*/ + os_aio_slot_t* slot) /*!< in: slot structure */ +{ + byte* cbuf2; + byte* cbuf; + + cbuf2 = static_cast(ut_malloc(UNIV_PAGE_SIZE*2)); + cbuf = static_cast(ut_align(cbuf2, UNIV_PAGE_SIZE)); + slot->page_encryption_page = static_cast(cbuf2); + slot->page_buf2 = static_cast(cbuf); +} + /**********************************************************************//** Allocate memory for temporal buffer used for page compression. This buffer is freed later. */ @@ -6551,3 +6680,4 @@ os_slot_alloc_lzo_mem( ut_a(slot->lzo_mem != NULL); } #endif + diff --git a/storage/xtradb/os/os0file.cc.orig b/storage/xtradb/os/os0file.cc.orig new file mode 100644 index 0000000000000..c954c16a6fb10 --- /dev/null +++ b/storage/xtradb/os/os0file.cc.orig @@ -0,0 +1,6683 @@ +/*********************************************************************** + +Copyright (c) 1995, 2013, Oracle and/or its affiliates. All Rights Reserved. +Copyright (c) 2009, Percona Inc. +Copyright (c) 2013, 2014, SkySQL Ab. All Rights Reserved. + +Portions of this file contain modifications contributed and copyrighted +by Percona Inc.. Those modifications are +gratefully acknowledged and are described briefly in the InnoDB +documentation. The contributions by Percona Inc. are incorporated with +their permission, and subject to the conditions contained in the file +COPYING.Percona. + +This program is free software; you can redistribute it and/or modify it +under the terms of the GNU General Public License as published by the +Free Software Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but +WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA + +***********************************************************************/ + +/**************************************************//** +@file os/os0file.cc +The interface to the operating system file i/o primitives + +Created 10/21/1995 Heikki Tuuri +*******************************************************/ + +#include "os0file.h" + +#ifdef UNIV_NONINL +#include "os0file.ic" +#endif +#include "ha_prototypes.h" +#include "ut0mem.h" +#include "srv0srv.h" +#include "srv0start.h" +#include "fil0fil.h" +#include "fsp0fsp.h" +#include "fil0pagecompress.h" +#include "fsp0pageencryption.h" +#include "fil0pageencryption.h" +#include "buf0buf.h" +#include "btr0types.h" +#include "trx0trx.h" +#include "srv0mon.h" +#include "srv0srv.h" +#ifdef HAVE_POSIX_FALLOCATE +#include "fcntl.h" +#endif +#ifndef UNIV_HOTBACKUP +# include "os0sync.h" +# include "os0thread.h" +#else /* !UNIV_HOTBACKUP */ +# ifdef __WIN__ +/* Add includes for the _stat() call to compile on Windows */ +# include +# include +# include +# endif /* __WIN__ */ +#endif /* !UNIV_HOTBACKUP */ + +#if defined(LINUX_NATIVE_AIO) +#include +#endif + +#ifdef _WIN32 +#define IOCP_SHUTDOWN_KEY (ULONG_PTR)-1 +#endif + +#if defined(UNIV_LINUX) && defined(HAVE_SYS_IOCTL_H) +# include +# ifndef DFS_IOCTL_ATOMIC_WRITE_SET +# define DFS_IOCTL_ATOMIC_WRITE_SET _IOW(0x95, 2, uint) +# endif +#endif + +#ifdef HAVE_LZO +#include "lzo/lzo1x.h" +#endif + +/** Insert buffer segment id */ +static const ulint IO_IBUF_SEGMENT = 0; + +/** Log segment id */ +static const ulint IO_LOG_SEGMENT = 1; + +/* This specifies the file permissions InnoDB uses when it creates files in +Unix; the value of os_innodb_umask is initialized in ha_innodb.cc to +my_umask */ + +#ifndef __WIN__ +/** Umask for creating files */ +UNIV_INTERN ulint os_innodb_umask = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP; +#else +/** Umask for creating files */ +UNIV_INTERN ulint os_innodb_umask = 0; +#endif /* __WIN__ */ + +#ifndef UNIV_HOTBACKUP +/* We use these mutexes to protect lseek + file i/o operation, if the +OS does not provide an atomic pread or pwrite, or similar */ +#define OS_FILE_N_SEEK_MUTEXES 16 +UNIV_INTERN os_ib_mutex_t os_file_seek_mutexes[OS_FILE_N_SEEK_MUTEXES]; + +/* In simulated aio, merge at most this many consecutive i/os */ +#define OS_AIO_MERGE_N_CONSECUTIVE 64 + +#ifdef WITH_INNODB_DISALLOW_WRITES +#define WAIT_ALLOW_WRITES() os_event_wait(srv_allow_writes_event) +#else +#define WAIT_ALLOW_WRITES() do { } while (0) +#endif /* WITH_INNODB_DISALLOW_WRITES */ + +/********************************************************************** + +InnoDB AIO Implementation: +========================= + +We support native AIO for windows and linux. For rest of the platforms +we simulate AIO by special io-threads servicing the IO-requests. + +Simulated AIO: +============== + +In platforms where we 'simulate' AIO following is a rough explanation +of the high level design. +There are four io-threads (for ibuf, log, read, write). +All synchronous IO requests are serviced by the calling thread using +os_file_write/os_file_read. The Asynchronous requests are queued up +in an array (there are four such arrays) by the calling thread. +Later these requests are picked up by the io-thread and are serviced +synchronously. + +Windows native AIO: +================== + +If srv_use_native_aio is not set then windows follow the same +code as simulated AIO. If the flag is set then native AIO interface +is used. On windows, one of the limitation is that if a file is opened +for AIO no synchronous IO can be done on it. Therefore we have an +extra fifth array to queue up synchronous IO requests. +There are innodb_file_io_threads helper threads. These threads work +on the four arrays mentioned above in Simulated AIO. No thread is +required for the sync array. +If a synchronous IO request is made, it is first queued in the sync +array. Then the calling thread itself waits on the request, thus +making the call synchronous. +If an AIO request is made the calling thread not only queues it in the +array but also submits the requests. The helper thread then collects +the completed IO request and calls completion routine on it. + +Linux native AIO: +================= + +If we have libaio installed on the system and innodb_use_native_aio +is set to TRUE we follow the code path of native AIO, otherwise we +do simulated AIO. +There are innodb_file_io_threads helper threads. These threads work +on the four arrays mentioned above in Simulated AIO. +If a synchronous IO request is made, it is handled by calling +os_file_write/os_file_read. +If an AIO request is made the calling thread not only queues it in the +array but also submits the requests. The helper thread then collects +the completed IO request and calls completion routine on it. + +**********************************************************************/ + +/** Flag: enable debug printout for asynchronous i/o */ +UNIV_INTERN ibool os_aio_print_debug = FALSE; + +#ifdef UNIV_PFS_IO +/* Keys to register InnoDB I/O with performance schema */ +UNIV_INTERN mysql_pfs_key_t innodb_file_data_key; +UNIV_INTERN mysql_pfs_key_t innodb_file_log_key; +UNIV_INTERN mysql_pfs_key_t innodb_file_temp_key; +UNIV_INTERN mysql_pfs_key_t innodb_file_bmp_key; +#endif /* UNIV_PFS_IO */ + +/** The asynchronous i/o array slot structure */ +struct os_aio_slot_t{ +#ifdef WIN_ASYNC_IO + OVERLAPPED control; /*!< Windows control block for the + aio request, MUST be first element in the structure*/ + void *arr; /*!< Array this slot belongs to*/ +#endif + + ibool is_read; /*!< TRUE if a read operation */ + ulint pos; /*!< index of the slot in the aio + array */ + ibool reserved; /*!< TRUE if this slot is reserved */ + time_t reservation_time;/*!< time when reserved */ + ulint len; /*!< length of the block to read or + write */ + byte* buf; /*!< buffer used in i/o */ + ulint type; /*!< OS_FILE_READ or OS_FILE_WRITE */ + os_offset_t offset; /*!< file offset in bytes */ + os_file_t file; /*!< file where to read or write */ + const char* name; /*!< file name or path */ + ibool io_already_done;/*!< used only in simulated aio: + TRUE if the physical i/o already + made and only the slot message + needs to be passed to the caller + of os_aio_simulated_handle */ + ulint space_id; + fil_node_t* message1; /*!< message which is given by the */ + void* message2; /*!< the requester of an aio operation + and which can be used to identify + which pending aio operation was + completed */ + ulint bitmap; + + byte* page_compression_page; /*!< Memory allocated for + page compressed page and + freed after the write + has been completed */ + + byte* page_encryption_page; /*!< Memory allocated for + page encrypted page and + freed after the write + has been completed */ + + + ibool page_compression; + ulint page_compression_level; + + ibool page_encryption; + ulint page_encryption_key; + + ulint* write_size; /*!< Actual write size initialized + after fist successfull trim + operation for this page and if + initialized we do not trim again if + actual page size does not decrease. */ + + byte* page_buf; /*!< Actual page buffer for + page compressed pages, do not + free this */ + + byte* page_buf2; /*!< Actual page buffer for + page encrypted pages, do not + free this */ + + + ibool page_compress_success; + ibool page_encryption_success; + +#ifdef LINUX_NATIVE_AIO + struct iocb control; /* Linux control block for aio */ + int n_bytes; /* bytes written/read. */ + int ret; /* AIO return code */ +#endif /* WIN_ASYNC_IO */ + byte *lzo_mem; /* Temporal memory used by LZO */ +}; + +/** The asynchronous i/o array structure */ +struct os_aio_array_t{ + os_ib_mutex_t mutex; /*!< the mutex protecting the aio array */ + os_event_t not_full; + /*!< The event which is set to the + signaled state when there is space in + the aio outside the ibuf segment */ + os_event_t is_empty; + /*!< The event which is set to the + signaled state when there are no + pending i/os in this array */ + ulint n_slots;/*!< Total number of slots in the aio + array. This must be divisible by + n_threads. */ + ulint n_segments; + /*!< Number of segments in the aio + array of pending aio requests. A + thread can wait separately for any one + of the segments. */ + ulint cur_seg;/*!< We reserve IO requests in round + robin fashion to different segments. + This points to the segment that is to + be used to service next IO request. */ + ulint n_reserved; + /*!< Number of reserved slots in the + aio array outside the ibuf segment */ + os_aio_slot_t* slots; /*!< Pointer to the slots in the array */ + +#if defined(LINUX_NATIVE_AIO) + io_context_t* aio_ctx; + /* completion queue for IO. There is + one such queue per segment. Each thread + will work on one ctx exclusively. */ + struct io_event* aio_events; + /* The array to collect completed IOs. + There is one such event for each + possible pending IO. The size of the + array is equal to n_slots. */ +#endif /* LINUX_NATIV_AIO */ +}; + +#if defined(LINUX_NATIVE_AIO) +/** timeout for each io_getevents() call = 500ms. */ +#define OS_AIO_REAP_TIMEOUT (500000000UL) + +/** time to sleep, in microseconds if io_setup() returns EAGAIN. */ +#define OS_AIO_IO_SETUP_RETRY_SLEEP (500000UL) + +/** number of attempts before giving up on io_setup(). */ +#define OS_AIO_IO_SETUP_RETRY_ATTEMPTS 5 +#endif + +/** Array of events used in simulated aio */ +static os_event_t* os_aio_segment_wait_events = NULL; + +/** The aio arrays for non-ibuf i/o and ibuf i/o, as well as sync aio. These +are NULL when the module has not yet been initialized. @{ */ +static os_aio_array_t* os_aio_read_array = NULL; /*!< Reads */ +static os_aio_array_t* os_aio_write_array = NULL; /*!< Writes */ +static os_aio_array_t* os_aio_ibuf_array = NULL; /*!< Insert buffer */ +static os_aio_array_t* os_aio_log_array = NULL; /*!< Redo log */ +static os_aio_array_t* os_aio_sync_array = NULL; /*!< Synchronous I/O */ +/* @} */ + +/** Number of asynchronous I/O segments. Set by os_aio_init(). */ +static ulint os_aio_n_segments = ULINT_UNDEFINED; + +/** If the following is TRUE, read i/o handler threads try to +wait until a batch of new read requests have been posted */ +static ibool os_aio_recommend_sleep_for_read_threads = FALSE; +#endif /* !UNIV_HOTBACKUP */ + +UNIV_INTERN ulint os_n_file_reads = 0; +UNIV_INTERN ulint os_bytes_read_since_printout = 0; +UNIV_INTERN ulint os_n_file_writes = 0; +UNIV_INTERN ulint os_n_fsyncs = 0; +UNIV_INTERN ulint os_n_file_reads_old = 0; +UNIV_INTERN ulint os_n_file_writes_old = 0; +UNIV_INTERN ulint os_n_fsyncs_old = 0; +UNIV_INTERN time_t os_last_printout; + +UNIV_INTERN ibool os_has_said_disk_full = FALSE; + +#if !defined(UNIV_HOTBACKUP) \ + && (!defined(HAVE_ATOMIC_BUILTINS) || UNIV_WORD_SIZE < 8) +/** The mutex protecting the following counts of pending I/O operations */ +static os_ib_mutex_t os_file_count_mutex; +#endif /* !UNIV_HOTBACKUP && (!HAVE_ATOMIC_BUILTINS || UNIV_WORD_SIZE < 8) */ + +/** Number of pending os_file_pread() operations */ +UNIV_INTERN ulint os_file_n_pending_preads = 0; +/** Number of pending os_file_pwrite() operations */ +UNIV_INTERN ulint os_file_n_pending_pwrites = 0; +/** Number of pending write operations */ +UNIV_INTERN ulint os_n_pending_writes = 0; +/** Number of pending read operations */ +UNIV_INTERN ulint os_n_pending_reads = 0; + +/** After first fallocate failure we will disable os_file_trim */ +UNIV_INTERN ibool os_fallocate_failed = FALSE; + +/**********************************************************************//** +Directly manipulate the allocated disk space by deallocating for the file referred to +by fd for the byte range starting at offset and continuing for len bytes. +Within the specified range, partial file system blocks are zeroed, and whole +file system blocks are removed from the file. After a successful call, +subsequent reads from this range will return zeroes. +@return true if success, false if error */ +UNIV_INTERN +ibool +os_file_trim( +/*=========*/ + os_file_t file, /*!< in: file to be trimmed */ + os_aio_slot_t* slot, /*!< in: slot structure */ + ulint len); /*!< in: length of area */ + +/**********************************************************************//** +Allocate memory for temporal buffer used for page compression. This +buffer is freed later. */ +UNIV_INTERN +void +os_slot_alloc_page_buf( +/*===================*/ + os_aio_slot_t* slot); /*!< in: slot structure */ + +#ifdef HAVE_LZO +/**********************************************************************//** +Allocate memory for temporal memory used for page compression when +LZO compression method is used */ +UNIV_INTERN +void +os_slot_alloc_lzo_mem( +/*===================*/ + os_aio_slot_t* slot); /*!< in: slot structure */ +#endif + +/**********************************************************************//** +Allocate memory for temporal buffer used for page encryption. This +buffer is freed later. */ +UNIV_INTERN +void +os_slot_alloc_page_buf2( +os_aio_slot_t* slot); /*!< in: slot structure */ +/****************************************************************//** +Does error handling when a file operation fails. +@return TRUE if we should retry the operation */ +ibool +os_file_handle_error_no_exit( +/*=========================*/ + const char* name, /*!< in: name of a file or NULL */ + const char* operation, /*!< in: operation */ + ibool on_error_silent,/*!< in: if TRUE then don't print + any message to the log. */ + const char* file, /*!< in: file name */ + const ulint line); /*!< in: line */ + +/****************************************************************//** +Tries to enable the atomic write feature, if available, for the specified file +handle. +@return TRUE if success */ +static __attribute__((warn_unused_result)) +ibool +os_file_set_atomic_writes( +/*======================*/ + const char* name, /*!< in: name of the file */ + os_file_t file); /*!< in: handle to the file */ + +#ifdef UNIV_DEBUG +# ifndef UNIV_HOTBACKUP +/**********************************************************************//** +Validates the consistency the aio system some of the time. +@return TRUE if ok or the check was skipped */ +UNIV_INTERN +ibool +os_aio_validate_skip(void) +/*======================*/ +{ +/** Try os_aio_validate() every this many times */ +# define OS_AIO_VALIDATE_SKIP 13 + + /** The os_aio_validate() call skip counter. + Use a signed type because of the race condition below. */ + static int os_aio_validate_count = OS_AIO_VALIDATE_SKIP; + + /* There is a race condition below, but it does not matter, + because this call is only for heuristic purposes. We want to + reduce the call frequency of the costly os_aio_validate() + check in debug builds. */ + if (--os_aio_validate_count > 0) { + return(TRUE); + } + + os_aio_validate_count = OS_AIO_VALIDATE_SKIP; + return(os_aio_validate()); +} +# endif /* !UNIV_HOTBACKUP */ +#endif /* UNIV_DEBUG */ + +#ifdef _WIN32 +/** IO completion port used by background io threads */ +static HANDLE completion_port; +/** IO completion port used by background io READ threads */ +static HANDLE read_completion_port; +/** Thread local storage index for the per-thread event used for synchronous IO */ +static DWORD tls_sync_io = TLS_OUT_OF_INDEXES; +#endif + +#ifdef __WIN__ +/***********************************************************************//** +Gets the operating system version. Currently works only on Windows. +@return OS_WIN95, OS_WIN31, OS_WINNT, OS_WIN2000, OS_WINXP, OS_WINVISTA, +OS_WIN7. */ +UNIV_INTERN +ulint +os_get_os_version(void) +/*===================*/ +{ + OSVERSIONINFO os_info; + + os_info.dwOSVersionInfoSize = sizeof(OSVERSIONINFO); + + ut_a(GetVersionEx(&os_info)); + + if (os_info.dwPlatformId == VER_PLATFORM_WIN32s) { + return(OS_WIN31); + } else if (os_info.dwPlatformId == VER_PLATFORM_WIN32_WINDOWS) { + return(OS_WIN95); + } else if (os_info.dwPlatformId == VER_PLATFORM_WIN32_NT) { + switch (os_info.dwMajorVersion) { + case 3: + case 4: + return(OS_WINNT); + case 5: + return (os_info.dwMinorVersion == 0) + ? OS_WIN2000 : OS_WINXP; + case 6: + return (os_info.dwMinorVersion == 0) + ? OS_WINVISTA : OS_WIN7; + default: + return(OS_WIN7); + } + } else { + ut_error; + return(0); + } +} +#endif /* __WIN__ */ + + +#ifdef _WIN32 +/* +Windows : Handling synchronous IO on files opened asynchronously. + +If file is opened for asynchronous IO (FILE_FLAG_OVERLAPPED) and also bound to +a completion port, then every IO on this file would normally be enqueued to the +completion port. Sometimes however we would like to do a synchronous IO. This is +possible if we initialitze have overlapped.hEvent with a valid event and set its +lowest order bit to 1 (see MSDN ReadFile and WriteFile description for more info) + +We'll create this special event once for each thread and store in thread local +storage. +*/ + + +/***********************************************************************//** +Initialize tls index.for event handle used for synchronized IO on files that +might be opened with FILE_FLAG_OVERLAPPED. +*/ +static void win_init_syncio_event() +{ + tls_sync_io = TlsAlloc(); + ut_a(tls_sync_io != TLS_OUT_OF_INDEXES); +} + +/***********************************************************************//** +Retrieve per-thread event for doing synchronous io on asyncronously opened files +*/ +static HANDLE win_get_syncio_event() +{ + HANDLE h; + if(tls_sync_io == TLS_OUT_OF_INDEXES){ + win_init_syncio_event(); + } + + h = (HANDLE)TlsGetValue(tls_sync_io); + if (h) + return h; + h = CreateEventA(NULL, FALSE, FALSE, NULL); + ut_a(h); + h = (HANDLE)((uintptr_t)h | 1); + TlsSetValue(tls_sync_io, h); + return h; +} + +/* + TLS destructor, inspired by Chromium code + http://src.chromium.org/svn/trunk/src/base/threading/thread_local_storage_win.cc +*/ + +static void win_free_syncio_event() +{ + HANDLE h = win_get_syncio_event(); + if (h) { + CloseHandle(h); + } +} + +static void NTAPI win_tls_thread_exit(PVOID module, DWORD reason, PVOID reserved) { + if (DLL_THREAD_DETACH == reason || DLL_PROCESS_DETACH == reason) + win_free_syncio_event(); +} + +extern "C" { +#ifdef _WIN64 +#pragma comment(linker, "/INCLUDE:_tls_used") +#pragma comment(linker, "/INCLUDE:p_thread_callback_base") +#pragma const_seg(".CRT$XLB") +extern const PIMAGE_TLS_CALLBACK p_thread_callback_base; +const PIMAGE_TLS_CALLBACK p_thread_callback_base = win_tls_thread_exit; +#pragma data_seg() +#else +#pragma comment(linker, "/INCLUDE:__tls_used") +#pragma comment(linker, "/INCLUDE:_p_thread_callback_base") +#pragma data_seg(".CRT$XLB") +PIMAGE_TLS_CALLBACK p_thread_callback_base = win_tls_thread_exit; +#pragma data_seg() +#endif +} +#endif /*_WIN32 */ + +/***********************************************************************//** +Retrieves the last error number if an error occurs in a file io function. +The number should be retrieved before any other OS calls (because they may +overwrite the error number). If the number is not known to this program, +the OS error number + 100 is returned. +@return error number, or OS error number + 100 */ +static +ulint +os_file_get_last_error_low( +/*=======================*/ + bool report_all_errors, /*!< in: TRUE if we want an error + message printed of all errors */ + bool on_error_silent) /*!< in: TRUE then don't print any + diagnostic to the log */ +{ +#ifdef __WIN__ + + ulint err = (ulint) GetLastError(); + if (err == ERROR_SUCCESS) { + return(0); + } + + if (report_all_errors + || (!on_error_silent + && err != ERROR_DISK_FULL + && err != ERROR_FILE_EXISTS)) { + + ut_print_timestamp(stderr); + fprintf(stderr, + " InnoDB: Operating system error number %lu" + " in a file operation.\n", (ulong) err); + + if (err == ERROR_PATH_NOT_FOUND) { + fprintf(stderr, + "InnoDB: The error means the system" + " cannot find the path specified.\n"); + + if (srv_is_being_started) { + fprintf(stderr, + "InnoDB: If you are installing InnoDB," + " remember that you must create\n" + "InnoDB: directories yourself, InnoDB" + " does not create them.\n"); + } + } else if (err == ERROR_ACCESS_DENIED) { + fprintf(stderr, + "InnoDB: The error means mysqld does not have" + " the access rights to\n" + "InnoDB: the directory. It may also be" + " you have created a subdirectory\n" + "InnoDB: of the same name as a data file.\n"); + } else if (err == ERROR_SHARING_VIOLATION + || err == ERROR_LOCK_VIOLATION) { + fprintf(stderr, + "InnoDB: The error means that another program" + " is using InnoDB's files.\n" + "InnoDB: This might be a backup or antivirus" + " software or another instance\n" + "InnoDB: of MySQL." + " Please close it to get rid of this error.\n"); + } else if (err == ERROR_WORKING_SET_QUOTA + || err == ERROR_NO_SYSTEM_RESOURCES) { + fprintf(stderr, + "InnoDB: The error means that there are no" + " sufficient system resources or quota to" + " complete the operation.\n"); + } else if (err == ERROR_OPERATION_ABORTED) { + fprintf(stderr, + "InnoDB: The error means that the I/O" + " operation has been aborted\n" + "InnoDB: because of either a thread exit" + " or an application request.\n" + "InnoDB: Retry attempt is made.\n"); + } else if (err == ECANCELED || err == ENOTTY) { + if (strerror(err) != NULL) { + fprintf(stderr, + "InnoDB: Error number %d" + " means '%s'.\n", + err, strerror(err)); + } + + if(srv_use_atomic_writes) { + fprintf(stderr, + "InnoDB: Error trying to enable atomic writes on " + "non-supported destination!\n"); + } + } else { + fprintf(stderr, + "InnoDB: Some operating system error numbers" + " are described at\n" + "InnoDB: " + REFMAN + "operating-system-error-codes.html\n"); + } + } + + fflush(stderr); + + if (err == ERROR_FILE_NOT_FOUND) { + return(OS_FILE_NOT_FOUND); + } else if (err == ERROR_DISK_FULL) { + return(OS_FILE_DISK_FULL); + } else if (err == ERROR_FILE_EXISTS) { + return(OS_FILE_ALREADY_EXISTS); + } else if (err == ERROR_SHARING_VIOLATION + || err == ERROR_LOCK_VIOLATION) { + return(OS_FILE_SHARING_VIOLATION); + } else if (err == ERROR_WORKING_SET_QUOTA + || err == ERROR_NO_SYSTEM_RESOURCES) { + return(OS_FILE_INSUFFICIENT_RESOURCE); + } else if (err == ERROR_OPERATION_ABORTED) { + return(OS_FILE_OPERATION_ABORTED); + } else if (err == ERROR_ACCESS_DENIED) { + return(OS_FILE_ACCESS_VIOLATION); + } else { + return(OS_FILE_ERROR_MAX + err); + } +#else + int err = errno; + if (err == 0) { + return(0); + } + + if (report_all_errors + || (err != ENOSPC && err != EEXIST && !on_error_silent)) { + + ut_print_timestamp(stderr); + fprintf(stderr, + " InnoDB: Operating system error number %d" + " in a file operation.\n", err); + + if (err == ENOENT) { + fprintf(stderr, + "InnoDB: The error means the system" + " cannot find the path specified.\n"); + + if (srv_is_being_started) { + fprintf(stderr, + "InnoDB: If you are installing InnoDB," + " remember that you must create\n" + "InnoDB: directories yourself, InnoDB" + " does not create them.\n"); + } + } else if (err == EACCES) { + fprintf(stderr, + "InnoDB: The error means mysqld does not have" + " the access rights to\n" + "InnoDB: the directory.\n"); + } else if (err == ECANCELED || err == ENOTTY) { + if (strerror(err) != NULL) { + fprintf(stderr, + "InnoDB: Error number %d" + " means '%s'.\n", + err, strerror(err)); + } + + + if(srv_use_atomic_writes) { + fprintf(stderr, + "InnoDB: Error trying to enable atomic writes on " + "non-supported destination!\n"); + } + } else { + if (strerror(err) != NULL) { + fprintf(stderr, + "InnoDB: Error number %d" + " means '%s'.\n", + err, strerror(err)); + } + + + fprintf(stderr, + "InnoDB: Some operating system" + " error numbers are described at\n" + "InnoDB: " + REFMAN + "operating-system-error-codes.html\n"); + } + } + + fflush(stderr); + + switch (err) { + case ENOSPC: + return(OS_FILE_DISK_FULL); + case ENOENT: + return(OS_FILE_NOT_FOUND); + case EEXIST: + return(OS_FILE_ALREADY_EXISTS); + case EXDEV: + case ENOTDIR: + case EISDIR: + return(OS_FILE_PATH_ERROR); + case EAGAIN: + if (srv_use_native_aio) { + return(OS_FILE_AIO_RESOURCES_RESERVED); + } + break; + case ECANCELED: + case ENOTTY: + return(OS_FILE_OPERATION_NOT_SUPPORTED); + case EINTR: + if (srv_use_native_aio) { + return(OS_FILE_AIO_INTERRUPTED); + } + break; + case EACCES: + return(OS_FILE_ACCESS_VIOLATION); + } + return(OS_FILE_ERROR_MAX + err); +#endif +} + +/***********************************************************************//** +Retrieves the last error number if an error occurs in a file io function. +The number should be retrieved before any other OS calls (because they may +overwrite the error number). If the number is not known to this program, +the OS error number + 100 is returned. +@return error number, or OS error number + 100 */ +UNIV_INTERN +ulint +os_file_get_last_error( +/*===================*/ + bool report_all_errors) /*!< in: TRUE if we want an error + message printed of all errors */ +{ + return(os_file_get_last_error_low(report_all_errors, false)); +} + +/****************************************************************//** +Does error handling when a file operation fails. +Conditionally exits (calling exit(3)) based on should_exit value and the +error type, if should_exit is TRUE then on_error_silent is ignored. +@return TRUE if we should retry the operation */ +ibool +os_file_handle_error_cond_exit( +/*===========================*/ + const char* name, /*!< in: name of a file or NULL */ + const char* operation, /*!< in: operation */ + ibool should_exit, /*!< in: call exit(3) if unknown error + and this parameter is TRUE */ + ibool on_error_silent,/*!< in: if TRUE then don't print + any message to the log iff it is + an unknown non-fatal error */ + const char* file, /*!< in: file name */ + const ulint line) /*!< in: line */ +{ + ulint err; + + err = os_file_get_last_error_low(false, on_error_silent); + + switch (err) { + case OS_FILE_DISK_FULL: + /* We only print a warning about disk full once */ + + if (os_has_said_disk_full) { + + return(FALSE); + } + + /* Disk full error is reported irrespective of the + on_error_silent setting. */ + + if (name) { + ut_print_timestamp(stderr); + fprintf(stderr, + " InnoDB: Encountered a problem with" + " file %s\n", name); + } + + ut_print_timestamp(stderr); + fprintf(stderr, + " InnoDB: Disk is full. Try to clean the disk" + " to free space.\n"); + + os_has_said_disk_full = TRUE; + + fprintf(stderr, + " InnoDB: at file %s and at line %ld\n", file, line); + + fflush(stderr); + + return(FALSE); + + case OS_FILE_AIO_RESOURCES_RESERVED: + case OS_FILE_AIO_INTERRUPTED: + + return(TRUE); + + case OS_FILE_PATH_ERROR: + case OS_FILE_ALREADY_EXISTS: + case OS_FILE_ACCESS_VIOLATION: + + return(FALSE); + + case OS_FILE_SHARING_VIOLATION: + + os_thread_sleep(10000000); /* 10 sec */ + return(TRUE); + + case OS_FILE_OPERATION_ABORTED: + case OS_FILE_INSUFFICIENT_RESOURCE: + + os_thread_sleep(100000); /* 100 ms */ + return(TRUE); + + default: + + /* If it is an operation that can crash on error then it + is better to ignore on_error_silent and print an error message + to the log. */ + + if (should_exit || !on_error_silent) { + fprintf(stderr, + " InnoDB: Operation %s to file %s and at line %ld\n", + operation, file, line); + } + + if (should_exit || !on_error_silent) { + ib_logf(IB_LOG_LEVEL_ERROR, "File %s: '%s' returned OS " + "error " ULINTPF ".%s", name ? name : "(unknown)", + operation, err, should_exit + ? " Cannot continue operation" : ""); + } + + if (should_exit) { + exit(1); + } + } + + return(FALSE); +} + +/****************************************************************//** +Does error handling when a file operation fails. +@return TRUE if we should retry the operation */ +static +ibool +os_file_handle_error( +/*=================*/ + const char* name, /*!< in: name of a file or NULL */ + const char* operation, /*!< in: operation */ + const char* file, /*!< in: file name */ + const ulint line) /*!< in: line */ +{ + /* exit in case of unknown error */ + return(os_file_handle_error_cond_exit(name, operation, TRUE, FALSE, file, line)); +} + +/****************************************************************//** +Does error handling when a file operation fails. +@return TRUE if we should retry the operation */ +ibool +os_file_handle_error_no_exit( +/*=========================*/ + const char* name, /*!< in: name of a file or NULL */ + const char* operation, /*!< in: operation */ + ibool on_error_silent,/*!< in: if TRUE then don't print + any message to the log. */ + const char* file, /*!< in: file name */ + const ulint line) /*!< in: line */ +{ + /* don't exit in case of unknown error */ + return(os_file_handle_error_cond_exit( + name, operation, FALSE, on_error_silent, file, line)); +} + +#undef USE_FILE_LOCK +#define USE_FILE_LOCK +#if defined(UNIV_HOTBACKUP) || defined(__WIN__) +/* InnoDB Hot Backup does not lock the data files. + * On Windows, mandatory locking is used. + */ +# undef USE_FILE_LOCK +#endif +#ifdef USE_FILE_LOCK +/****************************************************************//** +Obtain an exclusive lock on a file. +@return 0 on success */ +static +int +os_file_lock( +/*=========*/ + int fd, /*!< in: file descriptor */ + const char* name) /*!< in: file name */ +{ + struct flock lk; + + ut_ad(!srv_read_only_mode); + + lk.l_type = F_WRLCK; + lk.l_whence = SEEK_SET; + lk.l_start = lk.l_len = 0; + + if (fcntl(fd, F_SETLK, &lk) == -1) { + + ib_logf(IB_LOG_LEVEL_ERROR, + "Unable to lock %s, error: %d", name, errno); + + if (errno == EAGAIN || errno == EACCES) { + ib_logf(IB_LOG_LEVEL_INFO, + "Check that you do not already have " + "another mysqld process using the " + "same InnoDB data or log files."); + } + + return(-1); + } + + return(0); +} +#endif /* USE_FILE_LOCK */ + +#ifndef UNIV_HOTBACKUP +/****************************************************************//** +Creates the seek mutexes used in positioned reads and writes. */ +UNIV_INTERN +void +os_io_init_simple(void) +/*===================*/ +{ +#if !defined(HAVE_ATOMIC_BUILTINS) || UNIV_WORD_SIZE < 8 + os_file_count_mutex = os_mutex_create(); +#endif /* !HAVE_ATOMIC_BUILTINS || UNIV_WORD_SIZE < 8 */ + + for (ulint i = 0; i < OS_FILE_N_SEEK_MUTEXES; i++) { + os_file_seek_mutexes[i] = os_mutex_create(); + } +#ifdef _WIN32 + win_init_syncio_event(); +#endif +} + +/***********************************************************************//** +Creates a temporary file. This function is like tmpfile(3), but +the temporary file is created in the MySQL temporary directory. +@return temporary file handle, or NULL on error */ +UNIV_INTERN +FILE* +os_file_create_tmpfile(void) +/*========================*/ +{ + FILE* file = NULL; + int fd; + WAIT_ALLOW_WRITES(); + fd = innobase_mysql_tmpfile(); + + ut_ad(!srv_read_only_mode); + + if (fd >= 0) { + file = fdopen(fd, "w+b"); + } + + if (!file) { + ut_print_timestamp(stderr); + fprintf(stderr, + " InnoDB: Error: unable to create temporary file;" + " errno: %d\n", errno); + if (fd >= 0) { + close(fd); + } + } + + return(file); +} +#endif /* !UNIV_HOTBACKUP */ + +/***********************************************************************//** +The os_file_opendir() function opens a directory stream corresponding to the +directory named by the dirname argument. The directory stream is positioned +at the first entry. In both Unix and Windows we automatically skip the '.' +and '..' items at the start of the directory listing. +@return directory stream, NULL if error */ +UNIV_INTERN +os_file_dir_t +os_file_opendir( +/*============*/ + const char* dirname, /*!< in: directory name; it must not + contain a trailing '\' or '/' */ + ibool error_is_fatal) /*!< in: TRUE if we should treat an + error as a fatal error; if we try to + open symlinks then we do not wish a + fatal error if it happens not to be + a directory */ +{ + os_file_dir_t dir; +#ifdef __WIN__ + LPWIN32_FIND_DATA lpFindFileData; + char path[OS_FILE_MAX_PATH + 3]; + + ut_a(strlen(dirname) < OS_FILE_MAX_PATH); + + strcpy(path, dirname); + strcpy(path + strlen(path), "\\*"); + + /* Note that in Windows opening the 'directory stream' also retrieves + the first entry in the directory. Since it is '.', that is no problem, + as we will skip over the '.' and '..' entries anyway. */ + + lpFindFileData = static_cast( + ut_malloc(sizeof(WIN32_FIND_DATA))); + + dir = FindFirstFile((LPCTSTR) path, lpFindFileData); + + ut_free(lpFindFileData); + + if (dir == INVALID_HANDLE_VALUE) { + + if (error_is_fatal) { + os_file_handle_error(dirname, "opendir", __FILE__, __LINE__); + } + + return(NULL); + } + + return(dir); +#else + dir = opendir(dirname); + + if (dir == NULL && error_is_fatal) { + os_file_handle_error(dirname, "opendir", __FILE__, __LINE__); + } + + return(dir); +#endif /* __WIN__ */ +} + +/***********************************************************************//** +Closes a directory stream. +@return 0 if success, -1 if failure */ +UNIV_INTERN +int +os_file_closedir( +/*=============*/ + os_file_dir_t dir) /*!< in: directory stream */ +{ +#ifdef __WIN__ + BOOL ret; + + ret = FindClose(dir); + + if (!ret) { + os_file_handle_error_no_exit(NULL, "closedir", FALSE, __FILE__, __LINE__); + + return(-1); + } + + return(0); +#else + int ret; + + ret = closedir(dir); + + if (ret) { + os_file_handle_error_no_exit(NULL, "closedir", FALSE, __FILE__, __LINE__); + } + + return(ret); +#endif /* __WIN__ */ +} + +/***********************************************************************//** +This function returns information of the next file in the directory. We jump +over the '.' and '..' entries in the directory. +@return 0 if ok, -1 if error, 1 if at the end of the directory */ +UNIV_INTERN +int +os_file_readdir_next_file( +/*======================*/ + const char* dirname,/*!< in: directory name or path */ + os_file_dir_t dir, /*!< in: directory stream */ + os_file_stat_t* info) /*!< in/out: buffer where the info is returned */ +{ +#ifdef __WIN__ + LPWIN32_FIND_DATA lpFindFileData; + BOOL ret; + + lpFindFileData = static_cast( + ut_malloc(sizeof(WIN32_FIND_DATA))); +next_file: + ret = FindNextFile(dir, lpFindFileData); + + if (ret) { + ut_a(strlen((char*) lpFindFileData->cFileName) + < OS_FILE_MAX_PATH); + + if (strcmp((char*) lpFindFileData->cFileName, ".") == 0 + || strcmp((char*) lpFindFileData->cFileName, "..") == 0) { + + goto next_file; + } + + strcpy(info->name, (char*) lpFindFileData->cFileName); + + info->size = (ib_int64_t)(lpFindFileData->nFileSizeLow) + + (((ib_int64_t)(lpFindFileData->nFileSizeHigh)) + << 32); + + if (lpFindFileData->dwFileAttributes + & FILE_ATTRIBUTE_REPARSE_POINT) { + /* TODO: test Windows symlinks */ + /* TODO: MySQL has apparently its own symlink + implementation in Windows, dbname.sym can + redirect a database directory: + REFMAN "windows-symbolic-links.html" */ + info->type = OS_FILE_TYPE_LINK; + } else if (lpFindFileData->dwFileAttributes + & FILE_ATTRIBUTE_DIRECTORY) { + info->type = OS_FILE_TYPE_DIR; + } else { + /* It is probably safest to assume that all other + file types are normal. Better to check them rather + than blindly skip them. */ + + info->type = OS_FILE_TYPE_FILE; + } + } + + ut_free(lpFindFileData); + + if (ret) { + return(0); + } else if (GetLastError() == ERROR_NO_MORE_FILES) { + + return(1); + } else { + os_file_handle_error_no_exit(NULL, "readdir_next_file", FALSE, __FILE__, __LINE__); + return(-1); + } +#else + struct dirent* ent; + char* full_path; + int ret; + struct stat statinfo; +#ifdef HAVE_READDIR_R + char dirent_buf[sizeof(struct dirent) + + _POSIX_PATH_MAX + 100]; + /* In /mysys/my_lib.c, _POSIX_PATH_MAX + 1 is used as + the max file name len; but in most standards, the + length is NAME_MAX; we add 100 to be even safer */ +#endif + +next_file: + +#ifdef HAVE_READDIR_R + ret = readdir_r(dir, (struct dirent*) dirent_buf, &ent); + + if (ret != 0 +#ifdef UNIV_AIX + /* On AIX, only if we got non-NULL 'ent' (result) value and + a non-zero 'ret' (return) value, it indicates a failed + readdir_r() call. An NULL 'ent' with an non-zero 'ret' + would indicate the "end of the directory" is reached. */ + && ent != NULL +#endif + ) { + fprintf(stderr, + "InnoDB: cannot read directory %s, error %lu\n", + dirname, (ulong) ret); + + return(-1); + } + + if (ent == NULL) { + /* End of directory */ + + return(1); + } + + ut_a(strlen(ent->d_name) < _POSIX_PATH_MAX + 100 - 1); +#else + ent = readdir(dir); + + if (ent == NULL) { + + return(1); + } +#endif + ut_a(strlen(ent->d_name) < OS_FILE_MAX_PATH); + + if (strcmp(ent->d_name, ".") == 0 || strcmp(ent->d_name, "..") == 0) { + + goto next_file; + } + + strcpy(info->name, ent->d_name); + + full_path = static_cast( + ut_malloc(strlen(dirname) + strlen(ent->d_name) + 10)); + + sprintf(full_path, "%s/%s", dirname, ent->d_name); + + ret = stat(full_path, &statinfo); + + if (ret) { + + if (errno == ENOENT) { + /* readdir() returned a file that does not exist, + it must have been deleted in the meantime. Do what + would have happened if the file was deleted before + readdir() - ignore and go to the next entry. + If this is the last entry then info->name will still + contain the name of the deleted file when this + function returns, but this is not an issue since the + caller shouldn't be looking at info when end of + directory is returned. */ + + ut_free(full_path); + + goto next_file; + } + + os_file_handle_error_no_exit(full_path, "stat", FALSE, __FILE__, __LINE__); + + ut_free(full_path); + + return(-1); + } + + info->size = (ib_int64_t) statinfo.st_size; + + if (S_ISDIR(statinfo.st_mode)) { + info->type = OS_FILE_TYPE_DIR; + } else if (S_ISLNK(statinfo.st_mode)) { + info->type = OS_FILE_TYPE_LINK; + } else if (S_ISREG(statinfo.st_mode)) { + info->type = OS_FILE_TYPE_FILE; + } else { + info->type = OS_FILE_TYPE_UNKNOWN; + } + + ut_free(full_path); + + return(0); +#endif +} + +/*****************************************************************//** +This function attempts to create a directory named pathname. The new +directory gets default permissions. On Unix the permissions are +(0770 & ~umask). If the directory exists already, nothing is done and +the call succeeds, unless the fail_if_exists arguments is true. +If another error occurs, such as a permission error, this does not crash, +but reports the error and returns FALSE. +@return TRUE if call succeeds, FALSE on error */ +UNIV_INTERN +ibool +os_file_create_directory( +/*=====================*/ + const char* pathname, /*!< in: directory name as + null-terminated string */ + ibool fail_if_exists) /*!< in: if TRUE, pre-existing directory + is treated as an error. */ +{ +#ifdef __WIN__ + BOOL rcode; + + rcode = CreateDirectory((LPCTSTR) pathname, NULL); + if (!(rcode != 0 + || (GetLastError() == ERROR_ALREADY_EXISTS + && !fail_if_exists))) { + + os_file_handle_error_no_exit( + pathname, "CreateDirectory", FALSE, __FILE__, __LINE__); + + return(FALSE); + } + + return(TRUE); +#else + int rcode; + WAIT_ALLOW_WRITES(); + + rcode = mkdir(pathname, 0770); + + if (!(rcode == 0 || (errno == EEXIST && !fail_if_exists))) { + /* failure */ + os_file_handle_error_no_exit(pathname, "mkdir", FALSE, __FILE__, __LINE__); + + return(FALSE); + } + + return (TRUE); +#endif /* __WIN__ */ +} + +/****************************************************************//** +NOTE! Use the corresponding macro os_file_create_simple(), not directly +this function! +A simple function to open or create a file. +@return own: handle to the file, not defined if error, error number +can be retrieved with os_file_get_last_error */ +UNIV_INTERN +os_file_t +os_file_create_simple_func( +/*=======================*/ + const char* name, /*!< in: name of the file or path as a + null-terminated string */ + ulint create_mode,/*!< in: create mode */ + ulint access_type,/*!< in: OS_FILE_READ_ONLY or + OS_FILE_READ_WRITE */ + ibool* success)/*!< out: TRUE if succeed, FALSE if error */ +{ + os_file_t file; + ibool retry; + + *success = FALSE; +#ifdef __WIN__ + DWORD access; + DWORD create_flag; + DWORD attributes = 0; + + ut_a(!(create_mode & OS_FILE_ON_ERROR_SILENT)); + ut_a(!(create_mode & OS_FILE_ON_ERROR_NO_EXIT)); + + if (create_mode == OS_FILE_OPEN) { + + create_flag = OPEN_EXISTING; + + } else if (srv_read_only_mode) { + + create_flag = OPEN_EXISTING; + + } else if (create_mode == OS_FILE_CREATE) { + + create_flag = CREATE_NEW; + + } else if (create_mode == OS_FILE_CREATE_PATH) { + + ut_a(!srv_read_only_mode); + + /* Create subdirs along the path if needed */ + *success = os_file_create_subdirs_if_needed(name); + + if (!*success) { + + ib_logf(IB_LOG_LEVEL_ERROR, + "Unable to create subdirectories '%s'", + name); + + return((os_file_t) -1); + } + + create_flag = CREATE_NEW; + create_mode = OS_FILE_CREATE; + + } else { + ib_logf(IB_LOG_LEVEL_ERROR, + "Unknown file create mode (%lu) for file '%s'", + create_mode, name); + + return((os_file_t) -1); + } + + if (access_type == OS_FILE_READ_ONLY) { + access = GENERIC_READ; + } else if (srv_read_only_mode) { + + ib_logf(IB_LOG_LEVEL_INFO, + "read only mode set. Unable to " + "open file '%s' in RW mode, trying RO mode", name); + + access = GENERIC_READ; + + } else if (access_type == OS_FILE_READ_WRITE) { + access = GENERIC_READ | GENERIC_WRITE; + } else { + ib_logf(IB_LOG_LEVEL_ERROR, + "Unknown file access type (%lu) for file '%s'", + access_type, name); + + return((os_file_t) -1); + } + + do { + /* Use default security attributes and no template file. */ + + file = CreateFile( + (LPCTSTR) name, access, FILE_SHARE_READ, NULL, + create_flag, attributes, NULL); + + if (file == INVALID_HANDLE_VALUE) { + + *success = FALSE; + + retry = os_file_handle_error( + name, create_mode == OS_FILE_OPEN ? + "open" : "create", __FILE__, __LINE__); + + } else { + *success = TRUE; + retry = false; + } + + } while (retry); + +#else /* __WIN__ */ + int create_flag; + if (create_mode != OS_FILE_OPEN && create_mode != OS_FILE_OPEN_RAW) + WAIT_ALLOW_WRITES(); + + ut_a(!(create_mode & OS_FILE_ON_ERROR_SILENT)); + ut_a(!(create_mode & OS_FILE_ON_ERROR_NO_EXIT)); + + if (create_mode == OS_FILE_OPEN) { + + if (access_type == OS_FILE_READ_ONLY) { + create_flag = O_RDONLY; + } else if (srv_read_only_mode) { + create_flag = O_RDONLY; + } else { + create_flag = O_RDWR; + } + + } else if (srv_read_only_mode) { + + create_flag = O_RDONLY; + + } else if (create_mode == OS_FILE_CREATE) { + + create_flag = O_RDWR | O_CREAT | O_EXCL; + + } else if (create_mode == OS_FILE_CREATE_PATH) { + + /* Create subdirs along the path if needed */ + + *success = os_file_create_subdirs_if_needed(name); + + if (!*success) { + + ib_logf(IB_LOG_LEVEL_ERROR, + "Unable to create subdirectories '%s'", + name); + + return((os_file_t) -1); + } + + create_flag = O_RDWR | O_CREAT | O_EXCL; + create_mode = OS_FILE_CREATE; + } else { + + ib_logf(IB_LOG_LEVEL_ERROR, + "Unknown file create mode (%lu) for file '%s'", + create_mode, name); + + return((os_file_t) -1); + } + + do { + file = ::open(name, create_flag, os_innodb_umask); + + if (file == -1) { + *success = FALSE; + + retry = os_file_handle_error( + name, + create_mode == OS_FILE_OPEN + ? "open" : "create", __FILE__, __LINE__); + } else { + *success = TRUE; + retry = false; + } + + } while (retry); + +#ifdef USE_FILE_LOCK + if (!srv_read_only_mode + && *success + && access_type == OS_FILE_READ_WRITE + && os_file_lock(file, name)) { + + *success = FALSE; + close(file); + file = -1; + } +#endif /* USE_FILE_LOCK */ + +#endif /* __WIN__ */ + + return(file); +} + +/****************************************************************//** +NOTE! Use the corresponding macro +os_file_create_simple_no_error_handling(), not directly this function! +A simple function to open or create a file. +@return own: handle to the file, not defined if error, error number +can be retrieved with os_file_get_last_error */ +UNIV_INTERN +os_file_t +os_file_create_simple_no_error_handling_func( +/*=========================================*/ + const char* name, /*!< in: name of the file or path as a + null-terminated string */ + ulint create_mode,/*!< in: create mode */ + ulint access_type,/*!< in: OS_FILE_READ_ONLY, + OS_FILE_READ_WRITE, or + OS_FILE_READ_ALLOW_DELETE; the last option is + used by a backup program reading the file */ + ibool* success,/*!< out: TRUE if succeed, FALSE if error */ + ulint atomic_writes) /*! in: atomic writes table option + value */ +{ + os_file_t file; + atomic_writes_t awrites = (atomic_writes_t) atomic_writes; + + *success = FALSE; +#ifdef __WIN__ + DWORD access; + DWORD create_flag; + DWORD attributes = 0; + DWORD share_mode = FILE_SHARE_READ; + + ut_a(name); + + ut_a(!(create_mode & OS_FILE_ON_ERROR_SILENT)); + ut_a(!(create_mode & OS_FILE_ON_ERROR_NO_EXIT)); + + if (create_mode == OS_FILE_OPEN) { + create_flag = OPEN_EXISTING; + } else if (srv_read_only_mode) { + create_flag = OPEN_EXISTING; + } else if (create_mode == OS_FILE_CREATE) { + create_flag = CREATE_NEW; + } else { + + ib_logf(IB_LOG_LEVEL_ERROR, + "Unknown file create mode (%lu) for file '%s'", + create_mode, name); + + return((os_file_t) -1); + } + + if (access_type == OS_FILE_READ_ONLY) { + access = GENERIC_READ; + } else if (srv_read_only_mode) { + access = GENERIC_READ; + } else if (access_type == OS_FILE_READ_WRITE) { + access = GENERIC_READ | GENERIC_WRITE; + } else if (access_type == OS_FILE_READ_ALLOW_DELETE) { + + ut_a(!srv_read_only_mode); + + access = GENERIC_READ; + + /*!< A backup program has to give mysqld the maximum + freedom to do what it likes with the file */ + + share_mode |= FILE_SHARE_DELETE | FILE_SHARE_WRITE; + } else { + ib_logf(IB_LOG_LEVEL_ERROR, + "Unknown file access type (%lu) for file '%s'", + access_type, name); + + return((os_file_t) -1); + } + + file = CreateFile((LPCTSTR) name, + access, + share_mode, + NULL, // Security attributes + create_flag, + attributes, + NULL); // No template file + + /* If we have proper file handle and atomic writes should be used, + try to set atomic writes and if that fails when creating a new + table, produce a error. If atomic writes are used on existing + file, ignore error and use traditional writes for that file */ + if (file != INVALID_HANDLE_VALUE + && (awrites == ATOMIC_WRITES_ON || + (srv_use_atomic_writes && awrites == ATOMIC_WRITES_DEFAULT)) + && !os_file_set_atomic_writes(name, file)) { + if (create_mode == OS_FILE_CREATE) { + fprintf(stderr, "InnoDB: Error: Can't create file using atomic writes\n"); + CloseHandle(file); + os_file_delete_if_exists_func(name); + *success = FALSE; + file = INVALID_HANDLE_VALUE; + } + } + + *success = (file != INVALID_HANDLE_VALUE); +#else /* __WIN__ */ + int create_flag; + + ut_a(name); + if (create_mode != OS_FILE_OPEN && create_mode != OS_FILE_OPEN_RAW) + WAIT_ALLOW_WRITES(); + + ut_a(!(create_mode & OS_FILE_ON_ERROR_SILENT)); + ut_a(!(create_mode & OS_FILE_ON_ERROR_NO_EXIT)); + + if (create_mode == OS_FILE_OPEN) { + + if (access_type == OS_FILE_READ_ONLY) { + + create_flag = O_RDONLY; + + } else if (srv_read_only_mode) { + + create_flag = O_RDONLY; + + } else { + + ut_a(access_type == OS_FILE_READ_WRITE + || access_type == OS_FILE_READ_ALLOW_DELETE); + + create_flag = O_RDWR; + } + + } else if (srv_read_only_mode) { + + create_flag = O_RDONLY; + + } else if (create_mode == OS_FILE_CREATE) { + + create_flag = O_RDWR | O_CREAT | O_EXCL; + + } else { + ib_logf(IB_LOG_LEVEL_ERROR, + "Unknown file create mode (%lu) for file '%s'", + create_mode, name); + + return((os_file_t) -1); + } + + file = ::open(name, create_flag, os_innodb_umask); + + *success = file == -1 ? FALSE : TRUE; + +#ifdef USE_FILE_LOCK + if (!srv_read_only_mode + && *success + && access_type == OS_FILE_READ_WRITE + && os_file_lock(file, name)) { + + *success = FALSE; + close(file); + file = -1; + + } +#endif /* USE_FILE_LOCK */ + + /* If we have proper file handle and atomic writes should be used, + try to set atomic writes and if that fails when creating a new + table, produce a error. If atomic writes are used on existing + file, ignore error and use traditional writes for that file */ + if (file != -1 + && (awrites == ATOMIC_WRITES_ON || + (srv_use_atomic_writes && awrites == ATOMIC_WRITES_DEFAULT)) + && !os_file_set_atomic_writes(name, file)) { + if (create_mode == OS_FILE_CREATE) { + fprintf(stderr, "InnoDB: Error: Can't create file using atomic writes\n"); + close(file); + os_file_delete_if_exists_func(name); + *success = FALSE; + file = -1; + } + } + +#endif /* __WIN__ */ + + return(file); +} + +/****************************************************************//** +Tries to disable OS caching on an opened file descriptor. */ +UNIV_INTERN +void +os_file_set_nocache( +/*================*/ + int fd /*!< in: file descriptor to alter */ + __attribute__((unused)), + const char* file_name /*!< in: used in the diagnostic + message */ + __attribute__((unused)), + const char* operation_name __attribute__((unused))) + /*!< in: "open" or "create"; used + in the diagnostic message */ +{ + /* some versions of Solaris may not have DIRECTIO_ON */ +#if defined(UNIV_SOLARIS) && defined(DIRECTIO_ON) + if (directio(fd, DIRECTIO_ON) == -1) { + int errno_save = errno; + + ib_logf(IB_LOG_LEVEL_ERROR, + "Failed to set DIRECTIO_ON on file %s: %s: %s, " + "continuing anyway.", + file_name, operation_name, strerror(errno_save)); + } +#elif defined(O_DIRECT) + if (fcntl(fd, F_SETFL, O_DIRECT) == -1) { + int errno_save = errno; + static bool warning_message_printed = false; + if (errno_save == EINVAL) { + if (!warning_message_printed) { + warning_message_printed = true; +# ifdef UNIV_LINUX + ib_logf(IB_LOG_LEVEL_WARN, + "Failed to set O_DIRECT on file " + "%s: %s: %s, continuing anyway. " + "O_DIRECT is known to result " + "in 'Invalid argument' on Linux on " + "tmpfs, see MySQL Bug#26662.", + file_name, operation_name, + strerror(errno_save)); +# else /* UNIV_LINUX */ + goto short_warning; +# endif /* UNIV_LINUX */ + } + } else { +# ifndef UNIV_LINUX +short_warning: +# endif + ib_logf(IB_LOG_LEVEL_WARN, + "Failed to set O_DIRECT on file %s: %s: %s, " + "continuing anyway.", + file_name, operation_name, strerror(errno_save)); + } + } +#endif /* defined(UNIV_SOLARIS) && defined(DIRECTIO_ON) */ +} + + +/****************************************************************//** +Tries to enable the atomic write feature, if available, for the specified file +handle. +@return TRUE if success */ +static __attribute__((warn_unused_result)) +ibool +os_file_set_atomic_writes( +/*======================*/ + const char* name /*!< in: name of the file */ + __attribute__((unused)), + os_file_t file /*!< in: handle to the file */ + __attribute__((unused))) + +{ +#ifdef DFS_IOCTL_ATOMIC_WRITE_SET + int atomic_option = 1; + + if (ioctl(file, DFS_IOCTL_ATOMIC_WRITE_SET, &atomic_option)) { + + fprintf(stderr, "InnoDB: Warning:Trying to enable atomic writes on " + "file %s on non-supported platform!\n", name); + os_file_handle_error_no_exit(name, "ioctl(DFS_IOCTL_ATOMIC_WRITE_SET)", FALSE, __FILE__, __LINE__); + return(FALSE); + } + + return(TRUE); +#else + fprintf(stderr, "InnoDB: Error: trying to enable atomic writes on " + "file %s on non-supported platform!\n", name); + return(FALSE); +#endif +} + +/****************************************************************//** +NOTE! Use the corresponding macro os_file_create(), not directly +this function! +Opens an existing file or creates a new. +@return own: handle to the file, not defined if error, error number +can be retrieved with os_file_get_last_error */ +UNIV_INTERN +os_file_t +os_file_create_func( +/*================*/ + const char* name, /*!< in: name of the file or path as a + null-terminated string */ + ulint create_mode,/*!< in: create mode */ + ulint purpose,/*!< in: OS_FILE_AIO, if asynchronous, + non-buffered i/o is desired, + OS_FILE_NORMAL, if any normal file; + NOTE that it also depends on type, os_aio_.. + and srv_.. variables whether we really use + async i/o or unbuffered i/o: look in the + function source code for the exact rules */ + ulint type, /*!< in: OS_DATA_FILE or OS_LOG_FILE */ + ibool* success,/*!< out: TRUE if succeed, FALSE if error */ + ulint atomic_writes) /*! in: atomic writes table option + value */ +{ + os_file_t file; + ibool retry; + ibool on_error_no_exit; + ibool on_error_silent; + atomic_writes_t awrites = (atomic_writes_t) atomic_writes; + +#ifdef __WIN__ + DBUG_EXECUTE_IF( + "ib_create_table_fail_disk_full", + *success = FALSE; + SetLastError(ERROR_DISK_FULL); + return((os_file_t) -1); + ); +#else /* __WIN__ */ + DBUG_EXECUTE_IF( + "ib_create_table_fail_disk_full", + *success = FALSE; + errno = ENOSPC; + return((os_file_t) -1); + ); +#endif /* __WIN__ */ + +#ifdef __WIN__ + DWORD create_flag; + DWORD share_mode = FILE_SHARE_READ; + + on_error_no_exit = create_mode & OS_FILE_ON_ERROR_NO_EXIT + ? TRUE : FALSE; + + on_error_silent = create_mode & OS_FILE_ON_ERROR_SILENT + ? TRUE : FALSE; + + create_mode &= ~OS_FILE_ON_ERROR_NO_EXIT; + create_mode &= ~OS_FILE_ON_ERROR_SILENT; + + if (create_mode == OS_FILE_OPEN_RAW) { + + ut_a(!srv_read_only_mode); + + create_flag = OPEN_EXISTING; + + /* On Windows Physical devices require admin privileges and + have to have the write-share mode set. See the remarks + section for the CreateFile() function documentation in MSDN. */ + + share_mode |= FILE_SHARE_WRITE; + + } else if (create_mode == OS_FILE_OPEN + || create_mode == OS_FILE_OPEN_RETRY) { + + create_flag = OPEN_EXISTING; + + } else if (srv_read_only_mode) { + + create_flag = OPEN_EXISTING; + + } else if (create_mode == OS_FILE_CREATE) { + + create_flag = CREATE_NEW; + + } else if (create_mode == OS_FILE_OVERWRITE) { + + create_flag = CREATE_ALWAYS; + + } else { + ib_logf(IB_LOG_LEVEL_ERROR, + "Unknown file create mode (%lu) for file '%s'", + create_mode, name); + + return((os_file_t) -1); + } + + DWORD attributes = 0; + +#ifdef UNIV_HOTBACKUP + attributes |= FILE_FLAG_NO_BUFFERING; +#else + if (purpose == OS_FILE_AIO) { + +#ifdef WIN_ASYNC_IO + /* If specified, use asynchronous (overlapped) io and no + buffering of writes in the OS */ + + if (srv_use_native_aio) { + attributes |= FILE_FLAG_OVERLAPPED; + } +#endif /* WIN_ASYNC_IO */ + + } else if (purpose == OS_FILE_NORMAL) { + /* Use default setting. */ + } else { + ib_logf(IB_LOG_LEVEL_ERROR, + "Unknown purpose flag (%lu) while opening file '%s'", + purpose, name); + + return((os_file_t)(-1)); + } + +#ifdef UNIV_NON_BUFFERED_IO + // TODO: Create a bug, this looks wrong. The flush log + // parameter is dynamic. + if (type == OS_LOG_FILE && thd_flush_log_at_trx_commit(NULL) == 2) { + + /* Do not use unbuffered i/o for the log files because + value 2 denotes that we do not flush the log at every + commit, but only once per second */ + + } else if (srv_win_file_flush_method == SRV_WIN_IO_UNBUFFERED) { + + attributes |= FILE_FLAG_NO_BUFFERING; + } +#endif /* UNIV_NON_BUFFERED_IO */ + +#endif /* UNIV_HOTBACKUP */ + DWORD access = GENERIC_READ; + + if (!srv_read_only_mode) { + access |= GENERIC_WRITE; + } + + if (type == OS_LOG_FILE) { + if (srv_unix_file_flush_method == SRV_UNIX_O_DSYNC) { + /* Map O_DSYNC to WRITE_THROUGH */ + attributes |= FILE_FLAG_WRITE_THROUGH; + } else if (srv_unix_file_flush_method == SRV_UNIX_ALL_O_DIRECT) { + /* Open log file without buffering */ + attributes |= FILE_FLAG_NO_BUFFERING; + } + } + + do { + /* Use default security attributes and no template file. */ + file = CreateFile( + (LPCTSTR) name, access, share_mode, NULL, + create_flag, attributes, NULL); + + if (file == INVALID_HANDLE_VALUE) { + const char* operation; + + operation = (create_mode == OS_FILE_CREATE + && !srv_read_only_mode) + ? "create" : "open"; + + *success = FALSE; + + if (on_error_no_exit) { + retry = os_file_handle_error_no_exit( + name, operation, on_error_silent, __FILE__, __LINE__); + } else { + retry = os_file_handle_error(name, operation, __FILE__, __LINE__); + } + } else { + *success = TRUE; + retry = FALSE; + if (srv_use_native_aio && ((attributes & FILE_FLAG_OVERLAPPED) != 0)) { + ut_a(CreateIoCompletionPort(file, completion_port, 0, 0)); + } + } + + } while (retry); + + /* If we have proper file handle and atomic writes should be used, + try to set atomic writes and if that fails when creating a new + table, produce a error. If atomic writes are used on existing + file, ignore error and use traditional writes for that file */ + if (file != INVALID_HANDLE_VALUE + && (awrites == ATOMIC_WRITES_ON || + (srv_use_atomic_writes && awrites == ATOMIC_WRITES_DEFAULT)) + && !os_file_set_atomic_writes(name, file)) { + if (create_mode == OS_FILE_CREATE) { + fprintf(stderr, "InnoDB: Error: Can't create file using atomic writes\n"); + CloseHandle(file); + os_file_delete_if_exists_func(name); + *success = FALSE; + file = INVALID_HANDLE_VALUE; + } + } + +#else /* __WIN__ */ + int create_flag; + const char* mode_str = NULL; + if (create_mode != OS_FILE_OPEN && create_mode != OS_FILE_OPEN_RAW) + WAIT_ALLOW_WRITES(); + + on_error_no_exit = create_mode & OS_FILE_ON_ERROR_NO_EXIT + ? TRUE : FALSE; + on_error_silent = create_mode & OS_FILE_ON_ERROR_SILENT + ? TRUE : FALSE; + + create_mode &= ~OS_FILE_ON_ERROR_NO_EXIT; + create_mode &= ~OS_FILE_ON_ERROR_SILENT; + + if (create_mode == OS_FILE_OPEN + || create_mode == OS_FILE_OPEN_RAW + || create_mode == OS_FILE_OPEN_RETRY) { + + mode_str = "OPEN"; + + create_flag = srv_read_only_mode ? O_RDONLY : O_RDWR; + + } else if (srv_read_only_mode) { + + mode_str = "OPEN"; + + create_flag = O_RDONLY; + + } else if (create_mode == OS_FILE_CREATE) { + + mode_str = "CREATE"; + create_flag = O_RDWR | O_CREAT | O_EXCL; + + } else if (create_mode == OS_FILE_OVERWRITE) { + + mode_str = "OVERWRITE"; + create_flag = O_RDWR | O_CREAT | O_TRUNC; + + } else { + ib_logf(IB_LOG_LEVEL_ERROR, + "Unknown file create mode (%lu) for file '%s'", + create_mode, name); + + return((os_file_t) -1); + } + + ut_a(type == OS_LOG_FILE || type == OS_DATA_FILE); + ut_a(purpose == OS_FILE_AIO || purpose == OS_FILE_NORMAL); + +#ifdef O_SYNC + /* We let O_SYNC only affect log files; note that we map O_DSYNC to + O_SYNC because the datasync options seemed to corrupt files in 2001 + in both Linux and Solaris */ + + if (!srv_read_only_mode + && type == OS_LOG_FILE + && srv_unix_file_flush_method == SRV_UNIX_O_DSYNC) { + + create_flag |= O_SYNC; + } +#endif /* O_SYNC */ + + do { + file = ::open(name, create_flag, os_innodb_umask); + + if (file == -1) { + const char* operation; + + operation = (create_mode == OS_FILE_CREATE + && !srv_read_only_mode) + ? "create" : "open"; + + *success = FALSE; + + if (on_error_no_exit) { + retry = os_file_handle_error_no_exit( + name, operation, on_error_silent, __FILE__, __LINE__); + } else { + retry = os_file_handle_error(name, operation, __FILE__, __LINE__); + } + } else { + *success = TRUE; + retry = false; + } + + } while (retry); + + if (!srv_read_only_mode + && *success + && type != OS_LOG_FILE + && (srv_unix_file_flush_method == SRV_UNIX_O_DIRECT + || srv_unix_file_flush_method == SRV_UNIX_O_DIRECT_NO_FSYNC)) { + + os_file_set_nocache(file, name, mode_str); + } else if (!srv_read_only_mode + && *success + && srv_unix_file_flush_method == SRV_UNIX_ALL_O_DIRECT) { + os_file_set_nocache(file, name, mode_str); + } + +#ifdef USE_FILE_LOCK + if (!srv_read_only_mode + && *success + && create_mode != OS_FILE_OPEN_RAW + && os_file_lock(file, name)) { + + if (create_mode == OS_FILE_OPEN_RETRY) { + + ut_a(!srv_read_only_mode); + + ib_logf(IB_LOG_LEVEL_INFO, + "Retrying to lock the first data file"); + + for (int i = 0; i < 100; i++) { + os_thread_sleep(1000000); + + if (!os_file_lock(file, name)) { + *success = TRUE; + return(file); + } + } + + ib_logf(IB_LOG_LEVEL_INFO, + "Unable to open the first data file"); + } + + *success = FALSE; + close(file); + file = -1; + } +#endif /* USE_FILE_LOCK */ + + /* If we have proper file handle and atomic writes should be used, + try to set atomic writes and if that fails when creating a new + table, produce a error. If atomic writes are used on existing + file, ignore error and use traditional writes for that file */ + if (file != -1 + && (awrites == ATOMIC_WRITES_ON || + (srv_use_atomic_writes && awrites == ATOMIC_WRITES_DEFAULT)) + && !os_file_set_atomic_writes(name, file)) { + if (create_mode == OS_FILE_CREATE) { + fprintf(stderr, "InnoDB: Error: Can't create file using atomic writes\n"); + close(file); + os_file_delete_if_exists_func(name); + *success = FALSE; + file = -1; + } + } + + +#endif /* __WIN__ */ + + return(file); +} + +/***********************************************************************//** +Deletes a file if it exists. The file has to be closed before calling this. +@return TRUE if success */ +UNIV_INTERN +bool +os_file_delete_if_exists_func( +/*==========================*/ + const char* name) /*!< in: file path as a null-terminated + string */ +{ +#ifdef __WIN__ + bool ret; + ulint count = 0; +loop: + /* In Windows, deleting an .ibd file may fail if ibbackup is copying + it */ + + ret = DeleteFile((LPCTSTR) name); + + if (ret) { + return(true); + } + + DWORD lasterr = GetLastError(); + if (lasterr == ERROR_FILE_NOT_FOUND + || lasterr == ERROR_PATH_NOT_FOUND) { + /* the file does not exist, this not an error */ + + return(true); + } + + count++; + + if (count > 100 && 0 == (count % 10)) { + os_file_get_last_error(true); /* print error information */ + + ib_logf(IB_LOG_LEVEL_WARN, "Delete of file %s failed.", name); + } + + os_thread_sleep(1000000); /* sleep for a second */ + + if (count > 2000) { + + return(false); + } + + goto loop; +#else + int ret; + WAIT_ALLOW_WRITES(); + + ret = unlink(name); + + if (ret != 0 && errno != ENOENT) { + os_file_handle_error_no_exit(name, "delete", FALSE, __FILE__, __LINE__); + + return(false); + } + + return(true); +#endif /* __WIN__ */ +} + +/***********************************************************************//** +Deletes a file. The file has to be closed before calling this. +@return TRUE if success */ +UNIV_INTERN +bool +os_file_delete_func( +/*================*/ + const char* name) /*!< in: file path as a null-terminated + string */ +{ +#ifdef __WIN__ + BOOL ret; + ulint count = 0; +loop: + /* In Windows, deleting an .ibd file may fail if ibbackup is copying + it */ + + ret = DeleteFile((LPCTSTR) name); + + if (ret) { + return(true); + } + + if (GetLastError() == ERROR_FILE_NOT_FOUND) { + /* If the file does not exist, we classify this as a 'mild' + error and return */ + + return(false); + } + + count++; + + if (count > 100 && 0 == (count % 10)) { + os_file_get_last_error(true); /* print error information */ + + fprintf(stderr, + "InnoDB: Warning: cannot delete file %s\n" + "InnoDB: Are you running ibbackup" + " to back up the file?\n", name); + } + + os_thread_sleep(1000000); /* sleep for a second */ + + if (count > 2000) { + + return(false); + } + + goto loop; +#else + int ret; + WAIT_ALLOW_WRITES(); + + ret = unlink(name); + + if (ret != 0) { + os_file_handle_error_no_exit(name, "delete", FALSE, __FILE__, __LINE__); + + return(false); + } + + return(true); +#endif +} + +/***********************************************************************//** +NOTE! Use the corresponding macro os_file_rename(), not directly this function! +Renames a file (can also move it to another directory). It is safest that the +file is closed before calling this function. +@return TRUE if success */ +UNIV_INTERN +ibool +os_file_rename_func( +/*================*/ + const char* oldpath,/*!< in: old file path as a null-terminated + string */ + const char* newpath)/*!< in: new file path */ +{ +#ifdef UNIV_DEBUG + os_file_type_t type; + ibool exists; + + /* New path must not exist. */ + ut_ad(os_file_status(newpath, &exists, &type)); + ut_ad(!exists); + + /* Old path must exist. */ + ut_ad(os_file_status(oldpath, &exists, &type)); + ut_ad(exists); +#endif /* UNIV_DEBUG */ + +#ifdef __WIN__ + BOOL ret; + + ret = MoveFileEx((LPCTSTR)oldpath, (LPCTSTR)newpath, MOVEFILE_REPLACE_EXISTING); + + if (ret) { + return(TRUE); + } + + os_file_handle_error_no_exit(oldpath, "rename", FALSE, __FILE__, __LINE__); + + return(FALSE); +#else + int ret; + WAIT_ALLOW_WRITES(); + + ret = rename(oldpath, newpath); + + if (ret != 0) { + os_file_handle_error_no_exit(oldpath, "rename", FALSE, __FILE__, __LINE__); + + return(FALSE); + } + + return(TRUE); +#endif /* __WIN__ */ +} + +/***********************************************************************//** +NOTE! Use the corresponding macro os_file_close(), not directly this function! +Closes a file handle. In case of error, error number can be retrieved with +os_file_get_last_error. +@return TRUE if success */ +UNIV_INTERN +ibool +os_file_close_func( +/*===============*/ + os_file_t file) /*!< in, own: handle to a file */ +{ +#ifdef __WIN__ + BOOL ret; + + ut_a(file); + + ret = CloseHandle(file); + + if (ret) { + return(TRUE); + } + + os_file_handle_error(NULL, "close", __FILE__, __LINE__); + + return(FALSE); +#else + int ret; + + ret = close(file); + + if (ret == -1) { + os_file_handle_error(NULL, "close", __FILE__, __LINE__); + + return(FALSE); + } + + return(TRUE); +#endif /* __WIN__ */ +} + +/***********************************************************************//** +Closes a file handle. +@return TRUE if success */ +UNIV_INTERN +ibool +os_file_close_no_error_handling( +/*============================*/ + os_file_t file) /*!< in, own: handle to a file */ +{ +#ifdef __WIN__ + BOOL ret; + + ut_a(file); + + ret = CloseHandle(file); + + if (ret) { + return(TRUE); + } + + return(FALSE); +#else + int ret; + + ret = close(file); + + if (ret == -1) { + + return(FALSE); + } + + return(TRUE); +#endif /* __WIN__ */ +} + +/***********************************************************************//** +Gets a file size. +@return file size, or (os_offset_t) -1 on failure */ +UNIV_INTERN +os_offset_t +os_file_get_size( +/*=============*/ + os_file_t file) /*!< in: handle to a file */ +{ +#ifdef __WIN__ + os_offset_t offset; + DWORD high; + DWORD low; + + low = GetFileSize(file, &high); + + if ((low == 0xFFFFFFFF) && (GetLastError() != NO_ERROR)) { + return((os_offset_t) -1); + } + + offset = (os_offset_t) low | ((os_offset_t) high << 32); + + return(offset); +#else + return((os_offset_t) lseek(file, 0, SEEK_END)); +#endif /* __WIN__ */ +} + +/***********************************************************************//** +Write the specified number of zeros to a newly created file. +@return TRUE if success */ +UNIV_INTERN +ibool +os_file_set_size( +/*=============*/ + const char* name, /*!< in: name of the file or path as a + null-terminated string */ + os_file_t file, /*!< in: handle to a file */ + os_offset_t size) /*!< in: file size */ +{ + os_offset_t current_size; + ibool ret; + byte* buf; + byte* buf2; + ulint buf_size; + + current_size = 0; + +#ifdef HAVE_POSIX_FALLOCATE + if (srv_use_posix_fallocate) { + + if (posix_fallocate(file, current_size, size) == -1) { + + ib_logf(IB_LOG_LEVEL_ERROR, "preallocating file " + "space for file \'%s\' failed. Current size " + INT64PF ", desired size " INT64PF "\n", + name, current_size, size); + os_file_handle_error_no_exit (name, "posix_fallocate", + FALSE, __FILE__, __LINE__); + return(FALSE); + } + return(TRUE); + } +#endif + + /* Write up to 1 megabyte at a time. */ + buf_size = ut_min(64, (ulint) (size / UNIV_PAGE_SIZE)) + * UNIV_PAGE_SIZE; + buf2 = static_cast(ut_malloc(buf_size + UNIV_PAGE_SIZE)); + + /* Align the buffer for possible raw i/o */ + buf = static_cast(ut_align(buf2, UNIV_PAGE_SIZE)); + + /* Write buffer full of zeros */ + memset(buf, 0, buf_size); + + if (size >= (os_offset_t) 100 << 20) { + + fprintf(stderr, "InnoDB: Progress in MB:"); + } + + while (current_size < size) { + ulint n_bytes; + + if (size - current_size < (os_offset_t) buf_size) { + n_bytes = (ulint) (size - current_size); + } else { + n_bytes = buf_size; + } + + ret = os_file_write(name, file, buf, current_size, n_bytes); + + if (!ret) { + ut_free(buf2); + goto error_handling; + } + + /* Print about progress for each 100 MB written */ + if ((current_size + n_bytes) / (100 << 20) + != current_size / (100 << 20)) { + + fprintf(stderr, " %lu00", + (ulong) ((current_size + n_bytes) + / (100 << 20))); + } + + current_size += n_bytes; + } + + if (size >= (os_offset_t) 100 << 20) { + + fprintf(stderr, "\n"); + } + + ut_free(buf2); + + ret = os_file_flush(file); + + if (ret) { + return(TRUE); + } + +error_handling: + return(FALSE); +} + +/***********************************************************************//** +Truncates a file at its current position. +@return TRUE if success */ +UNIV_INTERN +ibool +os_file_set_eof( +/*============*/ + FILE* file) /*!< in: file to be truncated */ +{ +#ifdef __WIN__ + HANDLE h = (HANDLE) _get_osfhandle(fileno(file)); + return(SetEndOfFile(h)); +#else /* __WIN__ */ + WAIT_ALLOW_WRITES(); + return(!ftruncate(fileno(file), ftell(file))); +#endif /* __WIN__ */ +} + +/***********************************************************************//** +Truncates a file at the specified position. +@return TRUE if success */ +UNIV_INTERN +ibool +os_file_set_eof_at( + os_file_t file, /*!< in: handle to a file */ + ib_uint64_t new_len)/*!< in: new file length */ +{ +#ifdef __WIN__ + LARGE_INTEGER li, li2; + li.QuadPart = new_len; + return(SetFilePointerEx(file, li, &li2,FILE_BEGIN) + && SetEndOfFile(file)); +#else + WAIT_ALLOW_WRITES(); + /* TODO: works only with -D_FILE_OFFSET_BITS=64 ? */ + return(!ftruncate(file, new_len)); +#endif +} + + +#ifndef __WIN__ +/***********************************************************************//** +Wrapper to fsync(2) that retries the call on some errors. +Returns the value 0 if successful; otherwise the value -1 is returned and +the global variable errno is set to indicate the error. +@return 0 if success, -1 otherwise */ + +static +int +os_file_fsync( +/*==========*/ + os_file_t file) /*!< in: handle to a file */ +{ + int ret; + int failures; + ibool retry; + + failures = 0; + + do { + ret = fsync(file); + + os_n_fsyncs++; + + if (ret == -1 && errno == ENOLCK) { + + if (failures % 100 == 0) { + + ut_print_timestamp(stderr); + fprintf(stderr, + " InnoDB: fsync(): " + "No locks available; retrying\n"); + } + + os_thread_sleep(200000 /* 0.2 sec */); + + failures++; + + retry = TRUE; + } else if (ret == -1 && errno == EINTR) { + /* Handle signal interruptions correctly */ + retry = TRUE; + } else { + + retry = FALSE; + } + } while (retry); + + return(ret); +} +#endif /* !__WIN__ */ + +/***********************************************************************//** +NOTE! Use the corresponding macro os_file_flush(), not directly this function! +Flushes the write buffers of a given file to the disk. +@return TRUE if success */ +UNIV_INTERN +ibool +os_file_flush_func( +/*===============*/ + os_file_t file) /*!< in, own: handle to a file */ +{ +#ifdef __WIN__ + BOOL ret; + + ut_a(file); + + os_n_fsyncs++; + + ret = FlushFileBuffers(file); + + if (ret) { + return(TRUE); + } + + /* Since Windows returns ERROR_INVALID_FUNCTION if the 'file' is + actually a raw device, we choose to ignore that error if we are using + raw disks */ + + if (srv_start_raw_disk_in_use && GetLastError() + == ERROR_INVALID_FUNCTION) { + return(TRUE); + } + + os_file_handle_error(NULL, "flush", __FILE__, __LINE__); + + /* It is a fatal error if a file flush does not succeed, because then + the database can get corrupt on disk */ + ut_error; + + return(FALSE); +#else + int ret; + WAIT_ALLOW_WRITES(); + +#if defined(HAVE_DARWIN_THREADS) +# ifndef F_FULLFSYNC + /* The following definition is from the Mac OS X 10.3 */ +# define F_FULLFSYNC 51 /* fsync + ask the drive to flush to the media */ +# elif F_FULLFSYNC != 51 +# error "F_FULLFSYNC != 51: ABI incompatibility with Mac OS X 10.3" +# endif + /* Apple has disabled fsync() for internal disk drives in OS X. That + caused corruption for a user when he tested a power outage. Let us in + OS X use a nonstandard flush method recommended by an Apple + engineer. */ + + if (!srv_have_fullfsync) { + /* If we are not on an operating system that supports this, + then fall back to a plain fsync. */ + + ret = os_file_fsync(file); + } else { + ret = fcntl(file, F_FULLFSYNC, NULL); + + if (ret) { + /* If we are not on a file system that supports this, + then fall back to a plain fsync. */ + ret = os_file_fsync(file); + } + } +#else + ret = os_file_fsync(file); +#endif + + if (ret == 0) { + return(TRUE); + } + + /* Since Linux returns EINVAL if the 'file' is actually a raw device, + we choose to ignore that error if we are using raw disks */ + + if (srv_start_raw_disk_in_use && errno == EINVAL) { + + return(TRUE); + } + + ib_logf(IB_LOG_LEVEL_ERROR, "The OS said file flush did not succeed"); + + os_file_handle_error(NULL, "flush", __FILE__, __LINE__); + + /* It is a fatal error if a file flush does not succeed, because then + the database can get corrupt on disk */ + ut_error; + + return(FALSE); +#endif +} + +#ifndef __WIN__ +/*******************************************************************//** +Does a synchronous read operation in Posix. +@return number of bytes read, -1 if error */ +static __attribute__((nonnull(2), warn_unused_result)) +ssize_t +os_file_pread( +/*==========*/ + os_file_t file, /*!< in: handle to a file */ + void* buf, /*!< in: buffer where to read */ + ulint n, /*!< in: number of bytes to read */ + os_offset_t offset, /*!< in: file offset from where to read */ + trx_t* trx) +{ + off_t offs; +#if defined(HAVE_PREAD) && !defined(HAVE_BROKEN_PREAD) + ssize_t n_bytes; + ssize_t n_read; +#endif /* HAVE_PREAD && !HAVE_BROKEN_PREAD */ + ulint sec; + ulint ms; + ib_uint64_t start_time; + ib_uint64_t finish_time; + + ut_ad(n); + + /* If off_t is > 4 bytes in size, then we assume we can pass a + 64-bit address */ + offs = (off_t) offset; + + if (sizeof(off_t) <= 4) { + if (offset != (os_offset_t) offs) { + ib_logf(IB_LOG_LEVEL_ERROR, + "File read at offset > 4 GB"); + } + } + + os_n_file_reads++; + + if (UNIV_UNLIKELY(trx && trx->take_stats)) + { + trx->io_reads++; + trx->io_read += n; + ut_usectime(&sec, &ms); + start_time = (ib_uint64_t)sec * 1000000 + ms; + } else { + start_time = 0; + } +#if defined(HAVE_PREAD) && !defined(HAVE_BROKEN_PREAD) +#if defined(HAVE_ATOMIC_BUILTINS) && UNIV_WORD_SIZE == 8 + (void) os_atomic_increment_ulint(&os_n_pending_reads, 1); + (void) os_atomic_increment_ulint(&os_file_n_pending_preads, 1); + MONITOR_ATOMIC_INC(MONITOR_OS_PENDING_READS); +#else + os_mutex_enter(os_file_count_mutex); + os_file_n_pending_preads++; + os_n_pending_reads++; + MONITOR_INC(MONITOR_OS_PENDING_READS); + os_mutex_exit(os_file_count_mutex); +#endif /* HAVE_ATOMIC_BUILTINS && UNIV_WORD == 8 */ + + /* Handle partial reads and signal interruptions correctly */ + for (n_bytes = 0; n_bytes < (ssize_t) n; ) { + n_read = pread(file, buf, (ssize_t)n - n_bytes, offs); + if (n_read > 0) { + n_bytes += n_read; + offs += n_read; + buf = (char *)buf + n_read; + } else if (n_read == -1 && errno == EINTR) { + continue; + } else { + break; + } + } + +#if defined(HAVE_ATOMIC_BUILTINS) && UNIV_WORD_SIZE == 8 + (void) os_atomic_decrement_ulint(&os_n_pending_reads, 1); + (void) os_atomic_decrement_ulint(&os_file_n_pending_preads, 1); + MONITOR_ATOMIC_DEC(MONITOR_OS_PENDING_READS); +#else + os_mutex_enter(os_file_count_mutex); + os_file_n_pending_preads--; + os_n_pending_reads--; + MONITOR_DEC(MONITOR_OS_PENDING_READS); + os_mutex_exit(os_file_count_mutex); +#endif /* !HAVE_ATOMIC_BUILTINS || UNIV_WORD == 8 */ + + if (UNIV_UNLIKELY(start_time != 0)) + { + ut_usectime(&sec, &ms); + finish_time = (ib_uint64_t)sec * 1000000 + ms; + trx->io_reads_wait_timer += (ulint)(finish_time - start_time); + } + + return(n_bytes); +#else + { + off_t ret_offset; + ssize_t ret; + ssize_t n_read; +#ifndef UNIV_HOTBACKUP + ulint i; +#endif /* !UNIV_HOTBACKUP */ + +#if defined(HAVE_ATOMIC_BUILTINS) && UNIV_WORD_SIZE == 8 + (void) os_atomic_increment_ulint(&os_n_pending_reads, 1); + MONITOR_ATOMIC_INC(MONITOR_OS_PENDING_READS); +#else + os_mutex_enter(os_file_count_mutex); + os_n_pending_reads++; + MONITOR_INC(MONITOR_OS_PENDING_READS); + os_mutex_exit(os_file_count_mutex); +#endif /* HAVE_ATOMIC_BUILTINS && UNIV_WORD == 8 */ +#ifndef UNIV_HOTBACKUP + /* Protect the seek / read operation with a mutex */ + i = ((ulint) file) % OS_FILE_N_SEEK_MUTEXES; + + os_mutex_enter(os_file_seek_mutexes[i]); +#endif /* !UNIV_HOTBACKUP */ + + ret_offset = lseek(file, offs, SEEK_SET); + + if (ret_offset < 0) { + ret = -1; + } else { + /* Handle signal interruptions correctly */ + for (ret = 0; ret < (ssize_t) n; ) { + n_read = read(file, buf, (ssize_t)n); + if (n_read > 0) { + ret += n_read; + } else if (n_read == -1 && errno == EINTR) { + continue; + } else { + break; + } + } + } + +#ifndef UNIV_HOTBACKUP + os_mutex_exit(os_file_seek_mutexes[i]); +#endif /* !UNIV_HOTBACKUP */ + +#if defined(HAVE_ATOMIC_BUILTINS) && UNIV_WORD_SIZE == 8 + (void) os_atomic_decrement_ulint(&os_n_pending_reads, 1); + MONITOR_ATOIC_DEC(MONITOR_OS_PENDING_READS); +#else + os_mutex_enter(os_file_count_mutex); + os_n_pending_reads--; + MONITOR_DEC(MONITOR_OS_PENDING_READS); + os_mutex_exit(os_file_count_mutex); +#endif /* HAVE_ATOMIC_BUILTINS && UNIV_WORD_SIZE == 8 */ + + if (UNIV_UNLIKELY(start_time != 0) + { + ut_usectime(&sec, &ms); + finish_time = (ib_uint64_t)sec * 1000000 + ms; + trx->io_reads_wait_timer += (ulint)(finish_time - start_time); + } + + return(ret); + } +#endif +} + +/*******************************************************************//** +Does a synchronous write operation in Posix. +@return number of bytes written, -1 if error */ +static __attribute__((nonnull, warn_unused_result)) +ssize_t +os_file_pwrite( +/*===========*/ + os_file_t file, /*!< in: handle to a file */ + const void* buf, /*!< in: buffer from where to write */ + ulint n, /*!< in: number of bytes to write */ + os_offset_t offset) /*!< in: file offset where to write */ +{ + ssize_t ret; + ssize_t n_written; + off_t offs; + + ut_ad(n); + ut_ad(!srv_read_only_mode); + + /* If off_t is > 4 bytes in size, then we assume we can pass a + 64-bit address */ + offs = (off_t) offset; + + if (sizeof(off_t) <= 4) { + if (offset != (os_offset_t) offs) { + ib_logf(IB_LOG_LEVEL_ERROR, + "File write at offset > 4 GB."); + } + } + + os_n_file_writes++; + +#if defined(HAVE_PWRITE) && !defined(HAVE_BROKEN_PREAD) +#if !defined(HAVE_ATOMIC_BUILTINS) || UNIV_WORD_SIZE < 8 + os_mutex_enter(os_file_count_mutex); + os_file_n_pending_pwrites++; + os_n_pending_writes++; + MONITOR_INC(MONITOR_OS_PENDING_WRITES); + os_mutex_exit(os_file_count_mutex); +#else + (void) os_atomic_increment_ulint(&os_n_pending_writes, 1); + (void) os_atomic_increment_ulint(&os_file_n_pending_pwrites, 1); + MONITOR_ATOMIC_INC(MONITOR_OS_PENDING_WRITES); +#endif /* !HAVE_ATOMIC_BUILTINS || UNIV_WORD < 8 */ + + /* Handle partial writes and signal interruptions correctly */ + for (ret = 0; ret < (ssize_t) n; ) { + n_written = pwrite(file, buf, (ssize_t)n - ret, offs); + if (n_written >= 0) { + ret += n_written; + offs += n_written; + buf = (char *)buf + n_written; + } else if (n_written == -1 && errno == EINTR) { + continue; + } else { + break; + } + } + +#if !defined(HAVE_ATOMIC_BUILTINS) || UNIV_WORD_SIZE < 8 + os_mutex_enter(os_file_count_mutex); + os_file_n_pending_pwrites--; + os_n_pending_writes--; + MONITOR_DEC(MONITOR_OS_PENDING_WRITES); + os_mutex_exit(os_file_count_mutex); +#else + (void) os_atomic_decrement_ulint(&os_n_pending_writes, 1); + (void) os_atomic_decrement_ulint(&os_file_n_pending_pwrites, 1); + MONITOR_ATOMIC_DEC(MONITOR_OS_PENDING_WRITES); +#endif /* !HAVE_ATOMIC_BUILTINS || UNIV_WORD < 8 */ + + return(ret); +#else + { + off_t ret_offset; +# ifndef UNIV_HOTBACKUP + ulint i; +# endif /* !UNIV_HOTBACKUP */ + + os_mutex_enter(os_file_count_mutex); + os_n_pending_writes++; + MONITOR_INC(MONITOR_OS_PENDING_WRITES); + os_mutex_exit(os_file_count_mutex); + +# ifndef UNIV_HOTBACKUP + /* Protect the seek / write operation with a mutex */ + i = ((ulint) file) % OS_FILE_N_SEEK_MUTEXES; + + os_mutex_enter(os_file_seek_mutexes[i]); +# endif /* UNIV_HOTBACKUP */ + + ret_offset = lseek(file, offs, SEEK_SET); + + if (ret_offset < 0) { + ret = -1; + + goto func_exit; + } + + /* Handle signal interruptions correctly */ + for (ret = 0; ret < (ssize_t) n; ) { + n_written = write(file, buf, (ssize_t)n); + if (n_written > 0) { + ret += n_written; + } else if (n_written == -1 && errno == EINTR) { + continue; + } else { + break; + } + } + +func_exit: +# ifndef UNIV_HOTBACKUP + os_mutex_exit(os_file_seek_mutexes[i]); +# endif /* !UNIV_HOTBACKUP */ + + os_mutex_enter(os_file_count_mutex); + os_n_pending_writes--; + MONITOR_DEC(MONITOR_OS_PENDING_WRITES); + os_mutex_exit(os_file_count_mutex); + + return(ret); + } +#endif /* !UNIV_HOTBACKUP */ +} +#endif + +/*******************************************************************//** +NOTE! Use the corresponding macro os_file_read(), not directly this +function! +Requests a synchronous positioned read operation. +@return TRUE if request was successful, FALSE if fail */ +UNIV_INTERN +ibool +os_file_read_func( +/*==============*/ + os_file_t file, /*!< in: handle to a file */ + void* buf, /*!< in: buffer where to read */ + os_offset_t offset, /*!< in: file offset where to read */ + ulint n, /*!< in: number of bytes to read */ + trx_t* trx, + ibool compressed) /*!< in: is this file space + compressed ? */ +{ +#ifdef __WIN__ + BOOL ret; + DWORD len; + ibool retry; + OVERLAPPED overlapped; + + + /* On 64-bit Windows, ulint is 64 bits. But offset and n should be + no more than 32 bits. */ + ut_a((n & 0xFFFFFFFFUL) == n); + + os_n_file_reads++; + os_bytes_read_since_printout += n; + +try_again: + ut_ad(file); + ut_ad(buf); + ut_ad(n > 0); + + os_mutex_enter(os_file_count_mutex); + os_n_pending_reads++; + MONITOR_INC(MONITOR_OS_PENDING_READS); + os_mutex_exit(os_file_count_mutex); + + memset (&overlapped, 0, sizeof (overlapped)); + overlapped.Offset = (DWORD)(offset & 0xFFFFFFFF); + overlapped.OffsetHigh = (DWORD)(offset >> 32); + overlapped.hEvent = win_get_syncio_event(); + ret = ReadFile(file, buf, n, NULL, &overlapped); + if (ret) { + ret = GetOverlappedResult(file, &overlapped, (DWORD *)&len, FALSE); + } + else if(GetLastError() == ERROR_IO_PENDING) { + ret = GetOverlappedResult(file, &overlapped, (DWORD *)&len, TRUE); + } + os_mutex_enter(os_file_count_mutex); + os_n_pending_reads--; + MONITOR_DEC(MONITOR_OS_PENDING_READS); + os_mutex_exit(os_file_count_mutex); + + if (ret && len == n) { + /* Note that InnoDB writes files that are not formated + as file spaces and they do not have FIL_PAGE_TYPE + field, thus we must use here information is the actual + file space compressed. */ + if (compressed && fil_page_is_compressed((byte *)buf)) { + fil_decompress_page(NULL, (byte *)buf, len, NULL); + } + + return(TRUE); + } +#else /* __WIN__ */ + ibool retry; + ssize_t ret; + + os_bytes_read_since_printout += n; + +try_again: + ret = os_file_pread(file, buf, n, offset, trx); + + if ((ulint) ret == n) { + if (fil_page_is_encrypted((byte *)buf)) { + fil_decrypt_page(NULL, (byte *)buf, n, NULL); + } + /* Note that InnoDB writes files that are not formated + as file spaces and they do not have FIL_PAGE_TYPE + field, thus we must use here information is the actual + file space compressed. */ + if (compressed && fil_page_is_compressed((byte *)buf)) { + fil_decompress_page(NULL, (byte *)buf, n, NULL); + } + + + return(TRUE); + } + + ib_logf(IB_LOG_LEVEL_ERROR, + "Tried to read "ULINTPF" bytes at offset " UINT64PF". " + "Was only able to read %ld.", n, offset, (lint) ret); +#endif /* __WIN__ */ + retry = os_file_handle_error(NULL, "read", __FILE__, __LINE__); + + if (retry) { + goto try_again; + } + + fprintf(stderr, + "InnoDB: Fatal error: cannot read from file." + " OS error number %lu.\n", +#ifdef __WIN__ + (ulong) GetLastError() +#else + (ulong) errno +#endif /* __WIN__ */ + ); + fflush(stderr); + + ut_error; + + return(FALSE); +} + +/*******************************************************************//** +NOTE! Use the corresponding macro os_file_read_no_error_handling(), +not directly this function! +Requests a synchronous positioned read operation. This function does not do +any error handling. In case of error it returns FALSE. +@return TRUE if request was successful, FALSE if fail */ +UNIV_INTERN +ibool +os_file_read_no_error_handling_func( +/*================================*/ + os_file_t file, /*!< in: handle to a file */ + void* buf, /*!< in: buffer where to read */ + os_offset_t offset, /*!< in: file offset where to read */ + ulint n, /*!< in: number of bytes to read */ + ibool compressed) /*!< in: is this file space + compressed ? */ +{ +#ifdef __WIN__ + BOOL ret; + DWORD len; + ibool retry; + OVERLAPPED overlapped; + overlapped.Offset = (DWORD)(offset & 0xFFFFFFFF); + overlapped.OffsetHigh = (DWORD)(offset >> 32); + + + /* On 64-bit Windows, ulint is 64 bits. But offset and n should be + no more than 32 bits. */ + ut_a((n & 0xFFFFFFFFUL) == n); + + os_n_file_reads++; + os_bytes_read_since_printout += n; + +try_again: + ut_ad(file); + ut_ad(buf); + ut_ad(n > 0); + + os_mutex_enter(os_file_count_mutex); + os_n_pending_reads++; + MONITOR_INC(MONITOR_OS_PENDING_READS); + os_mutex_exit(os_file_count_mutex); + + memset (&overlapped, 0, sizeof (overlapped)); + overlapped.Offset = (DWORD)(offset & 0xFFFFFFFF); + overlapped.OffsetHigh = (DWORD)(offset >> 32); + overlapped.hEvent = win_get_syncio_event(); + ret = ReadFile(file, buf, n, NULL, &overlapped); + if (ret) { + ret = GetOverlappedResult(file, &overlapped, (DWORD *)&len, FALSE); + } + else if(GetLastError() == ERROR_IO_PENDING) { + ret = GetOverlappedResult(file, &overlapped, (DWORD *)&len, TRUE); + } + os_mutex_enter(os_file_count_mutex); + os_n_pending_reads--; + MONITOR_DEC(MONITOR_OS_PENDING_READS); + os_mutex_exit(os_file_count_mutex); + + if (ret && len == n) { + + /* Note that InnoDB writes files that are not formated + as file spaces and they do not have FIL_PAGE_TYPE + field, thus we must use here information is the actual + file space compressed. */ + if (compressed && fil_page_is_compressed((byte *)buf)) { + fil_decompress_page(NULL, (byte *)buf, n, NULL); + } + + return(TRUE); + } +#else /* __WIN__ */ + ibool retry; + ssize_t ret; + + os_bytes_read_since_printout += n; + +try_again: + ret = os_file_pread(file, buf, n, offset, NULL); + + if ((ulint) ret == n) { + + /* Note that InnoDB writes files that are not formated + as file spaces and they do not have FIL_PAGE_TYPE + field, thus we must use here information is the actual + file space compressed. */ + if (compressed && fil_page_is_compressed((byte *)buf)) { + fil_decompress_page(NULL, (byte *)buf, n, NULL); + } + + if (fil_page_is_encrypted((byte *)buf)) { + fil_decrypt_page(NULL, (byte *)buf, n, NULL); + } + + + + return(TRUE); + } +#endif /* __WIN__ */ + retry = os_file_handle_error_no_exit(NULL, "read", FALSE, __FILE__, __LINE__); + + if (retry) { + goto try_again; + } + + return(FALSE); +} + +/*******************************************************************//** +Rewind file to its start, read at most size - 1 bytes from it to str, and +NUL-terminate str. All errors are silently ignored. This function is +mostly meant to be used with temporary files. */ +UNIV_INTERN +void +os_file_read_string( +/*================*/ + FILE* file, /*!< in: file to read from */ + char* str, /*!< in: buffer where to read */ + ulint size) /*!< in: size of buffer */ +{ + size_t flen; + + if (size == 0) { + return; + } + + rewind(file); + flen = fread(str, 1, size - 1, file); + str[flen] = '\0'; +} + +/*******************************************************************//** +NOTE! Use the corresponding macro os_file_write(), not directly +this function! +Requests a synchronous write operation. +@return TRUE if request was successful, FALSE if fail */ +UNIV_INTERN +ibool +os_file_write_func( +/*===============*/ + const char* name, /*!< in: name of the file or path as a + null-terminated string */ + os_file_t file, /*!< in: handle to a file */ + const void* buf, /*!< in: buffer from which to write */ + os_offset_t offset, /*!< in: file offset where to write */ + ulint n) /*!< in: number of bytes to write */ +{ + ut_ad(!srv_read_only_mode); + +#ifdef __WIN__ + BOOL ret; + DWORD len; + ulint n_retries = 0; + ulint err; + OVERLAPPED overlapped; + + /* On 64-bit Windows, ulint is 64 bits. But offset and n should be + no more than 32 bits. */ + ut_a((n & 0xFFFFFFFFUL) == n); + + os_n_file_writes++; + + ut_ad(file); + ut_ad(buf); + ut_ad(n > 0); + +retry: + + os_mutex_enter(os_file_count_mutex); + os_n_pending_writes++; + MONITOR_INC(MONITOR_OS_PENDING_WRITES); + os_mutex_exit(os_file_count_mutex); + + memset (&overlapped, 0, sizeof (overlapped)); + overlapped.Offset = (DWORD)(offset & 0xFFFFFFFF); + overlapped.OffsetHigh = (DWORD)(offset >> 32); + + overlapped.hEvent = win_get_syncio_event(); + ret = WriteFile(file, buf, n, NULL, &overlapped); + if (ret) { + ret = GetOverlappedResult(file, &overlapped, (DWORD *)&len, FALSE); + } + else if(GetLastError() == ERROR_IO_PENDING) { + ret = GetOverlappedResult(file, &overlapped, (DWORD *)&len, TRUE); + } + + os_mutex_enter(os_file_count_mutex); + os_n_pending_writes--; + MONITOR_DEC(MONITOR_OS_PENDING_WRITES); + os_mutex_exit(os_file_count_mutex); + + if (ret && len == n) { + + return(TRUE); + } + + /* If some background file system backup tool is running, then, at + least in Windows 2000, we may get here a specific error. Let us + retry the operation 100 times, with 1 second waits. */ + + if (GetLastError() == ERROR_LOCK_VIOLATION && n_retries < 100) { + + os_thread_sleep(1000000); + + n_retries++; + + goto retry; + } + + if (!os_has_said_disk_full) { + + err = (ulint) GetLastError(); + + ut_print_timestamp(stderr); + + fprintf(stderr, + " InnoDB: Error: Write to file %s failed" + " at offset %llu.\n" + "InnoDB: %lu bytes should have been written," + " only %lu were written.\n" + "InnoDB: Operating system error number %lu.\n" + "InnoDB: Check that your OS and file system" + " support files of this size.\n" + "InnoDB: Check also that the disk is not full" + " or a disk quota exceeded.\n", + name, offset, + (ulong) n, (ulong) len, (ulong) err); + + if (strerror((int) err) != NULL) { + fprintf(stderr, + "InnoDB: Error number %lu means '%s'.\n", + (ulong) err, strerror((int) err)); + } + + fprintf(stderr, + "InnoDB: Some operating system error numbers" + " are described at\n" + "InnoDB: " + REFMAN "operating-system-error-codes.html\n"); + + os_has_said_disk_full = TRUE; + } + + return(FALSE); +#else + ssize_t ret; + WAIT_ALLOW_WRITES(); + + ret = os_file_pwrite(file, buf, n, offset); + + if ((ulint) ret == n) { + + return(TRUE); + } + + if (!os_has_said_disk_full) { + + ut_print_timestamp(stderr); + + fprintf(stderr, + " InnoDB: Error: Write to file %s failed" + " at offset "UINT64PF".\n" + "InnoDB: %lu bytes should have been written," + " only %ld were written.\n" + "InnoDB: Operating system error number %lu.\n" + "InnoDB: Check that your OS and file system" + " support files of this size.\n" + "InnoDB: Check also that the disk is not full" + " or a disk quota exceeded.\n", + name, offset, n, (lint) ret, + (ulint) errno); + if (strerror(errno) != NULL) { + fprintf(stderr, + "InnoDB: Error number %d means '%s'.\n", + errno, strerror(errno)); + } + + fprintf(stderr, + "InnoDB: Some operating system error numbers" + " are described at\n" + "InnoDB: " + REFMAN "operating-system-error-codes.html\n"); + + os_has_said_disk_full = TRUE; + } + + return(FALSE); +#endif +} + +/*******************************************************************//** +Check the existence and type of the given file. +@return TRUE if call succeeded */ +UNIV_INTERN +ibool +os_file_status( +/*===========*/ + const char* path, /*!< in: pathname of the file */ + ibool* exists, /*!< out: TRUE if file exists */ + os_file_type_t* type) /*!< out: type of the file (if it exists) */ +{ +#ifdef __WIN__ + int ret; + struct _stat64 statinfo; + + ret = _stat64(path, &statinfo); + if (ret && (errno == ENOENT || errno == ENOTDIR)) { + /* file does not exist */ + *exists = FALSE; + return(TRUE); + } else if (ret) { + /* file exists, but stat call failed */ + + os_file_handle_error_no_exit(path, "stat", FALSE, __FILE__, __LINE__); + + return(FALSE); + } + + if (_S_IFDIR & statinfo.st_mode) { + *type = OS_FILE_TYPE_DIR; + } else if (_S_IFREG & statinfo.st_mode) { + *type = OS_FILE_TYPE_FILE; + } else { + *type = OS_FILE_TYPE_UNKNOWN; + } + + *exists = TRUE; + + return(TRUE); +#else + int ret; + struct stat statinfo; + + ret = stat(path, &statinfo); + if (ret && (errno == ENOENT || errno == ENOTDIR)) { + /* file does not exist */ + *exists = FALSE; + return(TRUE); + } else if (ret) { + /* file exists, but stat call failed */ + + os_file_handle_error_no_exit(path, "stat", FALSE, __FILE__, __LINE__); + + return(FALSE); + } + + if (S_ISDIR(statinfo.st_mode)) { + *type = OS_FILE_TYPE_DIR; + } else if (S_ISLNK(statinfo.st_mode)) { + *type = OS_FILE_TYPE_LINK; + } else if (S_ISREG(statinfo.st_mode)) { + *type = OS_FILE_TYPE_FILE; + } else { + *type = OS_FILE_TYPE_UNKNOWN; + } + + *exists = TRUE; + + return(TRUE); +#endif +} + +/*******************************************************************//** +This function returns information about the specified file +@return DB_SUCCESS if all OK */ +UNIV_INTERN +dberr_t +os_file_get_status( +/*===============*/ + const char* path, /*!< in: pathname of the file */ + os_file_stat_t* stat_info, /*!< information of a file in a + directory */ + bool check_rw_perm) /*!< in: for testing whether the + file can be opened in RW mode */ +{ + int ret; + +#ifdef __WIN__ + struct _stat64 statinfo; + + ret = _stat64(path, &statinfo); + + if (ret && (errno == ENOENT || errno == ENOTDIR)) { + /* file does not exist */ + + return(DB_NOT_FOUND); + + } else if (ret) { + /* file exists, but stat call failed */ + + os_file_handle_error_no_exit(path, "stat", FALSE, __FILE__, __LINE__); + + return(DB_FAIL); + + } else if (_S_IFDIR & statinfo.st_mode) { + stat_info->type = OS_FILE_TYPE_DIR; + } else if (_S_IFREG & statinfo.st_mode) { + + DWORD access = GENERIC_READ; + + if (!srv_read_only_mode) { + access |= GENERIC_WRITE; + } + + stat_info->type = OS_FILE_TYPE_FILE; + + /* Check if we can open it in read-only mode. */ + + if (check_rw_perm) { + HANDLE fh; + + fh = CreateFile( + (LPCTSTR) path, // File to open + access, + 0, // No sharing + NULL, // Default security + OPEN_EXISTING, // Existing file only + FILE_ATTRIBUTE_NORMAL, // Normal file + NULL); // No attr. template + + if (fh == INVALID_HANDLE_VALUE) { + stat_info->rw_perm = false; + } else { + stat_info->rw_perm = true; + CloseHandle(fh); + } + } + } else { + stat_info->type = OS_FILE_TYPE_UNKNOWN; + } +#else + struct stat statinfo; + + ret = stat(path, &statinfo); + + if (ret && (errno == ENOENT || errno == ENOTDIR)) { + /* file does not exist */ + + return(DB_NOT_FOUND); + + } else if (ret) { + /* file exists, but stat call failed */ + + os_file_handle_error_no_exit(path, "stat", FALSE, __FILE__, __LINE__); + + return(DB_FAIL); + + } + + switch (statinfo.st_mode & S_IFMT) { + case S_IFDIR: + stat_info->type = OS_FILE_TYPE_DIR; + break; + case S_IFLNK: + stat_info->type = OS_FILE_TYPE_LINK; + break; + case S_IFBLK: + stat_info->type = OS_FILE_TYPE_BLOCK; + break; + case S_IFREG: + stat_info->type = OS_FILE_TYPE_FILE; + break; + default: + stat_info->type = OS_FILE_TYPE_UNKNOWN; + } + + + if (check_rw_perm && (stat_info->type == OS_FILE_TYPE_FILE + || stat_info->type == OS_FILE_TYPE_BLOCK)) { + int fh; + int access; + + access = !srv_read_only_mode ? O_RDWR : O_RDONLY; + + fh = ::open(path, access, os_innodb_umask); + + if (fh == -1) { + stat_info->rw_perm = false; + } else { + stat_info->rw_perm = true; + close(fh); + } + } + +#endif /* _WIN_ */ + + stat_info->ctime = statinfo.st_ctime; + stat_info->atime = statinfo.st_atime; + stat_info->mtime = statinfo.st_mtime; + stat_info->size = statinfo.st_size; + + return(DB_SUCCESS); +} + +/* path name separator character */ +#ifdef __WIN__ +# define OS_FILE_PATH_SEPARATOR '\\' +#else +# define OS_FILE_PATH_SEPARATOR '/' +#endif + +/****************************************************************//** +This function returns a new path name after replacing the basename +in an old path with a new basename. The old_path is a full path +name including the extension. The tablename is in the normal +form "databasename/tablename". The new base name is found after +the forward slash. Both input strings are null terminated. + +This function allocates memory to be returned. It is the callers +responsibility to free the return value after it is no longer needed. + +@return own: new full pathname */ +UNIV_INTERN +char* +os_file_make_new_pathname( +/*======================*/ + const char* old_path, /*!< in: pathname */ + const char* tablename) /*!< in: contains new base name */ +{ + ulint dir_len; + char* last_slash; + char* base_name; + char* new_path; + ulint new_path_len; + + /* Split the tablename into its database and table name components. + They are separated by a '/'. */ + last_slash = strrchr((char*) tablename, '/'); + base_name = last_slash ? last_slash + 1 : (char*) tablename; + + /* Find the offset of the last slash. We will strip off the + old basename.ibd which starts after that slash. */ + last_slash = strrchr((char*) old_path, OS_FILE_PATH_SEPARATOR); + dir_len = last_slash ? last_slash - old_path : strlen(old_path); + + /* allocate a new path and move the old directory path to it. */ + new_path_len = dir_len + strlen(base_name) + sizeof "/.ibd"; + new_path = static_cast(mem_alloc(new_path_len)); + memcpy(new_path, old_path, dir_len); + + ut_snprintf(new_path + dir_len, + new_path_len - dir_len, + "%c%s.ibd", + OS_FILE_PATH_SEPARATOR, + base_name); + + return(new_path); +} + +/****************************************************************//** +This function returns a remote path name by combining a data directory +path provided in a DATA DIRECTORY clause with the tablename which is +in the form 'database/tablename'. It strips the file basename (which +is the tablename) found after the last directory in the path provided. +The full filepath created will include the database name as a directory +under the path provided. The filename is the tablename with the '.ibd' +extension. All input and output strings are null-terminated. + +This function allocates memory to be returned. It is the callers +responsibility to free the return value after it is no longer needed. + +@return own: A full pathname; data_dir_path/databasename/tablename.ibd */ +UNIV_INTERN +char* +os_file_make_remote_pathname( +/*=========================*/ + const char* data_dir_path, /*!< in: pathname */ + const char* tablename, /*!< in: tablename */ + const char* extention) /*!< in: file extention; ibd,cfg */ +{ + ulint data_dir_len; + char* last_slash; + char* new_path; + ulint new_path_len; + + ut_ad(extention && strlen(extention) == 3); + + /* Find the offset of the last slash. We will strip off the + old basename or tablename which starts after that slash. */ + last_slash = strrchr((char*) data_dir_path, OS_FILE_PATH_SEPARATOR); + data_dir_len = last_slash ? last_slash - data_dir_path : strlen(data_dir_path); + + /* allocate a new path and move the old directory path to it. */ + new_path_len = data_dir_len + strlen(tablename) + + sizeof "/." + strlen(extention); + new_path = static_cast(mem_alloc(new_path_len)); + memcpy(new_path, data_dir_path, data_dir_len); + ut_snprintf(new_path + data_dir_len, + new_path_len - data_dir_len, + "%c%s.%s", + OS_FILE_PATH_SEPARATOR, + tablename, + extention); + + srv_normalize_path_for_win(new_path); + + return(new_path); +} + +/****************************************************************//** +This function reduces a null-terminated full remote path name into +the path that is sent by MySQL for DATA DIRECTORY clause. It replaces +the 'databasename/tablename.ibd' found at the end of the path with just +'tablename'. + +Since the result is always smaller than the path sent in, no new memory +is allocated. The caller should allocate memory for the path sent in. +This function manipulates that path in place. + +If the path format is not as expected, just return. The result is used +to inform a SHOW CREATE TABLE command. */ +UNIV_INTERN +void +os_file_make_data_dir_path( +/*========================*/ + char* data_dir_path) /*!< in/out: full path/data_dir_path */ +{ + char* ptr; + char* tablename; + ulint tablename_len; + + /* Replace the period before the extension with a null byte. */ + ptr = strrchr((char*) data_dir_path, '.'); + if (!ptr) { + return; + } + ptr[0] = '\0'; + + /* The tablename starts after the last slash. */ + ptr = strrchr((char*) data_dir_path, OS_FILE_PATH_SEPARATOR); + if (!ptr) { + return; + } + ptr[0] = '\0'; + tablename = ptr + 1; + + /* The databasename starts after the next to last slash. */ + ptr = strrchr((char*) data_dir_path, OS_FILE_PATH_SEPARATOR); + if (!ptr) { + return; + } + tablename_len = ut_strlen(tablename); + + ut_memmove(++ptr, tablename, tablename_len); + + ptr[tablename_len] = '\0'; +} + +/****************************************************************//** +The function os_file_dirname returns a directory component of a +null-terminated pathname string. In the usual case, dirname returns +the string up to, but not including, the final '/', and basename +is the component following the final '/'. Trailing '/' characters +are not counted as part of the pathname. + +If path does not contain a slash, dirname returns the string ".". + +Concatenating the string returned by dirname, a "/", and the basename +yields a complete pathname. + +The return value is a copy of the directory component of the pathname. +The copy is allocated from heap. It is the caller responsibility +to free it after it is no longer needed. + +The following list of examples (taken from SUSv2) shows the strings +returned by dirname and basename for different paths: + + path dirname basename + "/usr/lib" "/usr" "lib" + "/usr/" "/" "usr" + "usr" "." "usr" + "/" "/" "/" + "." "." "." + ".." "." ".." + +@return own: directory component of the pathname */ +UNIV_INTERN +char* +os_file_dirname( +/*============*/ + const char* path) /*!< in: pathname */ +{ + /* Find the offset of the last slash */ + const char* last_slash = strrchr(path, OS_FILE_PATH_SEPARATOR); + if (!last_slash) { + /* No slash in the path, return "." */ + + return(mem_strdup(".")); + } + + /* Ok, there is a slash */ + + if (last_slash == path) { + /* last slash is the first char of the path */ + + return(mem_strdup("/")); + } + + /* Non-trivial directory component */ + + return(mem_strdupl(path, last_slash - path)); +} + +/****************************************************************//** +Creates all missing subdirectories along the given path. +@return TRUE if call succeeded FALSE otherwise */ +UNIV_INTERN +ibool +os_file_create_subdirs_if_needed( +/*=============================*/ + const char* path) /*!< in: path name */ +{ + if (srv_read_only_mode) { + + ib_logf(IB_LOG_LEVEL_ERROR, + "read only mode set. Can't create subdirectories '%s'", + path); + + return(FALSE); + + } + + char* subdir = os_file_dirname(path); + + if (strlen(subdir) == 1 + && (*subdir == OS_FILE_PATH_SEPARATOR || *subdir == '.')) { + /* subdir is root or cwd, nothing to do */ + mem_free(subdir); + + return(TRUE); + } + + /* Test if subdir exists */ + os_file_type_t type; + ibool subdir_exists; + ibool success = os_file_status(subdir, &subdir_exists, &type); + + if (success && !subdir_exists) { + + /* subdir does not exist, create it */ + success = os_file_create_subdirs_if_needed(subdir); + + if (!success) { + mem_free(subdir); + + return(FALSE); + } + + success = os_file_create_directory(subdir, FALSE); + } + + mem_free(subdir); + + return(success); +} + +#ifndef UNIV_HOTBACKUP +/****************************************************************//** +Returns a pointer to the nth slot in the aio array. +@return pointer to slot */ +static +os_aio_slot_t* +os_aio_array_get_nth_slot( +/*======================*/ + os_aio_array_t* array, /*!< in: aio array */ + ulint index) /*!< in: index of the slot */ +{ + ut_a(index < array->n_slots); + + return(&array->slots[index]); +} + +#if defined(LINUX_NATIVE_AIO) +/******************************************************************//** +Creates an io_context for native linux AIO. +@return TRUE on success. */ +static +ibool +os_aio_linux_create_io_ctx( +/*=======================*/ + ulint max_events, /*!< in: number of events. */ + io_context_t* io_ctx) /*!< out: io_ctx to initialize. */ +{ + int ret; + ulint retries = 0; + +retry: + memset(io_ctx, 0x0, sizeof(*io_ctx)); + + /* Initialize the io_ctx. Tell it how many pending + IO requests this context will handle. */ + + ret = io_setup(max_events, io_ctx); + if (ret == 0) { +#if defined(UNIV_AIO_DEBUG) + fprintf(stderr, + "InnoDB: Linux native AIO:" + " initialized io_ctx for segment\n"); +#endif + /* Success. Return now. */ + return(TRUE); + } + + /* If we hit EAGAIN we'll make a few attempts before failing. */ + + switch (ret) { + case -EAGAIN: + if (retries == 0) { + /* First time around. */ + ut_print_timestamp(stderr); + fprintf(stderr, + " InnoDB: Warning: io_setup() failed" + " with EAGAIN. Will make %d attempts" + " before giving up.\n", + OS_AIO_IO_SETUP_RETRY_ATTEMPTS); + } + + if (retries < OS_AIO_IO_SETUP_RETRY_ATTEMPTS) { + ++retries; + fprintf(stderr, + "InnoDB: Warning: io_setup() attempt" + " %lu failed.\n", + retries); + os_thread_sleep(OS_AIO_IO_SETUP_RETRY_SLEEP); + goto retry; + } + + /* Have tried enough. Better call it a day. */ + ut_print_timestamp(stderr); + fprintf(stderr, + " InnoDB: Error: io_setup() failed" + " with EAGAIN after %d attempts.\n", + OS_AIO_IO_SETUP_RETRY_ATTEMPTS); + break; + + case -ENOSYS: + ut_print_timestamp(stderr); + fprintf(stderr, + " InnoDB: Error: Linux Native AIO interface" + " is not supported on this platform. Please" + " check your OS documentation and install" + " appropriate binary of InnoDB.\n"); + + break; + + default: + ut_print_timestamp(stderr); + fprintf(stderr, + " InnoDB: Error: Linux Native AIO setup" + " returned following error[%d]\n", -ret); + break; + } + + fprintf(stderr, + "InnoDB: You can disable Linux Native AIO by" + " setting innodb_use_native_aio = 0 in my.cnf\n"); + return(FALSE); +} + +/******************************************************************//** +Checks if the system supports native linux aio. On some kernel +versions where native aio is supported it won't work on tmpfs. In such +cases we can't use native aio as it is not possible to mix simulated +and native aio. +@return: TRUE if supported, FALSE otherwise. */ +static +ibool +os_aio_native_aio_supported(void) +/*=============================*/ +{ + int fd; + io_context_t io_ctx; + char name[1000]; + + if (!os_aio_linux_create_io_ctx(1, &io_ctx)) { + /* The platform does not support native aio. */ + return(FALSE); + } else if (!srv_read_only_mode) { + /* Now check if tmpdir supports native aio ops. */ + fd = innobase_mysql_tmpfile(); + + if (fd < 0) { + ib_logf(IB_LOG_LEVEL_WARN, + "Unable to create temp file to check " + "native AIO support."); + + return(FALSE); + } + } else { + + srv_normalize_path_for_win(srv_log_group_home_dir); + + ulint dirnamelen = strlen(srv_log_group_home_dir); + ut_a(dirnamelen < (sizeof name) - 10 - sizeof "ib_logfile"); + memcpy(name, srv_log_group_home_dir, dirnamelen); + + /* Add a path separator if needed. */ + if (dirnamelen && name[dirnamelen - 1] != SRV_PATH_SEPARATOR) { + name[dirnamelen++] = SRV_PATH_SEPARATOR; + } + + strcpy(name + dirnamelen, "ib_logfile0"); + + fd = ::open(name, O_RDONLY); + + if (fd == -1) { + + ib_logf(IB_LOG_LEVEL_WARN, + "Unable to open \"%s\" to check " + "native AIO read support.", name); + + return(FALSE); + } + } + + struct io_event io_event; + + memset(&io_event, 0x0, sizeof(io_event)); + + byte* buf = static_cast(ut_malloc(UNIV_PAGE_SIZE * 2)); + byte* ptr = static_cast(ut_align(buf, UNIV_PAGE_SIZE)); + + struct iocb iocb; + + /* Suppress valgrind warning. */ + memset(buf, 0x00, UNIV_PAGE_SIZE * 2); + memset(&iocb, 0x0, sizeof(iocb)); + + struct iocb* p_iocb = &iocb; + + if (!srv_read_only_mode) { + io_prep_pwrite(p_iocb, fd, ptr, UNIV_PAGE_SIZE, 0); + } else { + ut_a(UNIV_PAGE_SIZE >= 512); + io_prep_pread(p_iocb, fd, ptr, 512, 0); + } + + int err = io_submit(io_ctx, 1, &p_iocb); + + if (err >= 1) { + /* Now collect the submitted IO request. */ + err = io_getevents(io_ctx, 1, 1, &io_event, NULL); + } + + ut_free(buf); + close(fd); + + switch (err) { + case 1: + return(TRUE); + + case -EINVAL: + case -ENOSYS: + ib_logf(IB_LOG_LEVEL_ERROR, + "Linux Native AIO not supported. You can either " + "move %s to a file system that supports native " + "AIO or you can set innodb_use_native_aio to " + "FALSE to avoid this message.", + srv_read_only_mode ? name : "tmpdir"); + + /* fall through. */ + default: + ib_logf(IB_LOG_LEVEL_ERROR, + "Linux Native AIO check on %s returned error[%d]", + srv_read_only_mode ? name : "tmpdir", -err); + } + + return(FALSE); +} +#endif /* LINUX_NATIVE_AIO */ + +/******************************************************************//** +Creates an aio wait array. Note that we return NULL in case of failure. +We don't care about freeing memory here because we assume that a +failure will result in server refusing to start up. +@return own: aio array, NULL on failure */ +static +os_aio_array_t* +os_aio_array_create( +/*================*/ + ulint n, /*!< in: maximum number of pending aio + operations allowed; n must be + divisible by n_segments */ + ulint n_segments) /*!< in: number of segments in the aio array */ +{ + os_aio_array_t* array; +#ifdef LINUX_NATIVE_AIO + struct io_event* io_event = NULL; +#endif + ut_a(n > 0); + ut_a(n_segments > 0); + + array = static_cast(ut_malloc(sizeof(*array))); + memset(array, 0x0, sizeof(*array)); + + array->mutex = os_mutex_create(); + array->not_full = os_event_create(); + array->is_empty = os_event_create(); + + os_event_set(array->is_empty); + + array->n_slots = n; + array->n_segments = n_segments; + + array->slots = static_cast( + ut_malloc(n * sizeof(*array->slots))); + + memset(array->slots, 0x0, n * sizeof(*array->slots)); + +#if defined(LINUX_NATIVE_AIO) + array->aio_ctx = NULL; + array->aio_events = NULL; + + /* If we are not using native aio interface then skip this + part of initialization. */ + if (!srv_use_native_aio) { + goto skip_native_aio; + } + + /* Initialize the io_context array. One io_context + per segment in the array. */ + + array->aio_ctx = static_cast( + ut_malloc(n_segments * sizeof(*array->aio_ctx))); + + for (ulint i = 0; i < n_segments; ++i) { + if (!os_aio_linux_create_io_ctx(n/n_segments, + &array->aio_ctx[i])) { + /* If something bad happened during aio setup + we disable linux native aio. + The disadvantage will be a small memory leak + at shutdown but that's ok compared to a crash + or a not working server. + This frequently happens when running the test suite + with many threads on a system with low fs.aio-max-nr! + */ + + fprintf(stderr, + " InnoDB: Warning: Linux Native AIO disabled " + "because os_aio_linux_create_io_ctx() " + "failed. To get rid of this warning you can " + "try increasing system " + "fs.aio-max-nr to 1048576 or larger or " + "setting innodb_use_native_aio = 0 in my.cnf\n"); + srv_use_native_aio = FALSE; + goto skip_native_aio; + } + } + + /* Initialize the event array. One event per slot. */ + io_event = static_cast( + ut_malloc(n * sizeof(*io_event))); + + memset(io_event, 0x0, sizeof(*io_event) * n); + array->aio_events = io_event; + +skip_native_aio: +#endif /* LINUX_NATIVE_AIO */ + for (ulint i = 0; i < n; i++) { + os_aio_slot_t* slot; + + slot = os_aio_array_get_nth_slot(array, i); + slot->pos = i; + slot->reserved = FALSE; +#ifdef LINUX_NATIVE_AIO + memset(&slot->control, 0x0, sizeof(slot->control)); + slot->n_bytes = 0; + slot->ret = 0; +#endif /* WIN_ASYNC_IO */ + } + + return(array); +} + +/************************************************************************//** +Frees an aio wait array. */ +static +void +os_aio_array_free( +/*==============*/ + os_aio_array_t*& array) /*!< in, own: array to free */ +{ + ulint i; + + os_mutex_free(array->mutex); + os_event_free(array->not_full); + os_event_free(array->is_empty); + +#if defined(LINUX_NATIVE_AIO) + if (srv_use_native_aio) { + ut_free(array->aio_events); + ut_free(array->aio_ctx); + } +#endif /* LINUX_NATIVE_AIO */ + + for (i = 0; i < array->n_slots; i++) { + os_aio_slot_t* slot = os_aio_array_get_nth_slot(array, i); + if (slot->page_compression_page) { + ut_free(slot->page_compression_page); + slot->page_compression_page = NULL; + } + + if (slot->lzo_mem) { + ut_free(slot->lzo_mem); + slot->lzo_mem = NULL; + } + } + + for (i = 0; i < array->n_slots; i++) { + os_aio_slot_t* slot = os_aio_array_get_nth_slot(array, i); + if (slot->page_encryption_page) { + ut_free(slot->page_encryption_page); + slot->page_encryption_page = NULL; + } + } + + + ut_free(array->slots); + ut_free(array); + + array = 0; +} + +/*********************************************************************** +Initializes the asynchronous io system. Creates one array each for ibuf +and log i/o. Also creates one array each for read and write where each +array is divided logically into n_read_segs and n_write_segs +respectively. The caller must create an i/o handler thread for each +segment in these arrays. This function also creates the sync array. +No i/o handler thread needs to be created for that */ +UNIV_INTERN +ibool +os_aio_init( +/*========*/ + ulint n_per_seg, /*= 4); + } else { + ut_ad(n_segments > 0); + } + + os_aio_sync_array = os_aio_array_create(n_slots_sync, 1); + + if (os_aio_sync_array == NULL) { + return(FALSE); + } + + os_aio_n_segments = n_segments; + + os_aio_validate(); + + os_aio_segment_wait_events = static_cast( + ut_malloc(n_segments * sizeof *os_aio_segment_wait_events)); + + for (ulint i = 0; i < n_segments; ++i) { + os_aio_segment_wait_events[i] = os_event_create(); + } + + os_last_printout = ut_time(); + +#ifdef _WIN32 + ut_a(completion_port == 0 && read_completion_port == 0); + completion_port = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0); + read_completion_port = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0); + ut_a(completion_port && read_completion_port); +#endif + + return(TRUE); + +} + +/*********************************************************************** +Frees the asynchronous io system. */ +UNIV_INTERN +void +os_aio_free(void) +/*=============*/ +{ + if (os_aio_ibuf_array != 0) { + os_aio_array_free(os_aio_ibuf_array); + } + + if (os_aio_log_array != 0) { + os_aio_array_free(os_aio_log_array); + } + + if (os_aio_write_array != 0) { + os_aio_array_free(os_aio_write_array); + } + + if (os_aio_sync_array != 0) { + os_aio_array_free(os_aio_sync_array); + } + + os_aio_array_free(os_aio_read_array); + + for (ulint i = 0; i < os_aio_n_segments; i++) { + os_event_free(os_aio_segment_wait_events[i]); + } + + ut_free(os_aio_segment_wait_events); + os_aio_segment_wait_events = 0; + os_aio_n_segments = 0; +#ifdef _WIN32 + completion_port = 0; + read_completion_port = 0; +#endif +} + +#ifdef WIN_ASYNC_IO +/************************************************************************//** +Wakes up all async i/o threads in the array in Windows async i/o at +shutdown. */ +static +void +os_aio_array_wake_win_aio_at_shutdown( +/*==================================*/ + os_aio_array_t* array) /*!< in: aio array */ +{ + if(completion_port) + { + PostQueuedCompletionStatus(completion_port, 0, IOCP_SHUTDOWN_KEY, NULL); + PostQueuedCompletionStatus(read_completion_port, 0, IOCP_SHUTDOWN_KEY, NULL); + } +} +#endif + +/************************************************************************//** +Wakes up all async i/o threads so that they know to exit themselves in +shutdown. */ +UNIV_INTERN +void +os_aio_wake_all_threads_at_shutdown(void) +/*=====================================*/ +{ +#ifdef WIN_ASYNC_IO + /* This code wakes up all ai/o threads in Windows native aio */ + os_aio_array_wake_win_aio_at_shutdown(os_aio_read_array); + if (os_aio_write_array != 0) { + os_aio_array_wake_win_aio_at_shutdown(os_aio_write_array); + } + + if (os_aio_ibuf_array != 0) { + os_aio_array_wake_win_aio_at_shutdown(os_aio_ibuf_array); + } + + if (os_aio_log_array != 0) { + os_aio_array_wake_win_aio_at_shutdown(os_aio_log_array); + } + +#elif defined(LINUX_NATIVE_AIO) + + /* When using native AIO interface the io helper threads + wait on io_getevents with a timeout value of 500ms. At + each wake up these threads check the server status. + No need to do anything to wake them up. */ + + if (srv_use_native_aio) { + return; + } + + /* Fall through to simulated AIO handler wakeup if we are + not using native AIO. */ +#endif /* !WIN_ASYNC_AIO */ + + /* This loop wakes up all simulated ai/o threads */ + + for (ulint i = 0; i < os_aio_n_segments; i++) { + + os_event_set(os_aio_segment_wait_events[i]); + } +} + +/************************************************************************//** +Waits until there are no pending writes in os_aio_write_array. There can +be other, synchronous, pending writes. */ +UNIV_INTERN +void +os_aio_wait_until_no_pending_writes(void) +/*=====================================*/ +{ + ut_ad(!srv_read_only_mode); + os_event_wait(os_aio_write_array->is_empty); +} + +/**********************************************************************//** +Calculates segment number for a slot. +@return segment number (which is the number used by, for example, +i/o-handler threads) */ +static +ulint +os_aio_get_segment_no_from_slot( +/*============================*/ + os_aio_array_t* array, /*!< in: aio wait array */ + os_aio_slot_t* slot) /*!< in: slot in this array */ +{ + ulint segment; + ulint seg_len; + + if (array == os_aio_ibuf_array) { + ut_ad(!srv_read_only_mode); + + segment = IO_IBUF_SEGMENT; + + } else if (array == os_aio_log_array) { + ut_ad(!srv_read_only_mode); + + segment = IO_LOG_SEGMENT; + + } else if (array == os_aio_read_array) { + seg_len = os_aio_read_array->n_slots + / os_aio_read_array->n_segments; + + segment = (srv_read_only_mode ? 0 : 2) + slot->pos / seg_len; + } else { + ut_ad(!srv_read_only_mode); + ut_a(array == os_aio_write_array); + + seg_len = os_aio_write_array->n_slots + / os_aio_write_array->n_segments; + + segment = os_aio_read_array->n_segments + 2 + + slot->pos / seg_len; + } + + return(segment); +} + +/**********************************************************************//** +Calculates local segment number and aio array from global segment number. +@return local segment number within the aio array */ +static +ulint +os_aio_get_array_and_local_segment( +/*===============================*/ + os_aio_array_t** array, /*!< out: aio wait array */ + ulint global_segment)/*!< in: global segment number */ +{ + ulint segment; + + ut_a(global_segment < os_aio_n_segments); + + if (srv_read_only_mode) { + *array = os_aio_read_array; + + return(global_segment); + } else if (global_segment == IO_IBUF_SEGMENT) { + *array = os_aio_ibuf_array; + segment = 0; + + } else if (global_segment == IO_LOG_SEGMENT) { + *array = os_aio_log_array; + segment = 0; + + } else if (global_segment < os_aio_read_array->n_segments + 2) { + *array = os_aio_read_array; + + segment = global_segment - 2; + } else { + *array = os_aio_write_array; + + segment = global_segment - (os_aio_read_array->n_segments + 2); + } + + return(segment); +} + +/*******************************************************************//** +Requests for a slot in the aio array. If no slot is available, waits until +not_full-event becomes signaled. +@return pointer to slot */ +static +os_aio_slot_t* +os_aio_array_reserve_slot( +/*======================*/ + ulint type, /*!< in: OS_FILE_READ or OS_FILE_WRITE */ + os_aio_array_t* array, /*!< in: aio array */ + fil_node_t* message1,/*!< in: message to be passed along with + the aio operation */ + void* message2,/*!< in: message to be passed along with + the aio operation */ + os_file_t file, /*!< in: file handle */ + const char* name, /*!< in: name of the file or path as a + null-terminated string */ + void* buf, /*!< in: buffer where to read or from which + to write */ + os_offset_t offset, /*!< in: file offset */ + ulint len, /*!< in: length of the block to read or write */ + ulint space_id, + ibool page_compression, /*!< in: is page compression used + on this file space */ + ulint page_compression_level, /*!< page compression + level to be used */ + ibool page_encryption, /*!< in: is page encryption used + on this file space */ + ulint page_encryption_key, /*!< page encryption key + to be used */ + ulint* write_size)/*!< in/out: Actual write size initialized + after fist successfull trim + operation for this page and if + initialized we do not trim again if + actual page size does not decrease. */ +{ + os_aio_slot_t* slot = NULL; +#ifdef WIN_ASYNC_IO + OVERLAPPED* control; + +#elif defined(LINUX_NATIVE_AIO) + + struct iocb* iocb; + off_t aio_offset; + +#endif /* WIN_ASYNC_IO */ + ulint i; + ulint counter; + ulint slots_per_seg; + ulint local_seg; + +#ifdef WIN_ASYNC_IO + ut_a((len & 0xFFFFFFFFUL) == len); +#endif /* WIN_ASYNC_IO */ + + /* No need of a mutex. Only reading constant fields */ + slots_per_seg = array->n_slots / array->n_segments; + + /* We attempt to keep adjacent blocks in the same local + segment. This can help in merging IO requests when we are + doing simulated AIO */ + local_seg = (offset >> (UNIV_PAGE_SIZE_SHIFT + 6)) + % array->n_segments; + +loop: + os_mutex_enter(array->mutex); + + if (array->n_reserved == array->n_slots) { + os_mutex_exit(array->mutex); + + if (!srv_use_native_aio) { + /* If the handler threads are suspended, wake them + so that we get more slots */ + + os_aio_simulated_wake_handler_threads(); + } + + os_event_wait(array->not_full); + + goto loop; + } + + /* We start our search for an available slot from our preferred + local segment and do a full scan of the array. We are + guaranteed to find a slot in full scan. */ + for (i = local_seg * slots_per_seg, counter = 0; + counter < array->n_slots; + i++, counter++) { + + i %= array->n_slots; + + slot = os_aio_array_get_nth_slot(array, i); + + if (slot->reserved == FALSE) { + goto found; + } + } + + /* We MUST always be able to get hold of a reserved slot. */ + ut_error; + +found: + ut_a(slot->reserved == FALSE); + array->n_reserved++; + + if (array->n_reserved == 1) { + os_event_reset(array->is_empty); + } + + if (array->n_reserved == array->n_slots) { + os_event_reset(array->not_full); + } + + slot->reserved = TRUE; + slot->reservation_time = ut_time(); + slot->message1 = message1; + slot->message2 = message2; + slot->file = file; + slot->name = name; + slot->len = len; + slot->type = type; + slot->buf = static_cast(buf); + slot->offset = offset; + slot->io_already_done = FALSE; + slot->space_id = space_id; + + slot->page_compress_success = FALSE; + slot->page_encryption_success = FALSE; + + slot->write_size = write_size; + slot->page_compression_level = page_compression_level; + slot->page_compression = page_compression; + slot->page_encryption_key = page_encryption_key; + slot->page_encryption = page_encryption; + + /* If the space is page compressed and this is write operation + then we compress the page */ + if (message1 && type == OS_FILE_WRITE && page_compression ) { + ulint real_len = len; + byte* tmp = NULL; + + /* Release the array mutex while compressing */ + os_mutex_exit(array->mutex); + + // We allocate memory for page compressed buffer if and only + // if it is not yet allocated. + if (slot->page_buf == NULL) { + os_slot_alloc_page_buf(slot); + } + +#ifdef HAVE_LZO + if (innodb_compression_algorithm == 3 && slot->lzo_mem == NULL) { + os_slot_alloc_lzo_mem(slot); + } +#endif + + /* Call page compression */ + tmp = fil_compress_page(fil_node_get_space_id(slot->message1), + (byte *)buf, + slot->page_buf, + len, + page_compression_level, + &real_len, + slot->lzo_mem + ); + + /* If compression succeeded, set up the length and buffer */ + if (tmp != buf) { + len = real_len; + buf = slot->page_buf; + slot->len = real_len; + slot->page_compress_success = TRUE; + } else { + slot->page_compress_success = FALSE; + } + + /* Take array mutex back */ + os_mutex_enter(array->mutex); + + } //CMD + /* If the space is page encryption and this is write operation + then we encrypt the page */ + if (message1 && type == OS_FILE_WRITE && page_encryption ) { + ulint real_len = len; + byte* tmp = NULL; + + /* Release the array mutex while encrypting */ + os_mutex_exit(array->mutex); + + // We allocate memory for page encrypted buffer if and only + // if it is not yet allocated. + if (slot->page_buf2 == NULL) { + os_slot_alloc_page_buf2(slot); + } + + ut_ad(slot->page_buf2); + tmp = fil_encrypt_page(fil_node_get_space_id(slot->message1), (byte *)buf, slot->page_buf2, len, page_encryption_key, &real_len); + + /* If encryption succeeded, set up the length and buffer */ + if (tmp != buf) { + len = real_len; + buf = slot->page_buf2; + slot->len = real_len; + slot->page_encryption_success = TRUE; + } else { + slot->page_encryption_success = FALSE; + } + + /* Take array mutex back */ + os_mutex_enter(array->mutex); + } + +#ifdef WIN_ASYNC_IO + control = &slot->control; + control->Offset = (DWORD) offset & 0xFFFFFFFF; + control->OffsetHigh = (DWORD) (offset >> 32); + control->hEvent = 0; + slot->arr = array; + +#elif defined(LINUX_NATIVE_AIO) + + /* If we are not using native AIO skip this part. */ + if (!srv_use_native_aio) { + goto skip_native_aio; + } + + /* Check if we are dealing with 64 bit arch. + If not then make sure that offset fits in 32 bits. */ + aio_offset = (off_t) offset; + + ut_a(sizeof(aio_offset) >= sizeof(offset) + || ((os_offset_t) aio_offset) == offset); + + iocb = &slot->control; + + if (type == OS_FILE_READ) { + io_prep_pread(iocb, file, buf, len, aio_offset); + } else { + ut_a(type == OS_FILE_WRITE); + io_prep_pwrite(iocb, file, buf, len, aio_offset); + } + + iocb->data = (void*) slot; + slot->n_bytes = 0; + slot->ret = 0; + +skip_native_aio: +#endif /* LINUX_NATIVE_AIO */ + os_mutex_exit(array->mutex); + + return(slot); +} + +/*******************************************************************//** +Frees a slot in the aio array. */ +static +void +os_aio_array_free_slot( +/*===================*/ + os_aio_array_t* array, /*!< in: aio array */ + os_aio_slot_t* slot) /*!< in: pointer to slot */ +{ + os_mutex_enter(array->mutex); + + ut_ad(slot->reserved); + + slot->reserved = FALSE; + + array->n_reserved--; + + if (array->n_reserved == array->n_slots - 1) { + os_event_set(array->not_full); + } + + if (array->n_reserved == 0) { + os_event_set(array->is_empty); + } + +#ifdef LINUX_NATIVE_AIO + + if (srv_use_native_aio) { + memset(&slot->control, 0x0, sizeof(slot->control)); + slot->n_bytes = 0; + slot->ret = 0; + /*fprintf(stderr, "Freed up Linux native slot.\n");*/ + } else { + /* These fields should not be used if we are not + using native AIO. */ + ut_ad(slot->n_bytes == 0); + ut_ad(slot->ret == 0); + } + +#endif + os_mutex_exit(array->mutex); +} + +/**********************************************************************//** +Wakes up a simulated aio i/o-handler thread if it has something to do. */ +static +void +os_aio_simulated_wake_handler_thread( +/*=================================*/ + ulint global_segment) /*!< in: the number of the segment in the aio + arrays */ +{ + os_aio_array_t* array; + ulint segment; + + ut_ad(!srv_use_native_aio); + + segment = os_aio_get_array_and_local_segment(&array, global_segment); + + ulint n = array->n_slots / array->n_segments; + + segment *= n; + + /* Look through n slots after the segment * n'th slot */ + + os_mutex_enter(array->mutex); + + for (ulint i = 0; i < n; ++i) { + const os_aio_slot_t* slot; + + slot = os_aio_array_get_nth_slot(array, segment + i); + + if (slot->reserved) { + + /* Found an i/o request */ + + os_mutex_exit(array->mutex); + + os_event_t event; + + event = os_aio_segment_wait_events[global_segment]; + + os_event_set(event); + + return; + } + } + + os_mutex_exit(array->mutex); +} + +/**********************************************************************//** +Wakes up simulated aio i/o-handler threads if they have something to do. */ +UNIV_INTERN +void +os_aio_simulated_wake_handler_threads(void) +/*=======================================*/ +{ + if (srv_use_native_aio) { + /* We do not use simulated aio: do nothing */ + + return; + } + + os_aio_recommend_sleep_for_read_threads = FALSE; + + for (ulint i = 0; i < os_aio_n_segments; i++) { + os_aio_simulated_wake_handler_thread(i); + } +} + +/**********************************************************************//** +This function can be called if one wants to post a batch of reads and +prefers an i/o-handler thread to handle them all at once later. You must +call os_aio_simulated_wake_handler_threads later to ensure the threads +are not left sleeping! */ +UNIV_INTERN +void +os_aio_simulated_put_read_threads_to_sleep(void) +/*============================================*/ +{ + +/* The idea of putting background IO threads to sleep is only for +Windows when using simulated AIO. Windows XP seems to schedule +background threads too eagerly to allow for coalescing during +readahead requests. */ +#ifdef __WIN__ + os_aio_array_t* array; + + if (srv_use_native_aio) { + /* We do not use simulated aio: do nothing */ + + return; + } + + os_aio_recommend_sleep_for_read_threads = TRUE; + + for (ulint i = 0; i < os_aio_n_segments; i++) { + os_aio_get_array_and_local_segment(&array, i); + + if (array == os_aio_read_array) { + + os_event_reset(os_aio_segment_wait_events[i]); + } + } +#endif /* __WIN__ */ +} + +#if defined(LINUX_NATIVE_AIO) +/*******************************************************************//** +Dispatch an AIO request to the kernel. +@return TRUE on success. */ +static +ibool +os_aio_linux_dispatch( +/*==================*/ + os_aio_array_t* array, /*!< in: io request array. */ + os_aio_slot_t* slot) /*!< in: an already reserved slot. */ +{ + int ret; + ulint io_ctx_index; + struct iocb* iocb; + + ut_ad(slot != NULL); + ut_ad(array); + + ut_a(slot->reserved); + + /* Find out what we are going to work with. + The iocb struct is directly in the slot. + The io_context is one per segment. */ + + iocb = &slot->control; + io_ctx_index = (slot->pos * array->n_segments) / array->n_slots; + + ret = io_submit(array->aio_ctx[io_ctx_index], 1, &iocb); + +#if defined(UNIV_AIO_DEBUG) + fprintf(stderr, + "io_submit[%c] ret[%d]: slot[%p] ctx[%p] seg[%lu]\n", + (slot->type == OS_FILE_WRITE) ? 'w' : 'r', ret, slot, + array->aio_ctx[io_ctx_index], (ulong) io_ctx_index); +#endif + + /* io_submit returns number of successfully + queued requests or -errno. */ + if (UNIV_UNLIKELY(ret != 1)) { + errno = -ret; + return(FALSE); + } + + return(TRUE); +} +#endif /* LINUX_NATIVE_AIO */ + + +/*******************************************************************//** +NOTE! Use the corresponding macro os_aio(), not directly this function! +Requests an asynchronous i/o operation. +@return TRUE if request was queued successfully, FALSE if fail */ +UNIV_INTERN +ibool +os_aio_func( +/*========*/ + ulint type, /*!< in: OS_FILE_READ or OS_FILE_WRITE */ + ulint mode, /*!< in: OS_AIO_NORMAL, ..., possibly ORed + to OS_AIO_SIMULATED_WAKE_LATER: the + last flag advises this function not to wake + i/o-handler threads, but the caller will + do the waking explicitly later, in this + way the caller can post several requests in + a batch; NOTE that the batch must not be + so big that it exhausts the slots in aio + arrays! NOTE that a simulated batch + may introduce hidden chances of deadlocks, + because i/os are not actually handled until + all have been posted: use with great + caution! */ + const char* name, /*!< in: name of the file or path as a + null-terminated string */ + os_file_t file, /*!< in: handle to a file */ + void* buf, /*!< in: buffer where to read or from which + to write */ + os_offset_t offset, /*!< in: file offset where to read or write */ + ulint n, /*!< in: number of bytes to read or write */ + fil_node_t* message1,/*!< in: message for the aio handler + (can be used to identify a completed + aio operation); ignored if mode is + OS_AIO_SYNC */ + void* message2,/*!< in: message for the aio handler + (can be used to identify a completed + aio operation); ignored if mode is + OS_AIO_SYNC */ + ulint space_id, + trx_t* trx, + ibool page_compression, /*!< in: is page compression used + on this file space */ + ulint page_compression_level, /*!< page compression + level to be used */ + ibool page_encryption, /*!< in: is page encryption used + on this file space */ + ulint page_encryption_key, /*!< page encryption key + to be used */ + ulint* write_size)/*!< in/out: Actual write size initialized + after fist successfull trim + operation for this page and if + initialized we do not trim again if + actual page size does not decrease. */ +{ + os_aio_array_t* array; + os_aio_slot_t* slot; +#ifdef WIN_ASYNC_IO + DWORD len = (DWORD) n; + BOOL ret; +#endif + ulint wake_later; + + ut_ad(file); + ut_ad(buf); + ut_ad(n > 0); + ut_ad(n % OS_MIN_LOG_BLOCK_SIZE == 0); + ut_ad(offset % OS_MIN_LOG_BLOCK_SIZE == 0); + ut_ad(os_aio_validate_skip()); +#ifdef WIN_ASYNC_IO + ut_ad((n & 0xFFFFFFFFUL) == n); +#endif + + wake_later = mode & OS_AIO_SIMULATED_WAKE_LATER; + mode = mode & (~OS_AIO_SIMULATED_WAKE_LATER); + + if (mode == OS_AIO_SYNC) + { + ibool ret; + /* This is actually an ordinary synchronous read or write: + no need to use an i/o-handler thread */ + + if (type == OS_FILE_READ) { + ret = os_file_read_func(file, buf, offset, n, trx, + page_compression); + } + else { + ut_ad(!srv_read_only_mode); + ut_a(type == OS_FILE_WRITE); + + ret = os_file_write(name, file, buf, offset, n); + } + ut_a(ret); + return ret; + } + +try_again: + switch (mode) { + case OS_AIO_NORMAL: + if (type == OS_FILE_READ) { + array = os_aio_read_array; + } else { + ut_ad(!srv_read_only_mode); + array = os_aio_write_array; + } + break; + case OS_AIO_IBUF: + ut_ad(type == OS_FILE_READ); + /* Reduce probability of deadlock bugs in connection with ibuf: + do not let the ibuf i/o handler sleep */ + + wake_later = FALSE; + + if (srv_read_only_mode) { + array = os_aio_read_array; + } else { + array = os_aio_ibuf_array; + } + break; + case OS_AIO_LOG: + if (srv_read_only_mode) { + array = os_aio_read_array; + } else { + array = os_aio_log_array; + } + break; + case OS_AIO_SYNC: + array = os_aio_sync_array; +#if defined(LINUX_NATIVE_AIO) + /* In Linux native AIO we don't use sync IO array. */ + ut_a(!srv_use_native_aio); +#endif /* LINUX_NATIVE_AIO */ + break; + default: + ut_error; + array = NULL; /* Eliminate compiler warning */ + } + + if (trx && type == OS_FILE_READ) + { + trx->io_reads++; + trx->io_read += n; + } + slot = os_aio_array_reserve_slot(type, array, message1, message2, file, + name, buf, offset, n, space_id, + page_compression, page_compression_level, + page_encryption, page_encryption_key, write_size); + if (type == OS_FILE_READ) { + if (srv_use_native_aio) { + os_n_file_reads++; + os_bytes_read_since_printout += n; +#ifdef WIN_ASYNC_IO + ret = ReadFile(file, buf, (DWORD) n, &len, + &(slot->control)); + if(!ret && GetLastError() != ERROR_IO_PENDING) + goto err_exit; + +#elif defined(LINUX_NATIVE_AIO) + if (!os_aio_linux_dispatch(array, slot)) { + goto err_exit; + } +#endif /* WIN_ASYNC_IO */ + } else { + if (!wake_later) { + os_aio_simulated_wake_handler_thread( + os_aio_get_segment_no_from_slot( + array, slot)); + } + } + } else if (type == OS_FILE_WRITE) { + ut_ad(!srv_read_only_mode); + if (srv_use_native_aio) { + os_n_file_writes++; +#ifdef WIN_ASYNC_IO + ret = WriteFile(file, buf, (DWORD) n, &len, + &(slot->control)); + + if(!ret && GetLastError() != ERROR_IO_PENDING) + goto err_exit; +#elif defined(LINUX_NATIVE_AIO) + if (!os_aio_linux_dispatch(array, slot)) { + goto err_exit; + } +#endif /* WIN_ASYNC_IO */ + } else { + if (!wake_later) { + os_aio_simulated_wake_handler_thread( + os_aio_get_segment_no_from_slot( + array, slot)); + } + } + } else { + ut_error; + } + + /* aio was queued successfully! */ + return(TRUE); + +#if defined LINUX_NATIVE_AIO || defined WIN_ASYNC_IO +err_exit: +#endif /* LINUX_NATIVE_AIO || WIN_ASYNC_IO */ + os_aio_array_free_slot(array, slot); + + if (os_file_handle_error( + name,type == OS_FILE_READ ? "aio read" : "aio write", __FILE__, __LINE__)) { + + goto try_again; + } + + return(FALSE); +} + +#ifdef WIN_ASYNC_IO +#define READ_SEGMENT(x) (x < srv_n_read_io_threads) +#define WRITE_SEGMENT(x) !READ_SEGMENT(x) + +/**********************************************************************//** +This function is only used in Windows asynchronous i/o. +Waits for an aio operation to complete. This function is used to wait the +for completed requests. The aio array of pending requests is divided +into segments. The thread specifies which segment or slot it wants to wait +for. NOTE: this function will also take care of freeing the aio slot, +therefore no other thread is allowed to do the freeing! +@return TRUE if the aio operation succeeded */ +UNIV_INTERN +ibool +os_aio_windows_handle( +/*==================*/ + ulint segment, /*!< in: the number of the segment in the aio + arrays to wait for; segment 0 is the ibuf + i/o thread, segment 1 the log i/o thread, + then follow the non-ibuf read threads, and as + the last are the non-ibuf write threads; if + this is ULINT_UNDEFINED, then it means that + sync aio is used, and this parameter is + ignored */ + ulint pos, /*!< this parameter is used only in sync aio: + wait for the aio slot at this position */ + fil_node_t**message1, /*!< out: the messages passed with the aio + request; note that also in the case where + the aio operation failed, these output + parameters are valid and can be used to + restart the operation, for example */ + void** message2, + ulint* type, /*!< out: OS_FILE_WRITE or ..._READ */ + ulint* space_id) +{ + ulint orig_seg = segment; + os_aio_slot_t* slot; + ibool ret_val; + BOOL ret; + DWORD len; + BOOL retry = FALSE; + ULONG_PTR key; + HANDLE port = READ_SEGMENT(segment)? read_completion_port : completion_port; + + for(;;) { + ret = GetQueuedCompletionStatus(port, &len, &key, + (OVERLAPPED **)&slot, INFINITE); + + /* If shutdown key was received, repost the shutdown message and exit */ + if (ret && (key == IOCP_SHUTDOWN_KEY)) { + PostQueuedCompletionStatus(port, 0, key, NULL); + os_thread_exit(NULL); + } + + if (srv_shutdown_state == SRV_SHUTDOWN_EXIT_THREADS) { + os_thread_exit(NULL); + } + + if(WRITE_SEGMENT(segment)&& slot->type == OS_FILE_READ) { + /* + Redirect read completions to the dedicated completion port + and thread. We need to split read and write threads. If we do not + do that, and just allow all io threads process all IO, it is possible + to get stuck in a deadlock in buffer pool code, + + Currently, the problem is solved this way - "write io" threads + always get all completion notifications, from both async reads and + writes. Write completion is handled in the same thread that gets it. + Read completion is forwarded via PostQueueCompletionStatus()) + to the second completion port dedicated solely to reads. One of the + "read io" threads waiting on this port will finally handle the IO. + + Forwarding IO completion this way costs a context switch , and this + seems tolerable since asynchronous reads are by far less frequent. + */ + ut_a(PostQueuedCompletionStatus(read_completion_port, len, key, + &slot->control)); + } + else { + break; + } + } + + *message1 = slot->message1; + *message2 = slot->message2; + + *type = slot->type; + *space_id = slot->space_id; + + if (ret && len == slot->len) { + + ret_val = TRUE; + } else if (os_file_handle_error(slot->name, "Windows aio", __FILE__, __LINE__)) { + + retry = TRUE; + } else { + + ret_val = FALSE; + } + + if (retry) { + /* retry failed read/write operation synchronously. + No need to hold array->mutex. */ + +#ifdef UNIV_PFS_IO + /* This read/write does not go through os_file_read + and os_file_write APIs, need to register with + performance schema explicitly here. */ + struct PSI_file_locker* locker = NULL; + register_pfs_file_io_begin(locker, slot->file, slot->len, + (slot->type == OS_FILE_WRITE) + ? PSI_FILE_WRITE + : PSI_FILE_READ, + __FILE__, __LINE__); +#endif + + ut_a((slot->len & 0xFFFFFFFFUL) == slot->len); + + switch (slot->type) { + case OS_FILE_WRITE: + if (slot->message1 && slot->page_compression && slot->page_buf) { + ret_val = os_file_write(slot->name, slot->file, slot->page_buf, + slot->offset, slot->len); + } else { + + ret_val = os_file_write(slot->name, slot->file, slot->buf, + slot->offset, slot->len); + } + break; + case OS_FILE_READ: + ret_val = os_file_read(slot->file, slot->buf, + slot->offset, slot->len, slot->page_compression); + break; + default: + ut_error; + } + +#ifdef UNIV_PFS_IO + register_pfs_file_io_end(locker, len); +#endif + + if (!ret && GetLastError() == ERROR_IO_PENDING) { + /* aio was queued successfully! + We want a synchronous i/o operation on a + file where we also use async i/o: in Windows + we must use the same wait mechanism as for + async i/o */ + + ret = GetOverlappedResult(slot->file, + &(slot->control), + &len, TRUE); + } + + ret_val = ret && len == slot->len; + } + + if (slot->message1 && slot->page_compression) { + // We allocate memory for page compressed buffer if and only + // if it is not yet allocated. + if (slot->page_buf == NULL) { + os_slot_alloc_page_buf(slot); + } + +#ifdef HAVE_LZO + if (innodb_compression_algorithm == 3 && slot->lzo_mem == NULL) { + os_slot_alloc_lzo_mem(slot); + } +#endif + if (slot->type == OS_FILE_READ) { + fil_decompress_page(slot->page_buf, slot->buf, slot->len, slot->write_size); + } else { + if (slot->page_compress_success && fil_page_is_compressed(slot->page_buf)) { + if (srv_use_trim && os_fallocate_failed == FALSE) { + // Deallocate unused blocks from file system + os_file_trim(slot->file, slot, slot->len); + } + } + } + } + + os_aio_array_free_slot((os_aio_array_t *)slot->arr, slot); + + return(ret_val); +} +#endif + +#if defined(LINUX_NATIVE_AIO) +/******************************************************************//** +This function is only used in Linux native asynchronous i/o. This is +called from within the io-thread. If there are no completed IO requests +in the slot array, the thread calls this function to collect more +requests from the kernel. +The io-thread waits on io_getevents(), which is a blocking call, with +a timeout value. Unless the system is very heavy loaded, keeping the +io-thread very busy, the io-thread will spend most of its time waiting +in this function. +The io-thread also exits in this function. It checks server status at +each wakeup and that is why we use timed wait in io_getevents(). */ +static +void +os_aio_linux_collect( +/*=================*/ + os_aio_array_t* array, /*!< in/out: slot array. */ + ulint segment, /*!< in: local segment no. */ + ulint seg_size) /*!< in: segment size. */ +{ + int i; + int ret; + ulint start_pos; + ulint end_pos; + struct timespec timeout; + struct io_event* events; + struct io_context* io_ctx; + + /* sanity checks. */ + ut_ad(array != NULL); + ut_ad(seg_size > 0); + ut_ad(segment < array->n_segments); + + /* Which part of event array we are going to work on. */ + events = &array->aio_events[segment * seg_size]; + + /* Which io_context we are going to use. */ + io_ctx = array->aio_ctx[segment]; + + /* Starting point of the segment we will be working on. */ + start_pos = segment * seg_size; + + /* End point. */ + end_pos = start_pos + seg_size; + +retry: + + /* Initialize the events. The timeout value is arbitrary. + We probably need to experiment with it a little. */ + memset(events, 0, sizeof(*events) * seg_size); + timeout.tv_sec = 0; + timeout.tv_nsec = OS_AIO_REAP_TIMEOUT; + + ret = io_getevents(io_ctx, 1, seg_size, events, &timeout); + + if (ret > 0) { + for (i = 0; i < ret; i++) { + os_aio_slot_t* slot; + struct iocb* control; + + control = (struct iocb*) events[i].obj; + ut_a(control != NULL); + + slot = (os_aio_slot_t*) control->data; + + /* Some sanity checks. */ + ut_a(slot != NULL); + ut_a(slot->reserved); + +#if defined(UNIV_AIO_DEBUG) + fprintf(stderr, + "io_getevents[%c]: slot[%p] ctx[%p]" + " seg[%lu]\n", + (slot->type == OS_FILE_WRITE) ? 'w' : 'r', + slot, io_ctx, segment); +#endif + + /* We are not scribbling previous segment. */ + ut_a(slot->pos >= start_pos); + + /* We have not overstepped to next segment. */ + ut_a(slot->pos < end_pos); + + /* If the table is page compressed and this is read, + we decompress before we annouce the read is + complete. For writes, we free the compressed page. */ + if (slot->message1 && slot->page_compression) { + // We allocate memory for page compressed buffer if and only + // if it is not yet allocated. + if (slot->page_buf == NULL) { + os_slot_alloc_page_buf(slot); + } + +#ifdef HAVE_LZO + if (innodb_compression_algorithm == 3 && slot->lzo_mem == NULL) { + os_slot_alloc_lzo_mem(slot); + } +#endif + if (slot->type == OS_FILE_READ) { + fil_decompress_page(slot->page_buf, slot->buf, slot->len, slot->write_size); + } else { + if (slot->page_compress_success && + fil_page_is_compressed(slot->page_buf)) { + ut_ad(slot->page_compression_page); + if (srv_use_trim && os_fallocate_failed == FALSE) { + // Deallocate unused blocks from file system + os_file_trim(slot->file, slot, slot->len); + } + } + } + } + + /* page encryption */ + if (slot->message1 && slot->page_encryption) { + if (slot->page_buf2==NULL) { + os_slot_alloc_page_buf2(slot); + } + + ut_ad(slot->page_buf2); + + if (slot->type == OS_FILE_READ) { + if (fil_page_is_encrypted(slot->buf)) { + fil_decrypt_page(slot->page_buf2, slot->buf, slot->len, slot->write_size); + } + } else { + if (slot->page_encryption_success && + fil_page_is_encrypted(slot->page_buf2)) { + ut_ad(slot->page_encryption_page); + if (srv_use_trim && os_fallocate_failed == FALSE) { + // Deallocate unused blocks from file system + os_file_trim(slot->file, slot, slot->len); + } + } + } + } + + /* Mark this request as completed. The error handling + will be done in the calling function. */ + os_mutex_enter(array->mutex); + slot->n_bytes = events[i].res; + slot->ret = events[i].res2; + slot->io_already_done = TRUE; + os_mutex_exit(array->mutex); + } + return; + } + + if (UNIV_UNLIKELY(srv_shutdown_state == SRV_SHUTDOWN_EXIT_THREADS)) { + return; + } + + /* This error handling is for any error in collecting the + IO requests. The errors, if any, for any particular IO + request are simply passed on to the calling routine. */ + + switch (ret) { + case -EAGAIN: + /* Not enough resources! Try again. */ + case -EINTR: + /* Interrupted! I have tested the behaviour in case of an + interrupt. If we have some completed IOs available then + the return code will be the number of IOs. We get EINTR only + if there are no completed IOs and we have been interrupted. */ + case 0: + /* No pending request! Go back and check again. */ + goto retry; + } + + /* All other errors should cause a trap for now. */ + ut_print_timestamp(stderr); + fprintf(stderr, + " InnoDB: unexpected ret_code[%d] from io_getevents()!\n", + ret); + ut_error; +} + +/**********************************************************************//** +This function is only used in Linux native asynchronous i/o. +Waits for an aio operation to complete. This function is used to wait for +the completed requests. The aio array of pending requests is divided +into segments. The thread specifies which segment or slot it wants to wait +for. NOTE: this function will also take care of freeing the aio slot, +therefore no other thread is allowed to do the freeing! +@return TRUE if the IO was successful */ +UNIV_INTERN +ibool +os_aio_linux_handle( +/*================*/ + ulint global_seg, /*!< in: segment number in the aio array + to wait for; segment 0 is the ibuf + i/o thread, segment 1 is log i/o thread, + then follow the non-ibuf read threads, + and the last are the non-ibuf write + threads. */ + fil_node_t**message1, /*!< out: the messages passed with the */ + void** message2, /*!< aio request; note that in case the + aio operation failed, these output + parameters are valid and can be used to + restart the operation. */ + ulint* type, /*!< out: OS_FILE_WRITE or ..._READ */ + ulint* space_id) +{ + ulint segment; + os_aio_array_t* array; + os_aio_slot_t* slot; + ulint n; + ulint i; + ibool ret = FALSE; + + /* Should never be doing Sync IO here. */ + ut_a(global_seg != ULINT_UNDEFINED); + + /* Find the array and the local segment. */ + segment = os_aio_get_array_and_local_segment(&array, global_seg); + n = array->n_slots / array->n_segments; + + wait_for_event: + /* Loop until we have found a completed request. */ + for (;;) { + ibool any_reserved = FALSE; + os_mutex_enter(array->mutex); + for (i = 0; i < n; ++i) { + slot = os_aio_array_get_nth_slot( + array, i + segment * n); + if (!slot->reserved) { + continue; + } else if (slot->io_already_done) { + /* Something for us to work on. */ + goto found; + } else { + any_reserved = TRUE; + } + } + + os_mutex_exit(array->mutex); + + /* There is no completed request. + If there is no pending request at all, + and the system is being shut down, exit. */ + if (UNIV_UNLIKELY + (!any_reserved + && srv_shutdown_state == SRV_SHUTDOWN_EXIT_THREADS)) { + *message1 = NULL; + *message2 = NULL; + return(TRUE); + } + + /* Wait for some request. Note that we return + from wait iff we have found a request. */ + + srv_set_io_thread_op_info(global_seg, + "waiting for completed aio requests"); + os_aio_linux_collect(array, segment, n); + } + +found: + /* Note that it may be that there are more then one completed + IO requests. We process them one at a time. We may have a case + here to improve the performance slightly by dealing with all + requests in one sweep. */ + srv_set_io_thread_op_info(global_seg, + "processing completed aio requests"); + + /* Ensure that we are scribbling only our segment. */ + ut_a(i < n); + + ut_ad(slot != NULL); + ut_ad(slot->reserved); + ut_ad(slot->io_already_done); + + *message1 = slot->message1; + *message2 = slot->message2; + + *type = slot->type; + *space_id = slot->space_id; + + if (slot->ret == 0 && slot->n_bytes == (long) slot->len) { + + ret = TRUE; + } else if ((slot->ret == 0) && (slot->n_bytes > 0) + && (slot->n_bytes < (long) slot->len)) { + /* Partial read or write scenario */ + int submit_ret; + struct iocb* iocb; + slot->buf = (byte*)slot->buf + slot->n_bytes; + slot->offset = slot->offset + slot->n_bytes; + slot->len = slot->len - slot->n_bytes; + /* Resetting the bytes read/written */ + slot->n_bytes = 0; + slot->io_already_done = FALSE; + iocb = &(slot->control); + + if (slot->type == OS_FILE_READ) { + io_prep_pread(&slot->control, slot->file, slot->buf, + slot->len, (off_t) slot->offset); + } else { + ut_a(slot->type == OS_FILE_WRITE); + io_prep_pwrite(&slot->control, slot->file, slot->buf, + slot->len, (off_t) slot->offset); + } + /* Resubmit an I/O request */ + submit_ret = io_submit(array->aio_ctx[segment], 1, &iocb); + if (submit_ret < 0 ) { + /* Aborting in case of submit failure */ + ib_logf(IB_LOG_LEVEL_FATAL, + "Native Linux AIO interface. io_submit()" + " call failed when resubmitting a partial" + " I/O request on the file %s.", + slot->name); + } else { + ret = FALSE; + os_mutex_exit(array->mutex); + goto wait_for_event; + } + } else { + errno = -slot->ret; + + if (slot->ret == 0) { + fprintf(stderr, + "InnoDB: Number of bytes after aio %d requested %lu\n" + "InnoDB: from file %s\n", + slot->n_bytes, slot->len, slot->name); + } + + /* os_file_handle_error does tell us if we should retry + this IO. As it stands now, we don't do this retry when + reaping requests from a different context than + the dispatcher. This non-retry logic is the same for + windows and linux native AIO. + We should probably look into this to transparently + re-submit the IO. */ + os_file_handle_error(slot->name, "Linux aio", __FILE__, __LINE__); + + ret = FALSE; + } + + os_mutex_exit(array->mutex); + + os_aio_array_free_slot(array, slot); + + return(ret); +} +#endif /* LINUX_NATIVE_AIO */ + +/**********************************************************************//** +Does simulated aio. This function should be called by an i/o-handler +thread. +@return TRUE if the aio operation succeeded */ +UNIV_INTERN +ibool +os_aio_simulated_handle( +/*====================*/ + ulint global_segment, /*!< in: the number of the segment in the aio + arrays to wait for; segment 0 is the ibuf + i/o thread, segment 1 the log i/o thread, + then follow the non-ibuf read threads, and as + the last are the non-ibuf write threads */ + fil_node_t**message1, /*!< out: the messages passed with the aio + request; note that also in the case where + the aio operation failed, these output + parameters are valid and can be used to + restart the operation, for example */ + void** message2, + ulint* type, /*!< out: OS_FILE_WRITE or ..._READ */ + ulint* space_id) +{ + os_aio_array_t* array; + ulint segment; + os_aio_slot_t* consecutive_ios[OS_AIO_MERGE_N_CONSECUTIVE]; + ulint n_consecutive; + ulint total_len; + ulint offs; + os_offset_t lowest_offset; + ulint biggest_age; + ulint age; + byte* combined_buf; + byte* combined_buf2; + ibool ret; + ibool any_reserved; + ulint n; + os_aio_slot_t* aio_slot; + + /* Fix compiler warning */ + *consecutive_ios = NULL; + + segment = os_aio_get_array_and_local_segment(&array, global_segment); + +restart: + /* NOTE! We only access constant fields in os_aio_array. Therefore + we do not have to acquire the protecting mutex yet */ + + srv_set_io_thread_op_info(global_segment, + "looking for i/o requests (a)"); + ut_ad(os_aio_validate_skip()); + ut_ad(segment < array->n_segments); + + n = array->n_slots / array->n_segments; + + /* Look through n slots after the segment * n'th slot */ + + if (array == os_aio_read_array + && os_aio_recommend_sleep_for_read_threads) { + + /* Give other threads chance to add several i/os to the array + at once. */ + + goto recommended_sleep; + } + + srv_set_io_thread_op_info(global_segment, + "looking for i/o requests (b)"); + + /* Check if there is a slot for which the i/o has already been + done */ + any_reserved = FALSE; + + os_mutex_enter(array->mutex); + + for (ulint i = 0; i < n; i++) { + os_aio_slot_t* slot; + + slot = os_aio_array_get_nth_slot(array, i + segment * n); + + if (!slot->reserved) { + continue; + } else if (slot->io_already_done) { + + if (os_aio_print_debug) { + fprintf(stderr, + "InnoDB: i/o for slot %lu" + " already done, returning\n", + (ulong) i); + } + + aio_slot = slot; + ret = TRUE; + goto slot_io_done; + } else { + any_reserved = TRUE; + } + } + + /* There is no completed request. + If there is no pending request at all, + and the system is being shut down, exit. */ + if (!any_reserved && srv_shutdown_state == SRV_SHUTDOWN_EXIT_THREADS) { + os_mutex_exit(array->mutex); + *message1 = NULL; + *message2 = NULL; + return(TRUE); + } + + n_consecutive = 0; + + /* If there are at least 2 seconds old requests, then pick the oldest + one to prevent starvation. If several requests have the same age, + then pick the one at the lowest offset. */ + + biggest_age = 0; + lowest_offset = IB_UINT64_MAX; + + for (ulint i = 0; i < n; i++) { + os_aio_slot_t* slot; + + slot = os_aio_array_get_nth_slot(array, i + segment * n); + + if (slot->reserved) { + + age = (ulint) difftime( + ut_time(), slot->reservation_time); + + if ((age >= 2 && age > biggest_age) + || (age >= 2 && age == biggest_age + && slot->offset < lowest_offset)) { + + /* Found an i/o request */ + consecutive_ios[0] = slot; + + n_consecutive = 1; + + biggest_age = age; + lowest_offset = slot->offset; + } + } + } + + if (n_consecutive == 0) { + /* There were no old requests. Look for an i/o request at the + lowest offset in the array (we ignore the high 32 bits of the + offset in these heuristics) */ + + lowest_offset = IB_UINT64_MAX; + + for (ulint i = 0; i < n; i++) { + os_aio_slot_t* slot; + + slot = os_aio_array_get_nth_slot( + array, i + segment * n); + + if (slot->reserved && slot->offset < lowest_offset) { + + /* Found an i/o request */ + consecutive_ios[0] = slot; + + n_consecutive = 1; + + lowest_offset = slot->offset; + } + } + } + + if (n_consecutive == 0) { + + /* No i/o requested at the moment */ + + goto wait_for_io; + } + + /* if n_consecutive != 0, then we have assigned + something valid to consecutive_ios[0] */ + ut_ad(n_consecutive != 0); + ut_ad(consecutive_ios[0] != NULL); + + aio_slot = consecutive_ios[0]; + + /* Check if there are several consecutive blocks to read or write */ + +consecutive_loop: + for (ulint i = 0; i < n; i++) { + os_aio_slot_t* slot; + + slot = os_aio_array_get_nth_slot(array, i + segment * n); + + if (slot->reserved + && slot != aio_slot + && slot->offset == aio_slot->offset + aio_slot->len + && slot->type == aio_slot->type + && slot->file == aio_slot->file) { + + /* Found a consecutive i/o request */ + + consecutive_ios[n_consecutive] = slot; + n_consecutive++; + + aio_slot = slot; + + if (n_consecutive < OS_AIO_MERGE_N_CONSECUTIVE) { + + goto consecutive_loop; + } else { + break; + } + } + } + + srv_set_io_thread_op_info(global_segment, "consecutive i/o requests"); + + /* We have now collected n_consecutive i/o requests in the array; + allocate a single buffer which can hold all data, and perform the + i/o */ + + total_len = 0; + aio_slot = consecutive_ios[0]; + + for (ulint i = 0; i < n_consecutive; i++) { + total_len += consecutive_ios[i]->len; + } + + if (n_consecutive == 1) { + /* We can use the buffer of the i/o request */ + combined_buf = aio_slot->buf; + combined_buf2 = NULL; + } else { + combined_buf2 = static_cast( + ut_malloc(total_len + UNIV_PAGE_SIZE)); + + ut_a(combined_buf2); + + combined_buf = static_cast( + ut_align(combined_buf2, UNIV_PAGE_SIZE)); + } + + /* We release the array mutex for the time of the i/o: NOTE that + this assumes that there is just one i/o-handler thread serving + a single segment of slots! */ + + os_mutex_exit(array->mutex); + + if (aio_slot->type == OS_FILE_WRITE && n_consecutive > 1) { + /* Copy the buffers to the combined buffer */ + offs = 0; + + for (ulint i = 0; i < n_consecutive; i++) { + + ut_memcpy(combined_buf + offs, consecutive_ios[i]->buf, + consecutive_ios[i]->len); + + offs += consecutive_ios[i]->len; + } + } + + srv_set_io_thread_op_info(global_segment, "doing file i/o"); + + /* Do the i/o with ordinary, synchronous i/o functions: */ + if (aio_slot->type == OS_FILE_WRITE) { + ut_ad(!srv_read_only_mode); + ret = os_file_write( + aio_slot->name, aio_slot->file, combined_buf, + aio_slot->offset, total_len); + } else { + ret = os_file_read( + aio_slot->file, combined_buf, + aio_slot->offset, total_len, + aio_slot->page_compression); + } + + ut_a(ret); + srv_set_io_thread_op_info(global_segment, "file i/o done"); + + if (aio_slot->type == OS_FILE_READ && n_consecutive > 1) { + /* Copy the combined buffer to individual buffers */ + offs = 0; + + for (ulint i = 0; i < n_consecutive; i++) { + + ut_memcpy(consecutive_ios[i]->buf, combined_buf + offs, + consecutive_ios[i]->len); + offs += consecutive_ios[i]->len; + } + } + + if (combined_buf2) { + ut_free(combined_buf2); + } + + os_mutex_enter(array->mutex); + + /* Mark the i/os done in slots */ + + for (ulint i = 0; i < n_consecutive; i++) { + consecutive_ios[i]->io_already_done = TRUE; + } + + /* We return the messages for the first slot now, and if there were + several slots, the messages will be returned with subsequent calls + of this function */ + +slot_io_done: + + ut_a(aio_slot->reserved); + + *message1 = aio_slot->message1; + *message2 = aio_slot->message2; + + *type = aio_slot->type; + *space_id = aio_slot->space_id; + + os_mutex_exit(array->mutex); + + os_aio_array_free_slot(array, aio_slot); + + return(ret); + +wait_for_io: + srv_set_io_thread_op_info(global_segment, "resetting wait event"); + + /* We wait here until there again can be i/os in the segment + of this thread */ + + os_event_reset(os_aio_segment_wait_events[global_segment]); + + os_mutex_exit(array->mutex); + +recommended_sleep: + srv_set_io_thread_op_info(global_segment, "waiting for i/o request"); + + os_event_wait(os_aio_segment_wait_events[global_segment]); + + goto restart; +} + +/**********************************************************************//** +Validates the consistency of an aio array. +@return true if ok */ +static +bool +os_aio_array_validate( +/*==================*/ + os_aio_array_t* array) /*!< in: aio wait array */ +{ + ulint i; + ulint n_reserved = 0; + + os_mutex_enter(array->mutex); + + ut_a(array->n_slots > 0); + ut_a(array->n_segments > 0); + + for (i = 0; i < array->n_slots; i++) { + os_aio_slot_t* slot; + + slot = os_aio_array_get_nth_slot(array, i); + + if (slot->reserved) { + n_reserved++; + ut_a(slot->len > 0); + } + } + + ut_a(array->n_reserved == n_reserved); + + os_mutex_exit(array->mutex); + + return(true); +} + +/**********************************************************************//** +Validates the consistency the aio system. +@return TRUE if ok */ +UNIV_INTERN +ibool +os_aio_validate(void) +/*=================*/ +{ + os_aio_array_validate(os_aio_read_array); + + if (os_aio_write_array != 0) { + os_aio_array_validate(os_aio_write_array); + } + + if (os_aio_ibuf_array != 0) { + os_aio_array_validate(os_aio_ibuf_array); + } + + if (os_aio_log_array != 0) { + os_aio_array_validate(os_aio_log_array); + } + + if (os_aio_sync_array != 0) { + os_aio_array_validate(os_aio_sync_array); + } + + return(TRUE); +} + +/**********************************************************************//** +Prints pending IO requests per segment of an aio array. +We probably don't need per segment statistics but they can help us +during development phase to see if the IO requests are being +distributed as expected. */ +static +void +os_aio_print_segment_info( +/*======================*/ + FILE* file, /*!< in: file where to print */ + ulint* n_seg, /*!< in: pending IO array */ + os_aio_array_t* array) /*!< in: array to process */ +{ + ulint i; + + ut_ad(array); + ut_ad(n_seg); + ut_ad(array->n_segments > 0); + + if (array->n_segments == 1) { + return; + } + + fprintf(file, " ["); + for (i = 0; i < array->n_segments; i++) { + if (i != 0) { + fprintf(file, ", "); + } + + fprintf(file, "%lu", n_seg[i]); + } + fprintf(file, "] "); +} + +/**********************************************************************//** +Prints info about the aio array. */ +UNIV_INTERN +void +os_aio_print_array( +/*==============*/ + FILE* file, /*!< in: file where to print */ + os_aio_array_t* array) /*!< in: aio array to print */ +{ + ulint n_reserved = 0; + ulint n_res_seg[SRV_MAX_N_IO_THREADS]; + + os_mutex_enter(array->mutex); + + ut_a(array->n_slots > 0); + ut_a(array->n_segments > 0); + + memset(n_res_seg, 0x0, sizeof(n_res_seg)); + + for (ulint i = 0; i < array->n_slots; ++i) { + os_aio_slot_t* slot; + ulint seg_no; + + slot = os_aio_array_get_nth_slot(array, i); + + seg_no = (i * array->n_segments) / array->n_slots; + + if (slot->reserved) { + ++n_reserved; + ++n_res_seg[seg_no]; + + ut_a(slot->len > 0); + } + } + + ut_a(array->n_reserved == n_reserved); + + fprintf(file, " %lu", (ulong) n_reserved); + + os_aio_print_segment_info(file, n_res_seg, array); + + os_mutex_exit(array->mutex); +} + +/**********************************************************************//** +Prints info of the aio arrays. */ +UNIV_INTERN +void +os_aio_print( +/*=========*/ + FILE* file) /*!< in: file where to print */ +{ + time_t current_time; + double time_elapsed; + double avg_bytes_read; + + for (ulint i = 0; i < srv_n_file_io_threads; ++i) { + fprintf(file, "I/O thread %lu state: %s (%s)", + (ulong) i, + srv_io_thread_op_info[i], + srv_io_thread_function[i]); + +#ifndef __WIN__ + if (os_aio_segment_wait_events[i]->is_set) { + fprintf(file, " ev set"); + } +#endif /* __WIN__ */ + + fprintf(file, "\n"); + } + + fputs("Pending normal aio reads:", file); + + os_aio_print_array(file, os_aio_read_array); + + if (os_aio_write_array != 0) { + fputs(", aio writes:", file); + os_aio_print_array(file, os_aio_write_array); + } + + if (os_aio_ibuf_array != 0) { + fputs(",\n ibuf aio reads:", file); + os_aio_print_array(file, os_aio_ibuf_array); + } + + if (os_aio_log_array != 0) { + fputs(", log i/o's:", file); + os_aio_print_array(file, os_aio_log_array); + } + + if (os_aio_sync_array != 0) { + fputs(", sync i/o's:", file); + os_aio_print_array(file, os_aio_sync_array); + } + + putc('\n', file); + current_time = ut_time(); + time_elapsed = 0.001 + difftime(current_time, os_last_printout); + + fprintf(file, + "Pending flushes (fsync) log: %lu; buffer pool: %lu\n" + "%lu OS file reads, %lu OS file writes, %lu OS fsyncs\n", + (ulong) fil_n_pending_log_flushes, + (ulong) fil_n_pending_tablespace_flushes, + (ulong) os_n_file_reads, + (ulong) os_n_file_writes, + (ulong) os_n_fsyncs); + + if (os_file_n_pending_preads != 0 || os_file_n_pending_pwrites != 0) { + fprintf(file, + "%lu pending preads, %lu pending pwrites\n", + (ulong) os_file_n_pending_preads, + (ulong) os_file_n_pending_pwrites); + } + + if (os_n_file_reads == os_n_file_reads_old) { + avg_bytes_read = 0.0; + } else { + avg_bytes_read = (double) os_bytes_read_since_printout + / (os_n_file_reads - os_n_file_reads_old); + } + + fprintf(file, + "%.2f reads/s, %lu avg bytes/read," + " %.2f writes/s, %.2f fsyncs/s\n", + (os_n_file_reads - os_n_file_reads_old) + / time_elapsed, + (ulong) avg_bytes_read, + (os_n_file_writes - os_n_file_writes_old) + / time_elapsed, + (os_n_fsyncs - os_n_fsyncs_old) + / time_elapsed); + + os_n_file_reads_old = os_n_file_reads; + os_n_file_writes_old = os_n_file_writes; + os_n_fsyncs_old = os_n_fsyncs; + os_bytes_read_since_printout = 0; + + os_last_printout = current_time; +} + +/**********************************************************************//** +Refreshes the statistics used to print per-second averages. */ +UNIV_INTERN +void +os_aio_refresh_stats(void) +/*======================*/ +{ + os_n_file_reads_old = os_n_file_reads; + os_n_file_writes_old = os_n_file_writes; + os_n_fsyncs_old = os_n_fsyncs; + os_bytes_read_since_printout = 0; + + os_last_printout = time(NULL); +} + +#ifdef UNIV_DEBUG +/**********************************************************************//** +Checks that all slots in the system have been freed, that is, there are +no pending io operations. +@return TRUE if all free */ +UNIV_INTERN +ibool +os_aio_all_slots_free(void) +/*=======================*/ +{ + os_aio_array_t* array; + ulint n_res = 0; + + array = os_aio_read_array; + + os_mutex_enter(array->mutex); + + n_res += array->n_reserved; + + os_mutex_exit(array->mutex); + + if (!srv_read_only_mode) { + ut_a(os_aio_write_array == 0); + + array = os_aio_write_array; + + os_mutex_enter(array->mutex); + + n_res += array->n_reserved; + + os_mutex_exit(array->mutex); + + ut_a(os_aio_ibuf_array == 0); + + array = os_aio_ibuf_array; + + os_mutex_enter(array->mutex); + + n_res += array->n_reserved; + + os_mutex_exit(array->mutex); + } + + ut_a(os_aio_log_array == 0); + + array = os_aio_log_array; + + os_mutex_enter(array->mutex); + + n_res += array->n_reserved; + + os_mutex_exit(array->mutex); + + array = os_aio_sync_array; + + os_mutex_enter(array->mutex); + + n_res += array->n_reserved; + + os_mutex_exit(array->mutex); + + if (n_res == 0) { + + return(TRUE); + } + + return(FALSE); +} +#endif /* UNIV_DEBUG */ + +#endif /* !UNIV_HOTBACKUP */ + +#ifdef _WIN32 +#include +#ifndef FSCTL_FILE_LEVEL_TRIM +#define FSCTL_FILE_LEVEL_TRIM CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 130, METHOD_BUFFERED, FILE_WRITE_DATA) +typedef struct _FILE_LEVEL_TRIM_RANGE { + DWORDLONG Offset; + DWORDLONG Length; +} FILE_LEVEL_TRIM_RANGE, *PFILE_LEVEL_TRIM_RANGE; + +typedef struct _FILE_LEVEL_TRIM { + DWORD Key; + DWORD NumRanges; + FILE_LEVEL_TRIM_RANGE Ranges[1]; +} FILE_LEVEL_TRIM, *PFILE_LEVEL_TRIM; +#endif +#endif + +/**********************************************************************//** +Directly manipulate the allocated disk space by deallocating for the file referred to +by fd for the byte range starting at offset and continuing for len bytes. +Within the specified range, partial file system blocks are zeroed, and whole +file system blocks are removed from the file. After a successful call, +subsequent reads from this range will return zeroes. +@return true if success, false if error */ +UNIV_INTERN +ibool +os_file_trim( +/*=========*/ + os_file_t file, /*!< in: file to be trimmed */ + os_aio_slot_t* slot, /*!< in: slot structure */ + ulint len) /*!< in: length of area */ +{ + +#define SECT_SIZE 512 + size_t trim_len = UNIV_PAGE_SIZE - len; + os_offset_t off = slot->offset + len; + // len here should be alligned to sector size + ut_a((trim_len % SECT_SIZE) == 0); + ut_a((len % SECT_SIZE) == 0); + + // Nothing to do if trim length is zero or if actual write + // size is initialized and it is smaller than current write size. + // In first write if we trim we set write_size to actual bytes + // written and rest of the page is trimmed. In following writes + // there is no need to trim again if write_size only increases + // because rest of the page is already trimmed. If actual write + // size decreases we need to trim again. + if (trim_len == 0 || + (slot->write_size && + *slot->write_size > 0 && + len >= *slot->write_size)) { + +#ifdef UNIV_PAGECOMPRESS_DEBUG + fprintf(stderr, "Note: TRIM: write_size %lu trim_len %lu len %lu\n", + *slot->write_size, trim_len, len); +#endif + + if (*slot->write_size > 0 && len >= *slot->write_size) { + srv_stats.page_compressed_trim_op_saved.inc(); + } + + *slot->write_size = len; + + return (TRUE); + } + +#ifdef __linux__ +#if defined(FALLOC_FL_PUNCH_HOLE) && defined (FALLOC_FL_KEEP_SIZE) + int ret = fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, off, trim_len); + + if (ret) { + /* After first failure do not try to trim again */ + os_fallocate_failed = TRUE; + ut_print_timestamp(stderr); + fprintf(stderr, + " InnoDB: [Warning] fallocate call failed with error code %d.\n" + " InnoDB: start: %lx len: %lu payload: %lu\n" + " InnoDB: Disabling fallocate for now.\n", ret, (slot->offset+len), trim_len, len); + + os_file_handle_error_no_exit(slot->name, + " fallocate(FALLOC_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE) ", + FALSE, __FILE__, __LINE__); + + if (slot->write_size) { + *slot->write_size = 0; + } + + return (FALSE); + } else { + if (slot->write_size) { + *slot->write_size = len; + } + } +#else + ut_print_timestamp(stderr); + fprintf(stderr, + " InnoDB: [Warning] fallocate not supported on this installation." + " InnoDB: Disabling fallocate for now."); + os_fallocate_failed = TRUE; + if (slot->write_size) { + *slot->write_size = 0; + } + +#endif /* HAVE_FALLOCATE ... */ + +#elif defined(_WIN32) + FILE_LEVEL_TRIM flt; + flt.Key = 0; + flt.NumRanges = 1; + flt.Ranges[0].Offset = off; + flt.Ranges[0].Length = trim_len; + + BOOL ret = DeviceIoControl(file,FSCTL_FILE_LEVEL_TRIM,&flt, sizeof(flt), NULL, NULL, NULL, NULL); + + if (!ret) { + /* After first failure do not try to trim again */ + os_fallocate_failed = TRUE; + ut_print_timestamp(stderr); + fprintf(stderr, + " InnoDB: [Warning] fallocate call failed with error.\n" + " InnoDB: start: %lx len: %du payload: %lu\n" + " InnoDB: Disabling fallocate for now.\n", (slot->offset+len), trim_len, len); + + os_file_handle_error_no_exit(slot->name, + " DeviceIOControl(FSCTL_FILE_LEVEL_TRIM) ", + FALSE, __FILE__, __LINE__); + + if (slot->write_size) { + *slot->write_size = 0; + } + return (FALSE); + } else { + if (slot->write_size) { + *slot->write_size = len; + } + } +#endif + + srv_stats.page_compression_trim_sect512.add((trim_len / SECT_SIZE)); + srv_stats.page_compression_trim_sect4096.add((trim_len / (SECT_SIZE*8))); + srv_stats.page_compressed_trim_op.inc(); + + return (TRUE); + +} + +/**********************************************************************//** +Allocate memory for temporal buffer used for page encryption. This +buffer is freed later. */ +UNIV_INTERN +void +os_slot_alloc_page_buf2( +/*===================*/ + os_aio_slot_t* slot) /*!< in: slot structure */ +{ + byte* cbuf2; + byte* cbuf; + + cbuf2 = static_cast(ut_malloc(UNIV_PAGE_SIZE*2)); + cbuf = static_cast(ut_align(cbuf2, UNIV_PAGE_SIZE)); + slot->page_encryption_page = static_cast(cbuf2); + slot->page_buf2 = static_cast(cbuf); +} + +/**********************************************************************//** +Allocate memory for temporal buffer used for page compression. This +buffer is freed later. */ +UNIV_INTERN +void +os_slot_alloc_page_buf( +/*===================*/ + os_aio_slot_t* slot) /*!< in: slot structure */ +{ + byte* cbuf2; + byte* cbuf; + + ut_a(slot != NULL); + /* We allocate extra to avoid memory overwrite on compression */ + cbuf2 = static_cast(ut_malloc(UNIV_PAGE_SIZE*2)); + cbuf = static_cast(ut_align(cbuf2, UNIV_PAGE_SIZE)); + slot->page_compression_page = static_cast(cbuf2); + slot->page_buf = static_cast(cbuf); + ut_a(slot->page_buf != NULL); +} + +#ifdef HAVE_LZO +/**********************************************************************//** +Allocate memory for temporal memory used for page compression when +LZO compression method is used */ +UNIV_INTERN +void +os_slot_alloc_lzo_mem( +/*===================*/ + os_aio_slot_t* slot) /*!< in: slot structure */ +{ + ut_a(slot != NULL); + slot->lzo_mem = static_cast(ut_malloc(LZO1X_1_15_MEM_COMPRESS)); + ut_a(slot->lzo_mem != NULL); +} +#endif + diff --git a/storage/xtradb/os/os0file.cc.rej b/storage/xtradb/os/os0file.cc.rej new file mode 100644 index 0000000000000..6455224e46bc5 --- /dev/null +++ b/storage/xtradb/os/os0file.cc.rej @@ -0,0 +1,20 @@ +--- storage/xtradb/os/os0file.cc ++++ storage/xtradb/os/os0file.cc +@@ -3175,7 +3175,7 @@ + + if (fil_page_is_encrypted((byte *)buf)) { + // if (page_encryption) { +- fil_decrypt_page(NULL, (byte *)buf, n, NULL); ++ fil_decrypt_page(NULL, (byte *)buf, n, NULL, 0); + } + + +@@ -4692,7 +4692,7 @@ + + ut_ad(slot->page_buf2); + //FF +- tmp = fil_encrypt_page(fil_node_get_space_id(slot->message1), (byte *)buf, slot->page_buf2, len, page_encryption_key, &real_len); ++ tmp = fil_encrypt_page(fil_node_get_space_id(slot->message1), (byte *)buf, slot->page_buf2, len, page_encryption_key, &real_len, 0); + + /* If encryption succeeded, set up the length and buffer */ + if (tmp != buf) { diff --git a/storage/xtradb/srv/srv0mon.cc b/storage/xtradb/srv/srv0mon.cc index f276efdc021d6..0e0c3ebe4679a 100644 --- a/storage/xtradb/srv/srv0mon.cc +++ b/storage/xtradb/srv/srv0mon.cc @@ -1879,6 +1879,15 @@ srv_mon_process_existing_counter( case MONITOR_OVLD_PAGES_PAGE_COMPRESSION_ERROR: value = srv_stats.pages_page_compression_error; break; + case MONITOR_OVLD_PAGES_PAGE_ENCRYPTED: + value = srv_stats.pages_page_encrypted; + break; + case MONITOR_OVLD_PAGES_PAGE_DECRYPTED: + value = srv_stats.pages_page_decrypted; + break; + case MONITOR_OVLD_PAGES_PAGE_ENCRYPTION_ERROR: + value = srv_stats.pages_page_encryption_error; + break; default: ut_error; diff --git a/storage/xtradb/srv/srv0srv.cc b/storage/xtradb/srv/srv0srv.cc index 896bb8a3b77b9..59714b7252804 100644 --- a/storage/xtradb/srv/srv0srv.cc +++ b/storage/xtradb/srv/srv0srv.cc @@ -1906,6 +1906,10 @@ srv_export_innodb_status(void) export_vars.innodb_page_compressed_trim_op = srv_stats.page_compressed_trim_op; export_vars.innodb_page_compressed_trim_op_saved = srv_stats.page_compressed_trim_op_saved; export_vars.innodb_pages_page_decompressed = srv_stats.pages_page_decompressed; + export_vars.innodb_pages_page_compression_error = srv_stats.pages_page_compression_error; + export_vars.innodb_pages_page_decrypted = srv_stats.pages_page_decrypted; + export_vars.innodb_pages_page_encrypted = srv_stats.pages_page_encrypted; + export_vars.innodb_pages_page_encryption_error = srv_stats.pages_page_encryption_error; export_vars.innodb_defragment_compression_failures = btr_defragment_compression_failures; diff --git a/unittest/eperi/CMakeLists.txt b/unittest/eperi/CMakeLists.txt new file mode 100644 index 0000000000000..9f8745bbf7227 --- /dev/null +++ b/unittest/eperi/CMakeLists.txt @@ -0,0 +1,42 @@ +# Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved. +# +# This program is free software; you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation; version 2 of the License. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program; if not, write to the Free Software +# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql + ${PCRE_INCLUDES} + ${CMAKE_SOURCE_DIR}/include +# ${CMAKE_SOURCE_DIR}/include/mysql + ${CMAKE_SOURCE_DIR}/unittest/mytap + ${CMAKE_SOURCE_DIR}/extra/yassl/include + ${CMAKE_SOURCE_DIR}/storage/perfschema/unittest + ${CMAKE_SOURCE_DIR}/storage/perfschema + ${CMAKE_SOURCE_DIR}/storage/xtradb/include + ) + +MY_ADD_TESTS(eperi) +MY_ADD_TESTS(eperi_keyfile + LINK_LIBRARIES mysys pcre) +MY_ADD_TESTS(eperi_aes + LINK_LIBRARIES mysys_ssl dbug) + +MY_ADD_TESTS(enc0enc + + johnny + EXT "cc" + LINK_LIBRARIES xtradb perfschema mysys mysys_ssl sql mysql) + + +file(COPY + ${CMAKE_CURRENT_LIST_DIR}/keys.txt + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) diff --git a/unittest/eperi/enc0enc-t.cc b/unittest/eperi/enc0enc-t.cc new file mode 100644 index 0000000000000..40033f4f2a70c --- /dev/null +++ b/unittest/eperi/enc0enc-t.cc @@ -0,0 +1,47 @@ +/* + * enc0enc.c + * + * Created on: 22.08.2014 + * Author: florin + */ + +#include "enc0enc-t.h" +#include +#include +#include + +typedef unsigned long int ulint; + +extern int summef(int a, int b); +extern int summef2(int a, int b); +extern void fil_flush(ulint space_id); + + +int main() +{ + plan(9); + //MY_INIT("enc0enc-t"); + printf("%s\n", "main() enc0enc.c"); + int sum = summef(2, 3); + printf("%s, %d\n", "summef(int a, int b)", sum); + //fil_flush(3); + printf("%s, %d\n", "summef2(int a, int b)", summef2(2, 7)); +/* + string encrypted = encryptDecrypt("kylewbanks.com"); + cout << "Encrypted:" << encrypted << "\n"; + + string decrypted = encryptDecrypt(encrypted); + cout << "Decrypted:" << decrypted << "\n"; +*/ + return 0; +} + + +/* +int summe(int a, int b) +{ + return a + b; +} + +int summe(int a, int b); +*/ diff --git a/unittest/eperi/enc0enc-t.h b/unittest/eperi/enc0enc-t.h new file mode 100644 index 0000000000000..f17fa8e4a3c31 --- /dev/null +++ b/unittest/eperi/enc0enc-t.h @@ -0,0 +1,13 @@ +/* + * enc0enc.h + * + * Created on: 22.08.2014 + * Author: florin + */ + +#ifndef ENC0ENC_H_ +#define ENC0ENC_H_ + + + +#endif /* ENC0ENC_H_ */ diff --git a/unittest/eperi/eperi-t.c b/unittest/eperi/eperi-t.c new file mode 100644 index 0000000000000..255dad36475ab --- /dev/null +++ b/unittest/eperi/eperi-t.c @@ -0,0 +1,10 @@ +#include + + +int +main(int argc __attribute__((unused)),char *argv[]) +{ + plan(1); + ok(1, "Most simple test ever"); + return 0; +} diff --git a/unittest/eperi/eperi_aes-t.c b/unittest/eperi/eperi_aes-t.c new file mode 100644 index 0000000000000..8f12d150cee55 --- /dev/null +++ b/unittest/eperi/eperi_aes-t.c @@ -0,0 +1,361 @@ +#define EP_UNIT_TEST 1 +#define UNIV_INLINE +typedef unsigned char byte; +typedef unsigned long int ulint; +typedef unsigned long int ibool; + +#include +#include +#include +#include +#include +#include "../../storage/xtradb/include/fil0pageencryption.h" + + +extern byte* +fil_encrypt_page( +/*==============*/ + ulint space_id, /*!< in: tablespace id of the + table. */ + byte* buf, /*!< in: buffer from which to write; in aio + this must be appropriately aligned */ + byte* out_buf, /*!< out: compressed buffer */ + ulint len, /*!< in: length of input buffer.*/ + ulint compression_level, /*!< in: compression level */ + ulint* out_len, /*!< out: actual length of compressed page */ + ulint unit_test); + +/****************************************************************//** +For page encrypted pages decrypt the page after actual read +operation. +@return decrypted page */ +extern void +fil_decrypt_page( +/*================*/ + byte* page_buf, /*!< in: preallocated buffer or NULL */ + byte* buf, /*!< out: buffer from which to read; in aio + this must be appropriately aligned */ + ulint len, /*!< in: length of output buffer.*/ + ulint* write_size, /*!< in/out: Actual payload size of the decrypted data. */ + ulint unit_test); + + + + + +#define MY_AES_TEST_TEXTBLOCK "abcdefghijklmnopqrstuvwxyz\ + ABCDEFGHIJKLMNOPQRSTUVW\ + 1234567890ß^!\"§$%&/()=?`\ + öäüÖÄÜ+*#',.-;:_~’µ<>|³²¹¼\ + ½¬{[]}æ“¢ð€đŋħłµ”øþ@¶ſŧ↓„ł«»←\ + abcdefghijklmnopqrstuvwxyz\ + ABCDEFGHIJKLMNOPQRSTUVW\ + 1234567890ß^!\"§$%&/()=?`\ + öäüÖÄÜ+*#',.-;:_~’µ<>|³²¹¼\ + ½¬{[]}æ“¢ð€đŋħłµ”øþ@¶ſŧ↓„ł«»←\ + abcdefghijklmnopqrstuvwxyz\ + ABCDEFGHIJKLMNOPQRSTUVW\ + 1234567890ß^!\"§$%&/()=?`\ + öäüÖÄÜ+*#',.-;:_~’µ<>|³²¹¼\ + ½¬{[]}æ“¢ð€đŋħłµ”øþ@¶ſŧ↓„ł«»←\ + abcdefghijklmnopqrstuvwxyz\ + ABCDEFGHIJKLMNOPQRSTUVW\ + 1234567890ß^!\"§$%&/()=?`\ + öäüÖÄÜ+*#',.-;:_~’µ<>|³²¹¼\ + ½¬{[]}æ“¢ð€đŋħłµ”øþ@¶ſŧ↓„ł«»←\ + abcdefghijklmnopqrstuvwxyz\ + ABCDEFGHIJKLMNOPQRSTUVW\ + 1234567890ß^!\"§$%&/()=?`\ + öäüÖÄÜ+*#',.-;:_~’µ<>|³²¹¼\ + ½¬{[]}æ“¢ð€đŋħłµ”øþ@¶ſŧ↓„ł«»←\ + " + +#define MY_AES_TEST_JOSHUA " David Lightman: [typing] What is the primary goal?\ +Joshua: You should know, Professor. You programmed me.\ +David Lightman: Oh, come on.\ +David Lightman: [typing] What is the primary goal?\ +Joshua: To win the game.\ +" + + +byte* readFile(char* fileName) { +FILE *fileptr; +byte *buffer; +long filelen; + +fileptr = fopen(fileName, "rb"); // Open the file in binary mode +fseek(fileptr, 0, SEEK_END); // Jump to the end of the file +filelen = ftell(fileptr); // Get the current byte offset in the file +rewind(fileptr); // Jump back to the beginning of the file + +buffer = (char *)malloc((filelen+1)*sizeof(char)); // Enough memory for file + \0 +fread(buffer, filelen, 1, fileptr); // Read in the entire file +fclose(fileptr); // Close the file +return buffer; +} + +void +test_cbc_wrong_keylength() +{ + plan(2); + char* source = "Joshua: Shall we play a game"; + ulint s_len = (ulint)strlen(source); + char* key="899C0ECB592B2CEE46E64191B6E6DE9B97D8A8EEA43BEF78"; + uint8 k_len = 6; + char* iv = "F0974007D619466B9EBF8D4F6E302AA3"; + uint8 i_len = 16; + char* dest = (char *) malloc(2*s_len*sizeof(char)); + unsigned long int dest_len = 0; + + int rc = my_aes_encrypt_cbc(source, s_len, dest, &dest_len, key, k_len, iv, i_len); + ok(rc == -5, "Encryption - wrong keylength was detected."); + rc = my_aes_decrypt_cbc(source, s_len, dest, &dest_len, key, k_len, iv, i_len); + ok(rc == -5, "Decryption - wrong keylength was detected."); +} + +void +test_cbc_keysizes() +{ + plan(2); + char* source = MY_AES_TEST_JOSHUA; + ulint s_len = (ulint)strlen(source); + char* key="899C0ECB592B2CEE46E64191B6E6DE9B97D8A8EEA43BEF78"; + uint8 k_len = 24; + char* iv = "F0974007D619466B9EBF8D4F6E302AA3"; + uint8 i_len = 16; + char* dest = (char *) malloc(2*s_len*sizeof(char)); + ulint dest_len = 0; + my_aes_encrypt_cbc(source, s_len, dest, &dest_len, key, k_len, iv, i_len); + source = (char *) malloc(strlen(MY_AES_TEST_TEXTBLOCK) * sizeof(char)); + my_aes_decrypt_cbc(dest , strlen(dest), source, &dest_len, key, k_len, iv, i_len); + ok(strcmp(source, MY_AES_TEST_JOSHUA),"Decrypted text is identical to original text."); + + key="7B3B8DA94B77F91A6E05037B21AD5F6E86BD4657C45D97BC7FF14313A781B5A3"; + k_len = 32; + dest = (char *) malloc(2*s_len*sizeof(char)); + my_aes_encrypt_cbc(source, s_len, dest, &dest_len, key, k_len, iv, i_len); + source = (char *) malloc(strlen(MY_AES_TEST_TEXTBLOCK) * sizeof(char)); + my_aes_decrypt_cbc(dest , strlen(dest), source, &dest_len, key, k_len, iv, i_len); + ok(strcmp(source, MY_AES_TEST_JOSHUA),"Decrypted text is identical to original text."); + free(source); + free(dest); +} + +void +test_cbc_large() +{ + plan(1); + char* source = MY_AES_TEST_TEXTBLOCK; + ulint s_len = (ulint)strlen(source); + + char* key = "3C5DC9153A6FE5F22516E217C1603BF7"; + uint8 k_len = 16; + char* iv = "F0974007D619466B9EBF8D4F6E302AA3"; + uint8 i_len = 16; + char* dest = (char *) malloc( 2* s_len * sizeof(char)); + ulint dest_len = 0; + dump_buffer(10,source); + dump_buffer(10,dest); + my_aes_encrypt_cbc(source, s_len, dest, &dest_len, key, k_len, iv, i_len); + source = (char *) malloc(strlen(MY_AES_TEST_TEXTBLOCK) * sizeof(char)); + my_aes_decrypt_cbc(dest , strlen(dest), source, &dest_len, key, k_len, iv, i_len); + ok(strcmp(source, MY_AES_TEST_TEXTBLOCK),"Decrypted text is identical to original text."); + free(source); + free(dest); +} + +void +test_wrong_key() +{ + plan(1); + char* source = MY_AES_TEST_TEXTBLOCK; + ulint s_len = (ulint)strlen(source); + + char* key = "3C5DC9153A6FE5F22516E217C1603BF7"; + uint8 k_len = 16; + char* iv = "F0974007D619466B9EBF8D4F6E302AA3"; + uint8 i_len = 16; + char* dest = (char *) malloc( 2* s_len * sizeof(char)); + ulint dest_len = 0; + dump_buffer(10,source); + dump_buffer(10,dest); + my_aes_encrypt_cbc(source, s_len, dest, &dest_len, key, k_len, iv, i_len); + + iv = "F1A74007D619455B9EBF8D4F6E302AA3"; + source = (char *) malloc(strlen(MY_AES_TEST_TEXTBLOCK) * sizeof(char)); + my_aes_decrypt_cbc(dest , strlen(dest), source, &dest_len, key, k_len, iv, i_len); + ok(strcmp(source, MY_AES_TEST_TEXTBLOCK) != 0,"Using wrong iv results in wrong decryption."); + free(source); + free(dest); +} + +void +test_cbc() +{ + plan(1); + int i; + char source[20]; + for(i=0; i<20; i++) { + source[i] = 5; + } + ulint s_len = 20; + char dest[32]; + for(i=0; i<32; i++){ + dest[i]=0; + } + ulint dest_len = 0; + char* key = "583BE7F334F85E7D9DDB362E9AC38151"; + uint8 k_len = 16; + char* iv = "3325CC3F02203FB6B849990042E58BCB"; + uint8 i_len = 16; + int ec = my_aes_encrypt_cbc(source, s_len, &dest, &dest_len, key, k_len, iv, i_len); + ok(ec == AES_OK, "Checking return code."); + for(i=0; i<20; i++) { + source[i] = 0; + } + my_aes_decrypt_cbc(dest , dest_len, &source, &dest_len, key, k_len, iv, i_len); + ok(strcmp(source, "Beam me up, Scotty."),"Decrypted text is identical to original text."); + +} + +void +test_cbc_resultsize() +{ + plan(2); + char *source = (char*) malloc(5000*sizeof(char)); + source = "abcdefghijklmnopqrstfjdklfkjdsljsdlkfjsaklföjsfölkdsjfölsdkjklösjsdklfjdsklöfjsdalökfjdsklöjfölksdjfklösdajfklösdaj"; + ulint s_len = (ulint) strlen(source); + char* dest = (char *) malloc(2 * s_len * sizeof(char)); + ulint d_len = 0; + char* key = "583BE7F334F85E7D9DDB362E9AC38151"; + uint8 k_len = 16; + char* iv = "3325CC3F02203FB6B849990042E58BCB"; + uint8 i_len = 16; + my_aes_encrypt_cbc(source, s_len, dest, &d_len, key, k_len, iv, i_len); + ok(d_len==128, "Destination length ok."); +} + +void test_cbc_enc_dec() { + unsigned char inbuf[1024]="Hello,world!"; +unsigned char encbuf[1024]; + +unsigned char key32[] = {0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa}; +unsigned char deckey32[] = {0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa} +; +unsigned char iv[] = {0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}; +unsigned char deciv[] = {0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}; + +AES_KEY aeskey; +AES_KEY aesdeckey; + +//Now enrypt +memset(encbuf, 0, sizeof(encbuf)); +AES_set_encrypt_key(key32, 32*8, &aeskey); +AES_cbc_encrypt(inbuf, encbuf, 16, &aeskey, iv, AES_ENCRYPT); + +//Now decrypt +unsigned char decbuf[1024]; +memset(decbuf, 0, sizeof(decbuf)); + +AES_set_decrypt_key(deckey32, 32*8, &aesdeckey); +AES_cbc_encrypt(encbuf, decbuf, 16, &aesdeckey, deciv, AES_DECRYPT); + + +int i = memcmp(decbuf,inbuf,16); +ok (i==0, "in==out"); + +} + +void test_cbc_enc_dec2() { + unsigned char inbuf[1024]="Hello,world!"; +unsigned char encbuf[1024]; + +unsigned char key32[] = {0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa}; +unsigned char deckey32[] = {0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa} +; +unsigned char iv[] = {0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}; +unsigned char deciv[] = {0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}; + +AES_KEY aeskey; +AES_KEY aesdeckey; + +//Now enrypt +memset(encbuf, 0, sizeof(encbuf)); +AES_set_encrypt_key(key32, 32*8, &aeskey); +AES_cbc_encrypt(inbuf, encbuf, 16, &aeskey, iv, AES_ENCRYPT); + +//Now decrypt +unsigned char decbuf[1024]; +memset(decbuf, 0, sizeof(decbuf)); + +AES_set_decrypt_key(deckey32, 32*8, &aesdeckey); +AES_cbc_encrypt(encbuf, decbuf, 16, &aesdeckey, deciv, AES_DECRYPT); +dump_buffer(16, decbuf); +dump_buffer(16, encbuf); + +int i = memcmp(decbuf,inbuf,16); +ok (i==0, "in==out"); + +} + + + +void test_cbc_enc_() { + unsigned char inbuf[1024]="Hello,world!"; +unsigned char encbuf[1024]; + +unsigned char key32[] = {0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa}; +unsigned char deckey32[] = {0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa} +; +unsigned char iv[] = {0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}; +unsigned char deciv[] = {0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}; + +AES_KEY aeskey; +AES_KEY aesdeckey; + +//Now enrypt +memset(encbuf, 0, sizeof(encbuf)); +AES_set_encrypt_key(key32, 32*8, &aeskey); +AES_cbc_encrypt(inbuf, encbuf, 16, &aeskey, iv, AES_ENCRYPT); + +//Now decrypt +unsigned char decbuf[1024]; +memset(decbuf, 0, sizeof(decbuf)); + +AES_set_decrypt_key(deckey32, 32*8, &aesdeckey); +AES_cbc_encrypt(encbuf, decbuf, 16, &aesdeckey, deciv, AES_DECRYPT); +dump_buffer(16, decbuf); +dump_buffer(16, encbuf); + +int i = memcmp(decbuf,inbuf,16); +ok (i==0, "in==out"); + +} + + + +void test_page_enc_dec() { + char* buf = readFile("xaa"); + char* dest = (char *) malloc(16384*sizeof(char)); + //fil_encrypt_page(0,buf,dest,0,0,NULL,1); + + //fil_decrypt_page(NULL, dest, 0,NULL,1); + + ulint i = memcmp(buf,dest, 16384); + ok (i==0, "in==out"); +} + + + +int +main(int argc __attribute__((unused)),char *argv[]) +{ + test_cbc(); + test_cbc_large(); + test_cbc_keysizes(); + test_cbc_wrong_keylength(); + test_cbc_resultsize(); + test_cbc_enc_dec(); + test_wrong_key(); + return 0; +} diff --git a/unittest/eperi/eperi_keyfile-t.c b/unittest/eperi/eperi_keyfile-t.c new file mode 100644 index 0000000000000..30077490d35ae --- /dev/null +++ b/unittest/eperi/eperi_keyfile-t.c @@ -0,0 +1,38 @@ +#include +#include +#include + +void +printAll(struct keyentry **all, int length) +{ + int i; + for(i=0;iid, entry->iv, entry->key); +} + +int +main(int argc __attribute__((unused)),char *argv[]) +{ + plan(6); + struct keyentry **allKeys = (struct keyentry**) malloc( 256 * sizeof(struct keyentry*)); + FILE *fp = fopen("keys.txt", "r"); + parseFile(fp, allKeys, 256); + //printAll(allKeys, 256); + fclose(fp); + ok(allKeys[1]->id == 1, "Key id 1 is present"); + ok(!strcmp(allKeys[2]->iv,"35B2FF0795FB84BBD666DB8430CA214E"), "Testing IV value of key 2"); + ok(!strcmp(allKeys[15]->key, "B374A26A71490437AA024E4FADD5B497FDFF1A8EA6FF12F6FB65AF2720B59CCF"),"Testing key value of key 15"); + ok((allKeys[47] == 0), "Key id 47 should be null."); + ok(allKeys[255]->id == 255, "Last key inserted"); + ok((allKeys[256] == 0), "Cannot insert more keys than defined."); + return 0; +} diff --git a/unittest/eperi/johnny-t.cc b/unittest/eperi/johnny-t.cc new file mode 100644 index 0000000000000..89819b947a2c6 --- /dev/null +++ b/unittest/eperi/johnny-t.cc @@ -0,0 +1,131 @@ +/* + * johnny.cc + * + * Created on: 23.08.2014 + * Author: florin + */ +#define EP_UNIT_TEST 1 +#define UNIV_INLINE +typedef unsigned char byte; +typedef unsigned long int ulint; +typedef unsigned long int ibool; + +#include "johnny-t.h" +#include +#include + +#include + +extern int summef(int a, int b); +extern int summef2(int a, int b); +extern int multiplikation(int a, int b); + + +extern byte* +fil_encrypt_page( +/*==============*/ + ulint space_id, /*!< in: tablespace id of the + table. */ + byte* buf, /*!< in: buffer from which to write; in aio + this must be appropriately aligned */ + byte* out_buf, /*!< out: compressed buffer */ + ulint len, /*!< in: length of input buffer.*/ + ulint compression_level, /*!< in: compression level */ + ulint* out_len, /*!< out: actual length of compressed page */ + ulint unit_test); + +/****************************************************************//** +For page encrypted pages decrypt the page after actual read +operation. +@return decrypted page */ +extern void +fil_decrypt_page( +/*================*/ + byte* page_buf, /*!< in: preallocated buffer or NULL */ + byte* buf, /*!< out: buffer from which to read; in aio + this must be appropriately aligned */ + ulint len, /*!< in: length of output buffer.*/ + ulint* write_size, /*!< in/out: Actual payload size of the decrypted data. */ + ulint unit_test); + + + + + +johnny::johnny() { + printf("%s, %d\n", "johnny summef2(int a, int b)", summef2(9, 9)); + +} + +johnny::johnny(int a, int b) { + printf("%s, %d\n", "johnny multiplikation(int a, int b)", multiplikation(a, b)); + +} + +johnny::~johnny() { + // TODO Auto-generated destructor stub +} + + + +byte* readFile(char* fileName) { +FILE *fileptr; +byte *buffer; +long filelen; + +fileptr = fopen(fileName, "rb"); // Open the file in binary mode +fseek(fileptr, 0, SEEK_END); // Jump to the end of the file +filelen = ftell(fileptr); // Get the current byte offset in the file +rewind(fileptr); // Jump back to the beginning of the file + +buffer = (byte *)malloc((filelen+1)*sizeof(byte)); // Enough memory for file + \0 +fread(buffer, filelen, 1, fileptr); // Read in the entire file +fclose(fileptr); // Close the file +return buffer; +} + + +void testIt(char* filename, ulint cmp_checksum) { + byte* buf = readFile(filename); + byte* dest = (byte *) malloc(16384*sizeof(byte)); + ulint out_len; + fil_encrypt_page(0,buf,dest,0,255,&out_len,1); + + fil_decrypt_page(NULL, dest, 16384 ,NULL,1); + ulint a = 0; + ulint b = 0; + if (cmp_checksum) { + a = 4; + b = 8; + } + ulint i = memcmp(buf + a,dest +a, 16384 - (a+b)); + if (i!=0) { + fprintf(stderr,"falsch"); + fflush(stderr); + } + char str[80]; + strcpy (str,"File "); + strcat (str,filename ); + + ok (i==0, str); +} +void test_page_enc_dec() { +testIt("xaa",0); + testIt("xab",0); + testIt("xac",0); + testIt("xad",0); + + + testIt("xae",1); + testIt("xaf",1); +} + + +int main() +{ + + + test_page_enc_dec(); + + return 0; +} diff --git a/unittest/eperi/johnny-t.h b/unittest/eperi/johnny-t.h new file mode 100644 index 0000000000000..15323162813f6 --- /dev/null +++ b/unittest/eperi/johnny-t.h @@ -0,0 +1,19 @@ +/* + * johnny.h + * + * Created on: 23.08.2014 + * Author: florin + */ + +#ifndef JOHNNY_H_ +#define JOHNNY_H_ + + +class johnny { +public: + johnny(); + johnny(int a, int b); + virtual ~johnny(); +}; + +#endif /* JOHNNY_H_ */ diff --git a/unittest/eperi/keys.txt b/unittest/eperi/keys.txt new file mode 100644 index 0000000000000..9d0a279a895d3 --- /dev/null +++ b/unittest/eperi/keys.txt @@ -0,0 +1,11 @@ +#Page encryption key file +#Each entry consists of ;; +1;F5502320F8429037B8DAEF761B189D12;770A8A65DA156D24EE2A093277530142 +2;35B2FF0795FB84BBD666DB8430CA214E;4D92199549E0F2EF009B4160F3582E5528A11A45017F3EF8 +3;7E892875A52C59A3B588306B13C31FBD;B374A26A71490437AA024E4FADD5B497FDFF1A8EA6FF12F6FB65AF2720B59CCF +15;7E892875A52C59A3B588306B13C31FBD;B374A26A71490437AA024E4FADD5B497FDFF1A8EA6FF12F6FB65AF2720B59CCF +#15;7E892875A52C59A3B588306B13C31FBD;B374A26A71490437AA024E4FADD5B497FDFF1A8EA6FF12F6FB65AF2720B59CCF +1024;7E892875A52C59A3B588306B13C31FBD;B374A26A71490437AA024E4FADD5B497FDFF1A8EA6FF12F6FB65AF2720B59CCF +3;7E892875A52C59A3B5883z6B13C31FBD;B374A26A71490437AA024E4FADD5B497FDFF1A8EA6FF12F6FB65AF2720B59CCF +255;F5502320F8429037B8DAEF761B189D12;770A8A65DA156D24EE2A093277530142 +256;F5502320F8429037B8DAEF761B189D12;770A8A65DA156D24EE2A093277530142 diff --git a/unittest/eperi/kf.txt b/unittest/eperi/kf.txt new file mode 100644 index 0000000000000..e468e2850e74f --- /dev/null +++ b/unittest/eperi/kf.txt @@ -0,0 +1 @@ +Florin diff --git a/unittest/eperi/kfo.txt b/unittest/eperi/kfo.txt new file mode 100644 index 0000000000000..b27dd3b900c93 --- /dev/null +++ b/unittest/eperi/kfo.txt @@ -0,0 +1 @@ +Salted__�Yc۸V��ʱ��T����/r��ҩs \ No newline at end of file diff --git a/unittest/eperi/pfs_jim-t.cc b/unittest/eperi/pfs_jim-t.cc new file mode 100644 index 0000000000000..580adc41aef06 --- /dev/null +++ b/unittest/eperi/pfs_jim-t.cc @@ -0,0 +1,126 @@ +/* Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software Foundation, + 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */ + +#include +#include +#include +#include +#include +#include +#include + +#include "stub_pfs_global.h" +#include "stub_server_misc.h" + +#include /* memset */ + +extern int summef(int a, int b); +extern int summef2(int a, int b); + + +void test_oomf() +{ + int rc; + PFS_global_param param; + + memset(& param, 0xFF, sizeof(param)); + param.m_enabled= true; + param.m_mutex_class_sizing= 0; + param.m_rwlock_class_sizing= 0; + param.m_cond_class_sizing= 0; + param.m_thread_class_sizing= 10; + param.m_table_share_sizing= 0; + param.m_file_class_sizing= 0; + param.m_mutex_sizing= 0; + param.m_rwlock_sizing= 0; + param.m_cond_sizing= 0; + param.m_thread_sizing= 1000; + param.m_table_sizing= 0; + param.m_file_sizing= 0; + param.m_file_handle_sizing= 0; + param.m_events_waits_history_sizing= 10; + param.m_events_waits_history_long_sizing= 0; + param.m_setup_actor_sizing= 0; + param.m_setup_object_sizing= 0; + param.m_host_sizing= 0; + param.m_user_sizing= 0; + param.m_account_sizing= 1000; + param.m_stage_class_sizing= 50; + param.m_events_stages_history_sizing= 0; + param.m_events_stages_history_long_sizing= 0; + param.m_statement_class_sizing= 50; + param.m_events_statements_history_sizing= 0; + param.m_events_statements_history_long_sizing= 0; + param.m_session_connect_attrs_sizing= 0; + + /* Setup */ + + stub_alloc_always_fails= false; + stub_alloc_fails_after_count= 1000; + + init_event_name_sizing(& param); + rc= init_stage_class(param.m_stage_class_sizing); + ok(rc == 0, "init stage class"); + rc= init_statement_class(param.m_statement_class_sizing); + ok(rc == 0, "init statement class"); + + /* Tests */ + + stub_alloc_fails_after_count= 1; + rc= init_account(& param); + ok(rc == 1, "oom (account)"); + cleanup_account(); + + stub_alloc_fails_after_count= 2; + rc= init_account(& param); + ok(rc == 1, "oom (account waits)"); + cleanup_account(); + + stub_alloc_fails_after_count= 3; + rc= init_account(& param); + ok(rc == 1, "oom (account stages)"); + cleanup_account(); + + stub_alloc_fails_after_count= 4; + rc= init_account(& param); + ok(rc == 1, "oom (account statements)"); + cleanup_account(); + + cleanup_statement_class(); + cleanup_stage_class(); +} + +void do_all_tests() +{ + PFS_atomic::init(); + + test_oomf(); + + PFS_atomic::cleanup(); +} + +int main(int, char **) +{ + plan(6); + MY_INIT("pfs_account-oomff-t"); + do_all_tests(); + int sum = summef(3, 9); + printf("summef meine Zahl: %d\n", sum); + sum = summef2(7, 3); + printf("summef2 meine Zahl: %d\n", sum); + my_end(0); + return exit_status(); +} + diff --git a/unittest/eperi/xaa b/unittest/eperi/xaa new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/unittest/eperi/xab b/unittest/eperi/xab new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/unittest/eperi/xac b/unittest/eperi/xac new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/unittest/eperi/xad b/unittest/eperi/xad new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/unittest/eperi/xae b/unittest/eperi/xae new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/unittest/eperi/xaf b/unittest/eperi/xaf new file mode 100644 index 0000000000000..e69de29bb2d1d From b6bb3f154c0cfe62cbcbedb924efd21531029a16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Clemens=20D=C3=B6rrh=C3=B6fer?= Date: Wed, 24 Sep 2014 15:27:24 +0200 Subject: [PATCH 02/70] Cmake Problem fix --- CMakeLists.txt | 1 + mysys/CMakeLists.txt | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 68f3b01eb28a2..a4a71a96e37db 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -358,6 +358,7 @@ IF(WITH_UNIT_TESTS) ADD_SUBDIRECTORY(unittest/examples) ADD_SUBDIRECTORY(unittest/mysys) ADD_SUBDIRECTORY(unittest/my_decimal) + ADD_SUBDIRECTORY(unittest/eperi) IF(NOT WITHOUT_SERVER) ADD_SUBDIRECTORY(unittest/sql) ENDIF() diff --git a/mysys/CMakeLists.txt b/mysys/CMakeLists.txt index e342427c73749..6dff5736fddf4 100644 --- a/mysys/CMakeLists.txt +++ b/mysys/CMakeLists.txt @@ -43,7 +43,7 @@ SET(MYSYS_SOURCES array.c charset-def.c charset.c checksum.c my_default.c my_atomic.c my_getncpus.c my_safehash.c my_chmod.c my_rnd.c my_uuid.c wqueue.c waiting_threads.c ma_dyncol.c my_rdtsc.c my_context.c psi_noop.c - file_logger.c) + file_logger.c keyfile.c) IF (WIN32) SET (MYSYS_SOURCES ${MYSYS_SOURCES} my_winthread.c my_wincond.c my_winerr.c my_winfile.c my_windac.c my_conio.c) From a939f5a2fe4d2a0b05ae7005efac7bec92854bcc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Clemens=20D=C3=B6rrh=C3=B6fer?= Date: Wed, 24 Sep 2014 15:27:42 +0200 Subject: [PATCH 03/70] gitignore --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 24d82cb758c2d..0371ea94786be 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,6 @@ +.cproject +.project +Debug/* *-t *.a *.ctest From 08bb1a4a7085a0c4587406039bd6cbb6a632c0d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludger=20G=C3=B6ckel?= Date: Wed, 24 Sep 2014 16:10:14 +0200 Subject: [PATCH 04/70] fixed unit tests --- CMakeLists.txt | 2 + storage/xtradb/fil/fil0pageencryption.cc | 49 ++++++------------- storage/xtradb/include/fil0pageencryption.h | 3 -- unittest/eperi/CMakeLists.txt | 29 +++++++++-- unittest/eperi/enc0enc-t.cc | 47 ------------------ unittest/eperi/enc0enc-t.h | 13 ----- unittest/eperi/{johnny-t.cc => pageenc-t.cc} | 21 ++------ unittest/eperi/{johnny-t.h => pageenc-t.h} | 7 --- unittest/eperi/xaa | Bin 0 -> 16384 bytes unittest/eperi/xab | Bin 0 -> 16384 bytes unittest/eperi/xac | Bin 0 -> 16384 bytes unittest/eperi/xad | Bin 0 -> 16384 bytes unittest/eperi/xae | Bin 0 -> 16384 bytes unittest/eperi/xaf | Bin 0 -> 16384 bytes 14 files changed, 44 insertions(+), 127 deletions(-) delete mode 100644 unittest/eperi/enc0enc-t.cc delete mode 100644 unittest/eperi/enc0enc-t.h rename unittest/eperi/{johnny-t.cc => pageenc-t.cc} (88%) rename unittest/eperi/{johnny-t.h => pageenc-t.h} (62%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 68f3b01eb28a2..7c5e20063bdbb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -358,6 +358,8 @@ IF(WITH_UNIT_TESTS) ADD_SUBDIRECTORY(unittest/examples) ADD_SUBDIRECTORY(unittest/mysys) ADD_SUBDIRECTORY(unittest/my_decimal) + ADD_SUBDIRECTORY(unittest/eperi) + IF(NOT WITHOUT_SERVER) ADD_SUBDIRECTORY(unittest/sql) ENDIF() diff --git a/storage/xtradb/fil/fil0pageencryption.cc b/storage/xtradb/fil/fil0pageencryption.cc index b062098afa94f..7b58ab832d50f 100644 --- a/storage/xtradb/fil/fil0pageencryption.cc +++ b/storage/xtradb/fil/fil0pageencryption.cc @@ -1,19 +1,8 @@ /***************************************************************************** -Copyright (C) 2013, 2014, SkySQL Ab. All Rights Reserved. - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., -51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +Copyright (C) 2014 +eperi GmbH *****************************************************************************/ /******************************************************************//** @@ -317,9 +306,8 @@ fil_decrypt_page( ; } - /* Get flush lsn bytes 1..3 to determine, whether the page is PAGE_COMPRESSED and PAGE_ENCRYPTION */ - + /* flush lsn bytes 1..5 are used to store original page type, etc.*/ offset_ctrl_data = page_size - FIL_PAGE_DATA_END - FIL_PAGE_FILE_FLUSH_LSN; @@ -384,6 +372,18 @@ fil_decrypt_page( stringiv, iv_len ); + + + /* If decrypt fails it means that page is corrupted or has an unknown key */ + if (err != AES_OK) { + fprintf(stderr, "InnoDB: Corruption: Page is marked as encrypted\n" + "InnoDB: but decrypt failed with error %d.\n" + "InnoDB: size %lu len %lu, key%d\n", err, data_size, + len, page_encryption_key); + fflush(stderr); + ut_error; + } + ut_ad(tmp_write_size == 63); /* copy 1st part from buf to tmp_page_buf */ @@ -478,25 +478,6 @@ fil_decrypt_page( -int summef(int a, int b) -{ - return a + b; -} - -int summef2(int a, int b) -{ - int s = (a + b)^2; - s = (a + b)*2; - return s; -} - -int multiplikation(int a, int b) -{ - return a * b; -} - - - void do_check_sum( ulint page_size, byte* buf) { ib_uint32_t checksum; diff --git a/storage/xtradb/include/fil0pageencryption.h b/storage/xtradb/include/fil0pageencryption.h index c192d40fda812..30b0d8ee78ba6 100644 --- a/storage/xtradb/include/fil0pageencryption.h +++ b/storage/xtradb/include/fil0pageencryption.h @@ -90,8 +90,5 @@ fil_decrypt_page( void do_check_sum( ulint page_size, byte* buf); -int summef(int a, int b); -int summef2(int a, int b); -int multiplikation(int a, int b); #endif // fil0pageencryption_h diff --git a/unittest/eperi/CMakeLists.txt b/unittest/eperi/CMakeLists.txt index 9f8745bbf7227..0e605c7a4e66e 100644 --- a/unittest/eperi/CMakeLists.txt +++ b/unittest/eperi/CMakeLists.txt @@ -25,14 +25,13 @@ INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql ) MY_ADD_TESTS(eperi) -MY_ADD_TESTS(eperi_keyfile - LINK_LIBRARIES mysys pcre) +#MY_ADD_TESTS(eperi_keyfile +# LINK_LIBRARIES mysys pcre) MY_ADD_TESTS(eperi_aes LINK_LIBRARIES mysys_ssl dbug) -MY_ADD_TESTS(enc0enc - - johnny +MY_ADD_TESTS( + pageenc EXT "cc" LINK_LIBRARIES xtradb perfschema mysys mysys_ssl sql mysql) @@ -40,3 +39,23 @@ MY_ADD_TESTS(enc0enc file(COPY ${CMAKE_CURRENT_LIST_DIR}/keys.txt DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) + +file(COPY + ${CMAKE_CURRENT_LIST_DIR}/xaa + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) +file(COPY + ${CMAKE_CURRENT_LIST_DIR}/xab + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) +file(COPY + ${CMAKE_CURRENT_LIST_DIR}/xac + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) +file(COPY + ${CMAKE_CURRENT_LIST_DIR}/xad + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) +file(COPY + ${CMAKE_CURRENT_LIST_DIR}/xae + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) +file(COPY + ${CMAKE_CURRENT_LIST_DIR}/xaf + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) + \ No newline at end of file diff --git a/unittest/eperi/enc0enc-t.cc b/unittest/eperi/enc0enc-t.cc deleted file mode 100644 index 40033f4f2a70c..0000000000000 --- a/unittest/eperi/enc0enc-t.cc +++ /dev/null @@ -1,47 +0,0 @@ -/* - * enc0enc.c - * - * Created on: 22.08.2014 - * Author: florin - */ - -#include "enc0enc-t.h" -#include -#include -#include - -typedef unsigned long int ulint; - -extern int summef(int a, int b); -extern int summef2(int a, int b); -extern void fil_flush(ulint space_id); - - -int main() -{ - plan(9); - //MY_INIT("enc0enc-t"); - printf("%s\n", "main() enc0enc.c"); - int sum = summef(2, 3); - printf("%s, %d\n", "summef(int a, int b)", sum); - //fil_flush(3); - printf("%s, %d\n", "summef2(int a, int b)", summef2(2, 7)); -/* - string encrypted = encryptDecrypt("kylewbanks.com"); - cout << "Encrypted:" << encrypted << "\n"; - - string decrypted = encryptDecrypt(encrypted); - cout << "Decrypted:" << decrypted << "\n"; -*/ - return 0; -} - - -/* -int summe(int a, int b) -{ - return a + b; -} - -int summe(int a, int b); -*/ diff --git a/unittest/eperi/enc0enc-t.h b/unittest/eperi/enc0enc-t.h deleted file mode 100644 index f17fa8e4a3c31..0000000000000 --- a/unittest/eperi/enc0enc-t.h +++ /dev/null @@ -1,13 +0,0 @@ -/* - * enc0enc.h - * - * Created on: 22.08.2014 - * Author: florin - */ - -#ifndef ENC0ENC_H_ -#define ENC0ENC_H_ - - - -#endif /* ENC0ENC_H_ */ diff --git a/unittest/eperi/johnny-t.cc b/unittest/eperi/pageenc-t.cc similarity index 88% rename from unittest/eperi/johnny-t.cc rename to unittest/eperi/pageenc-t.cc index 89819b947a2c6..456b15bcaa74c 100644 --- a/unittest/eperi/johnny-t.cc +++ b/unittest/eperi/pageenc-t.cc @@ -1,5 +1,5 @@ /* - * johnny.cc + * pageenc.cc * * Created on: 23.08.2014 * Author: florin @@ -10,7 +10,7 @@ typedef unsigned char byte; typedef unsigned long int ulint; typedef unsigned long int ibool; -#include "johnny-t.h" +#include "pageenc-t.h" #include #include @@ -52,19 +52,7 @@ fil_decrypt_page( -johnny::johnny() { - printf("%s, %d\n", "johnny summef2(int a, int b)", summef2(9, 9)); -} - -johnny::johnny(int a, int b) { - printf("%s, %d\n", "johnny multiplikation(int a, int b)", multiplikation(a, b)); - -} - -johnny::~johnny() { - // TODO Auto-generated destructor stub -} @@ -99,10 +87,7 @@ void testIt(char* filename, ulint cmp_checksum) { b = 8; } ulint i = memcmp(buf + a,dest +a, 16384 - (a+b)); - if (i!=0) { - fprintf(stderr,"falsch"); - fflush(stderr); - } + char str[80]; strcpy (str,"File "); strcat (str,filename ); diff --git a/unittest/eperi/johnny-t.h b/unittest/eperi/pageenc-t.h similarity index 62% rename from unittest/eperi/johnny-t.h rename to unittest/eperi/pageenc-t.h index 15323162813f6..aff6b99aa723a 100644 --- a/unittest/eperi/johnny-t.h +++ b/unittest/eperi/pageenc-t.h @@ -9,11 +9,4 @@ #define JOHNNY_H_ -class johnny { -public: - johnny(); - johnny(int a, int b); - virtual ~johnny(); -}; - #endif /* JOHNNY_H_ */ diff --git a/unittest/eperi/xaa b/unittest/eperi/xaa index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..bd299739b271318e9ccf10b4f48beef648136695 100644 GIT binary patch literal 16384 zcmeI(u?@m75Cza91eAsvu>c#;Q!-3uVgLqV7?uEkBp~tzNbjWl*ynTede6Jdqxw3G zrh+(1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U fAV7cs0RjXF5FkK+009C72oU&9;M|`hedqrIzn})h literal 0 HcmV?d00001 diff --git a/unittest/eperi/xac b/unittest/eperi/xac index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..cfc22173a77d54e0ee54823ec854ed5e7561aff2 100644 GIT binary patch literal 16384 zcmeI(Jr05}6ae6dpjR*+gx%4rcq9fd#o$4lHP}W9gn^aBd@pI6*PrzD+k5`Jx`;TW zHpiuDqn^_^4Jjgi{WxEh_4V3Teth2hbgs^O_KLCY>BYH@<=eLMT>oBmmwSu5kpKY# z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs a0RjXF5FkK+009C7{z2ezy}cv-^Zx?YR6KnE literal 0 HcmV?d00001 diff --git a/unittest/eperi/xad b/unittest/eperi/xad index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0577d155b869a91e872547cc204fabb1eb07f0f8 100644 GIT binary patch literal 16384 zcmeI(KTZNc6bIn9Y@on~3IgE*w00iA#@+)Mf?*T>MYFYD!p_or=o!3;TmarKhHwV* zO)~HOUS=}I*U#hj?l&UJ!~e=0F&>WYzuR;wz39Z}d)W?e(am05oSxz^TQ7Gf_UCk4 zbroceibaQMp_W&7A{{E^Q-6oci{YE?C5+ZYwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM q7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjMyaxtA00031 literal 0 HcmV?d00001 diff --git a/unittest/eperi/xaf b/unittest/eperi/xaf index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..294f4016d05bdd696670c4840f1f36a71f9239de 100644 GIT binary patch literal 16384 zcmeIuF#!Mo0K%a4Pi+hzh(KY$fB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM q7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjMyaxtA00031 literal 0 HcmV?d00001 From 71eac244841f8d7d56eec91853bcecb2a8d80bad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludger=20G=C3=B6ckel?= Date: Wed, 24 Sep 2014 16:10:14 +0200 Subject: [PATCH 05/70] Bugfix unittests --- CMakeLists.txt | 1 + mysys/CMakeLists.txt | 2 +- storage/xtradb/fil/fil0pageencryption.cc | 49 ++++++------------- storage/xtradb/include/fil0pageencryption.h | 3 -- unittest/eperi/CMakeLists.txt | 29 +++++++++-- unittest/eperi/enc0enc-t.cc | 47 ------------------ unittest/eperi/enc0enc-t.h | 13 ----- unittest/eperi/{johnny-t.cc => pageenc-t.cc} | 21 ++------ unittest/eperi/{johnny-t.h => pageenc-t.h} | 7 --- unittest/eperi/xaa | Bin 0 -> 16384 bytes unittest/eperi/xab | Bin 0 -> 16384 bytes unittest/eperi/xac | Bin 0 -> 16384 bytes unittest/eperi/xad | Bin 0 -> 16384 bytes unittest/eperi/xae | Bin 0 -> 16384 bytes unittest/eperi/xaf | Bin 0 -> 16384 bytes 15 files changed, 44 insertions(+), 128 deletions(-) delete mode 100644 unittest/eperi/enc0enc-t.cc delete mode 100644 unittest/eperi/enc0enc-t.h rename unittest/eperi/{johnny-t.cc => pageenc-t.cc} (88%) rename unittest/eperi/{johnny-t.h => pageenc-t.h} (62%) diff --git a/CMakeLists.txt b/CMakeLists.txt index 68f3b01eb28a2..a4a71a96e37db 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -358,6 +358,7 @@ IF(WITH_UNIT_TESTS) ADD_SUBDIRECTORY(unittest/examples) ADD_SUBDIRECTORY(unittest/mysys) ADD_SUBDIRECTORY(unittest/my_decimal) + ADD_SUBDIRECTORY(unittest/eperi) IF(NOT WITHOUT_SERVER) ADD_SUBDIRECTORY(unittest/sql) ENDIF() diff --git a/mysys/CMakeLists.txt b/mysys/CMakeLists.txt index e342427c73749..6dff5736fddf4 100644 --- a/mysys/CMakeLists.txt +++ b/mysys/CMakeLists.txt @@ -43,7 +43,7 @@ SET(MYSYS_SOURCES array.c charset-def.c charset.c checksum.c my_default.c my_atomic.c my_getncpus.c my_safehash.c my_chmod.c my_rnd.c my_uuid.c wqueue.c waiting_threads.c ma_dyncol.c my_rdtsc.c my_context.c psi_noop.c - file_logger.c) + file_logger.c keyfile.c) IF (WIN32) SET (MYSYS_SOURCES ${MYSYS_SOURCES} my_winthread.c my_wincond.c my_winerr.c my_winfile.c my_windac.c my_conio.c) diff --git a/storage/xtradb/fil/fil0pageencryption.cc b/storage/xtradb/fil/fil0pageencryption.cc index b062098afa94f..7b58ab832d50f 100644 --- a/storage/xtradb/fil/fil0pageencryption.cc +++ b/storage/xtradb/fil/fil0pageencryption.cc @@ -1,19 +1,8 @@ /***************************************************************************** -Copyright (C) 2013, 2014, SkySQL Ab. All Rights Reserved. - -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., -51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +Copyright (C) 2014 +eperi GmbH *****************************************************************************/ /******************************************************************//** @@ -317,9 +306,8 @@ fil_decrypt_page( ; } - /* Get flush lsn bytes 1..3 to determine, whether the page is PAGE_COMPRESSED and PAGE_ENCRYPTION */ - + /* flush lsn bytes 1..5 are used to store original page type, etc.*/ offset_ctrl_data = page_size - FIL_PAGE_DATA_END - FIL_PAGE_FILE_FLUSH_LSN; @@ -384,6 +372,18 @@ fil_decrypt_page( stringiv, iv_len ); + + + /* If decrypt fails it means that page is corrupted or has an unknown key */ + if (err != AES_OK) { + fprintf(stderr, "InnoDB: Corruption: Page is marked as encrypted\n" + "InnoDB: but decrypt failed with error %d.\n" + "InnoDB: size %lu len %lu, key%d\n", err, data_size, + len, page_encryption_key); + fflush(stderr); + ut_error; + } + ut_ad(tmp_write_size == 63); /* copy 1st part from buf to tmp_page_buf */ @@ -478,25 +478,6 @@ fil_decrypt_page( -int summef(int a, int b) -{ - return a + b; -} - -int summef2(int a, int b) -{ - int s = (a + b)^2; - s = (a + b)*2; - return s; -} - -int multiplikation(int a, int b) -{ - return a * b; -} - - - void do_check_sum( ulint page_size, byte* buf) { ib_uint32_t checksum; diff --git a/storage/xtradb/include/fil0pageencryption.h b/storage/xtradb/include/fil0pageencryption.h index c192d40fda812..30b0d8ee78ba6 100644 --- a/storage/xtradb/include/fil0pageencryption.h +++ b/storage/xtradb/include/fil0pageencryption.h @@ -90,8 +90,5 @@ fil_decrypt_page( void do_check_sum( ulint page_size, byte* buf); -int summef(int a, int b); -int summef2(int a, int b); -int multiplikation(int a, int b); #endif // fil0pageencryption_h diff --git a/unittest/eperi/CMakeLists.txt b/unittest/eperi/CMakeLists.txt index 9f8745bbf7227..0e605c7a4e66e 100644 --- a/unittest/eperi/CMakeLists.txt +++ b/unittest/eperi/CMakeLists.txt @@ -25,14 +25,13 @@ INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql ) MY_ADD_TESTS(eperi) -MY_ADD_TESTS(eperi_keyfile - LINK_LIBRARIES mysys pcre) +#MY_ADD_TESTS(eperi_keyfile +# LINK_LIBRARIES mysys pcre) MY_ADD_TESTS(eperi_aes LINK_LIBRARIES mysys_ssl dbug) -MY_ADD_TESTS(enc0enc - - johnny +MY_ADD_TESTS( + pageenc EXT "cc" LINK_LIBRARIES xtradb perfschema mysys mysys_ssl sql mysql) @@ -40,3 +39,23 @@ MY_ADD_TESTS(enc0enc file(COPY ${CMAKE_CURRENT_LIST_DIR}/keys.txt DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) + +file(COPY + ${CMAKE_CURRENT_LIST_DIR}/xaa + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) +file(COPY + ${CMAKE_CURRENT_LIST_DIR}/xab + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) +file(COPY + ${CMAKE_CURRENT_LIST_DIR}/xac + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) +file(COPY + ${CMAKE_CURRENT_LIST_DIR}/xad + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) +file(COPY + ${CMAKE_CURRENT_LIST_DIR}/xae + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) +file(COPY + ${CMAKE_CURRENT_LIST_DIR}/xaf + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) + \ No newline at end of file diff --git a/unittest/eperi/enc0enc-t.cc b/unittest/eperi/enc0enc-t.cc deleted file mode 100644 index 40033f4f2a70c..0000000000000 --- a/unittest/eperi/enc0enc-t.cc +++ /dev/null @@ -1,47 +0,0 @@ -/* - * enc0enc.c - * - * Created on: 22.08.2014 - * Author: florin - */ - -#include "enc0enc-t.h" -#include -#include -#include - -typedef unsigned long int ulint; - -extern int summef(int a, int b); -extern int summef2(int a, int b); -extern void fil_flush(ulint space_id); - - -int main() -{ - plan(9); - //MY_INIT("enc0enc-t"); - printf("%s\n", "main() enc0enc.c"); - int sum = summef(2, 3); - printf("%s, %d\n", "summef(int a, int b)", sum); - //fil_flush(3); - printf("%s, %d\n", "summef2(int a, int b)", summef2(2, 7)); -/* - string encrypted = encryptDecrypt("kylewbanks.com"); - cout << "Encrypted:" << encrypted << "\n"; - - string decrypted = encryptDecrypt(encrypted); - cout << "Decrypted:" << decrypted << "\n"; -*/ - return 0; -} - - -/* -int summe(int a, int b) -{ - return a + b; -} - -int summe(int a, int b); -*/ diff --git a/unittest/eperi/enc0enc-t.h b/unittest/eperi/enc0enc-t.h deleted file mode 100644 index f17fa8e4a3c31..0000000000000 --- a/unittest/eperi/enc0enc-t.h +++ /dev/null @@ -1,13 +0,0 @@ -/* - * enc0enc.h - * - * Created on: 22.08.2014 - * Author: florin - */ - -#ifndef ENC0ENC_H_ -#define ENC0ENC_H_ - - - -#endif /* ENC0ENC_H_ */ diff --git a/unittest/eperi/johnny-t.cc b/unittest/eperi/pageenc-t.cc similarity index 88% rename from unittest/eperi/johnny-t.cc rename to unittest/eperi/pageenc-t.cc index 89819b947a2c6..456b15bcaa74c 100644 --- a/unittest/eperi/johnny-t.cc +++ b/unittest/eperi/pageenc-t.cc @@ -1,5 +1,5 @@ /* - * johnny.cc + * pageenc.cc * * Created on: 23.08.2014 * Author: florin @@ -10,7 +10,7 @@ typedef unsigned char byte; typedef unsigned long int ulint; typedef unsigned long int ibool; -#include "johnny-t.h" +#include "pageenc-t.h" #include #include @@ -52,19 +52,7 @@ fil_decrypt_page( -johnny::johnny() { - printf("%s, %d\n", "johnny summef2(int a, int b)", summef2(9, 9)); -} - -johnny::johnny(int a, int b) { - printf("%s, %d\n", "johnny multiplikation(int a, int b)", multiplikation(a, b)); - -} - -johnny::~johnny() { - // TODO Auto-generated destructor stub -} @@ -99,10 +87,7 @@ void testIt(char* filename, ulint cmp_checksum) { b = 8; } ulint i = memcmp(buf + a,dest +a, 16384 - (a+b)); - if (i!=0) { - fprintf(stderr,"falsch"); - fflush(stderr); - } + char str[80]; strcpy (str,"File "); strcat (str,filename ); diff --git a/unittest/eperi/johnny-t.h b/unittest/eperi/pageenc-t.h similarity index 62% rename from unittest/eperi/johnny-t.h rename to unittest/eperi/pageenc-t.h index 15323162813f6..aff6b99aa723a 100644 --- a/unittest/eperi/johnny-t.h +++ b/unittest/eperi/pageenc-t.h @@ -9,11 +9,4 @@ #define JOHNNY_H_ -class johnny { -public: - johnny(); - johnny(int a, int b); - virtual ~johnny(); -}; - #endif /* JOHNNY_H_ */ diff --git a/unittest/eperi/xaa b/unittest/eperi/xaa index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..bd299739b271318e9ccf10b4f48beef648136695 100644 GIT binary patch literal 16384 zcmeI(u?@m75Cza91eAsvu>c#;Q!-3uVgLqV7?uEkBp~tzNbjWl*ynTede6Jdqxw3G zrh+(1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U fAV7cs0RjXF5FkK+009C72oU&9;M|`hedqrIzn})h literal 0 HcmV?d00001 diff --git a/unittest/eperi/xac b/unittest/eperi/xac index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..cfc22173a77d54e0ee54823ec854ed5e7561aff2 100644 GIT binary patch literal 16384 zcmeI(Jr05}6ae6dpjR*+gx%4rcq9fd#o$4lHP}W9gn^aBd@pI6*PrzD+k5`Jx`;TW zHpiuDqn^_^4Jjgi{WxEh_4V3Teth2hbgs^O_KLCY>BYH@<=eLMT>oBmmwSu5kpKY# z1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs a0RjXF5FkK+009C7{z2ezy}cv-^Zx?YR6KnE literal 0 HcmV?d00001 diff --git a/unittest/eperi/xad b/unittest/eperi/xad index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..0577d155b869a91e872547cc204fabb1eb07f0f8 100644 GIT binary patch literal 16384 zcmeI(KTZNc6bIn9Y@on~3IgE*w00iA#@+)Mf?*T>MYFYD!p_or=o!3;TmarKhHwV* zO)~HOUS=}I*U#hj?l&UJ!~e=0F&>WYzuR;wz39Z}d)W?e(am05oSxz^TQ7Gf_UCk4 zbroceibaQMp_W&7A{{E^Q-6oci{YE?C5+ZYwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM q7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjMyaxtA00031 literal 0 HcmV?d00001 diff --git a/unittest/eperi/xaf b/unittest/eperi/xaf index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..294f4016d05bdd696670c4840f1f36a71f9239de 100644 GIT binary patch literal 16384 zcmeIuF#!Mo0K%a4Pi+hzh(KY$fB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM q7%*VKfB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjMyaxtA00031 literal 0 HcmV?d00001 From 997cfc479a1930980b9bcd1c5cfdf11df03a2fe1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludger=20G=C3=B6ckel?= Date: Fri, 26 Sep 2014 11:33:45 +0200 Subject: [PATCH 06/70] page compression activated and page compression combined with page encryption fixed --- dbug/dbug.c | 2 - storage/xtradb/fil/fil0fil.cc | 13 +- storage/xtradb/fil/fil0pagecompress.cc | 3 +- storage/xtradb/fil/fil0pageencryption.cc | 414 ++++++++++---------- storage/xtradb/include/fil0pageencryption.h | 27 +- storage/xtradb/os/os0file.cc | 65 +-- unittest/eperi/eperi_aes-t.c | 27 -- unittest/eperi/pageenc-t.cc | 13 +- 8 files changed, 263 insertions(+), 301 deletions(-) diff --git a/dbug/dbug.c b/dbug/dbug.c index 8ab183dc85869..b26500ee6a515 100644 --- a/dbug/dbug.c +++ b/dbug/dbug.c @@ -2187,7 +2187,6 @@ const char* _db_get_func_(void) void dump_buffer(unsigned n, const unsigned char* buf) { -#if defined(UNIV_DEBUG) int on_this_line = 0; int counter = 0; int cc =0; @@ -2227,7 +2226,6 @@ while (n-- > 0) { } fprintf( stream, "\n"); fflush(stream); -#endif } diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 96a80aaab6be0..e8a59622b67d0 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -754,7 +754,10 @@ fil_node_open_file( ut_ad(mutex_own(&(system->mutex))); ut_a(node->n_pending == 0); ut_a(node->open == FALSE); - + if (strcmp(node->name,"test/b")==0) { + fprintf(stderr,"file access: %s", node->name); + fflush(stderr); + } if (node->size == 0) { /* It must be a single-table tablespace and we do not know the size of the file yet. First we open the file in the normal @@ -2144,7 +2147,9 @@ fil_read_first_page( /* Align the memory for a possible read from a raw device */ page = static_cast(ut_align(buf, UNIV_PAGE_SIZE)); - +if (orig_space_id==18446744073709551615) { + //return NULL; +} os_file_read(data_file, page, 0, UNIV_PAGE_SIZE, orig_space_id != ULINT_UNDEFINED ? fil_space_is_page_compressed(orig_space_id) : @@ -2158,9 +2163,9 @@ fil_read_first_page( ulint write_size=0; fil_decompress_page(NULL, page, UNIV_PAGE_SIZE, &write_size); } - *space_id = fsp_header_get_space_id(page); + flushed_lsn = mach_read_from_8(page + FIL_PAGE_FILE_FLUSH_LSN); if (!one_read_already) { @@ -2590,7 +2595,7 @@ static ulint fil_check_pending_io( /*=================*/ - fil_space_t* space, /*!< in/out: Tablespace to check */ + fil_space_t* space, /*!< in/out: Tablespace to chemismatchck */ fil_node_t** node, /*!< out: Node in space list */ ulint count) /*!< in: number of attempts so far */ { diff --git a/storage/xtradb/fil/fil0pagecompress.cc b/storage/xtradb/fil/fil0pagecompress.cc index e92f35872369a..8c4b2cb13aa54 100644 --- a/storage/xtradb/fil/fil0pagecompress.cc +++ b/storage/xtradb/fil/fil0pagecompress.cc @@ -268,7 +268,8 @@ fil_compress_page( int level = 0; ulint header_len = FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE; ulint write_size=0; - ulint comp_method = innodb_compression_algorithm; /* Cache to avoid + ulint comp_method = 1;//innodb_compression_algorithm; + /* Cache to avoid change during function execution */ ut_ad(buf); diff --git a/storage/xtradb/fil/fil0pageencryption.cc b/storage/xtradb/fil/fil0pageencryption.cc index 7b58ab832d50f..f82e2e8861a0b 100644 --- a/storage/xtradb/fil/fil0pageencryption.cc +++ b/storage/xtradb/fil/fil0pageencryption.cc @@ -1,16 +1,16 @@ /***************************************************************************** -Copyright (C) 2014 + Copyright (C) 2014 -eperi GmbH -*****************************************************************************/ + eperi GmbH + *****************************************************************************/ /******************************************************************//** -@file fil/fil0pageencryption.cc -Implementation for page encryption file spaces. + @file fil/fil0pageencryption.cc + Implementation for page encryption file spaces. -Created 08/25/2014 Florin Fugaciu -***********************************************************************/ + Created 08/25/2014 Florin Fugaciu + ***********************************************************************/ #include "fil0fil.h" #include "fil0pageencryption.h" @@ -19,39 +19,33 @@ Created 08/25/2014 Florin Fugaciu #include "buf0checksum.h" #include -#include - //#define UNIV_PAGEENCRIPTION_DEBUG //#define CRYPT_FF /* calculate a checksum to verify decryption */ -ulint fil_page_encryption_calc_checksum( - unsigned char* buf, size_t size) { +ulint fil_page_encryption_calc_checksum(unsigned char* buf, size_t size) { ulint checksum = 0; - checksum = ut_fold_binary(buf, - size); + checksum = ut_fold_binary(buf, size); checksum = checksum & 0xFFFFFFFFUL; return checksum; } /****************************************************************//** -For page encrypted pages encrypt the page before actual write -operation. -@return encrypted page to be written*/ + For page encrypted pages encrypt the page before actual write + operation. + @return encrypted page to be written*/ byte* fil_encrypt_page( /*==============*/ - ulint space_id, /*!< in: tablespace id of the table. */ - byte* buf, /*!< in: buffer from which to write; in aio - this must be appropriately aligned */ - byte* out_buf, /*!< out: encrypted buffer */ - ulint len, /*!< in: length of input buffer.*/ - ulint encryption_key,/*!< in: encryption key */ - ulint* out_len, /*!< out: actual length of encrypted page */ - ulint unit_test - ) -{ +ulint space_id, /*!< in: tablespace id of the table. */ +byte* buf, /*!< in: buffer from which to write; in aio + this must be appropriately aligned */ +byte* out_buf, /*!< out: encrypted buffer */ +ulint len, /*!< in: length of input buffer.*/ +ulint encryption_key,/*!< in: encryption key */ +ulint* out_len /*!< out: actual length of encrypted page */ +) { int err = AES_OK; int key = 0; @@ -59,26 +53,26 @@ fil_encrypt_page( ulint remainder = 0; ulint data_size = 0; ulint page_size = UNIV_PAGE_SIZE; - ulint orig_page_type=0; + ulint orig_page_type = 0; ulint write_size = 0; ib_uint64_t flush_lsn = 0; ib_uint32_t checksum = 0; - ulint page_compressed = 0; ulint offset_ctrl_data = 0; fil_space_t* space = NULL; byte* tmp_buf; - - - ut_ad(buf); ut_ad(out_buf); + ulint unit_test = 0; + ut_ad(buf);ut_ad(out_buf); key = encryption_key; +#ifndef EP_UNIT_TEST + unit_test = 1; +#endif //TODO encryption default key - /* If no encryption key was provided to this table, use system - default key - if (key == 0) { - key = 0; - }*/ - + /* If no encryption key was provided to this table, use system + default key + if (key == 0) { + key = 0; + }*/ if (!unit_test) { ut_ad(fil_space_is_page_encrypted(space_id)); @@ -86,11 +80,10 @@ fil_encrypt_page( space = fil_space_get_by_id(space_id); fil_system_exit(); - #ifdef UNIV_DEBUG - fprintf(stderr, - "InnoDB: Note: Preparing for encryption for space %lu name %s len %lu\n", - space_id, fil_space_name(space), len); + fprintf(stderr, + "InnoDB: Note: Preparing for encryption for space %lu name %s len %lu\n", + space_id, fil_space_name(space), len); #endif /* UNIV_DEBUG */ } @@ -103,78 +96,68 @@ fil_encrypt_page( remainder = (page_size - header_len - FIL_PAGE_DATA_END) - (data_size - 1); /* read bytes 1..5 of FIL_PAGE_FILE_FLUSH_LSN */ - flush_lsn = ut_ull_create(mach_read_from_3(buf + FIL_PAGE_FILE_FLUSH_LSN), mach_read_from_2(buf + FIL_PAGE_FILE_FLUSH_LSN + 3)); + flush_lsn = ut_ull_create(mach_read_from_3(buf + FIL_PAGE_FILE_FLUSH_LSN), + mach_read_from_2(buf + FIL_PAGE_FILE_FLUSH_LSN + 3)); /* read original page type */ orig_page_type = mach_read_from_2(buf + FIL_PAGE_TYPE); - if (flush_lsn!=0x0) { + if (flush_lsn != 0x0) { /** currently no support because FIL_PAGE_FILE_FLUSH_LSN field is used to store control data */ fprintf(stderr, - "InnoDB: Warning: Encryption failed for space %lu name %s. Unsupported or unknown page type.\n", - space_id, fil_space_name(space), len, err, write_size); + "InnoDB: Warning: Encryption failed for space %lu name %s. Unsupported or unknown page type.\n", + space_id, fil_space_name(space), len, err, write_size); - srv_stats.pages_page_encryption_error.inc(); - *out_len = len; + srv_stats.pages_page_encryption_error.inc(); + *out_len = len; return (buf); } /* calculate a checksum, can be used to verify decryption */ - checksum = fil_page_encryption_calc_checksum(buf + header_len, page_size - (FIL_PAGE_DATA_END + header_len)); - + checksum = fil_page_encryption_calc_checksum(buf + header_len, + page_size - (FIL_PAGE_DATA_END + header_len)); char *stringkey = "BDE472A295675CA92E0467EADBC0E023"; uint8 key_len = 16; char *stringiv = "2D1AF8D3974E0BD3EFED5A6F82594F5E"; uint8 iv_len = 16; + //AES_KEY aeskey; + //AES_set_encrypt_key(hexkey, key_len*8, &aeskey); + //AES_cbc_encrypt((uchar*)buf +header_len + offset, (uchar *)out_buf + header_len, data_size-offset, &aeskey, iv, AES_ENCRYPT); + write_size = data_size; + /* + err = my_aes_encrypt_cbc((char*)buf + header_len + offset, + data_size - offset, + (char *) out_buf + header_len, + &write_size, + stringkey, + key_len, + stringiv, + iv_len); - //AES_KEY aeskey; - //AES_set_encrypt_key(hexkey, key_len*8, &aeskey); - //AES_cbc_encrypt((uchar*)buf +header_len + offset, (uchar *)out_buf + header_len, data_size-offset, &aeskey, iv, AES_ENCRYPT); - write_size = data_size; - /* - err = my_aes_encrypt_cbc((char*)buf + header_len + offset, - data_size - offset, - (char *) out_buf + header_len, - &write_size, - stringkey, - key_len, - stringiv, - iv_len); - -*/ + */ /* 1st encryption: data_size -1 bytes starting from FIL_PAGE_DATA */ - err = my_aes_encrypt_cbc((char*)buf +header_len, - data_size-1, - (char *)out_buf + header_len, - &write_size, - stringkey, - key_len, - stringiv, - iv_len);; + err = my_aes_encrypt_cbc((char*) buf + header_len, data_size - 1, + (char *) out_buf + header_len, &write_size, stringkey, key_len, + stringiv, iv_len); + ; ut_ad(write_size == data_size); - /* copy remaining bytes to output buffer */ - memcpy(out_buf + header_len + data_size, buf + header_len + data_size -1 , remainder); - + memcpy(out_buf + header_len + data_size, buf + header_len + data_size - 1, + remainder); /* create temporary buffer for 2nd encryption */ tmp_buf = static_cast(ut_malloc(64)); /* 2nd encryption: 63 bytes from out_buf, result length is 64 bytes */ - err = my_aes_encrypt_cbc((char*)out_buf + page_size -FIL_PAGE_DATA_END -62, - 63, - (char*)tmp_buf, - &write_size, - stringkey, - key_len, - stringiv, - iv_len); + err = my_aes_encrypt_cbc( + (char*) out_buf + page_size - FIL_PAGE_DATA_END - 62, 63, + (char*) tmp_buf, &write_size, stringkey, key_len, stringiv, iv_len); ut_ad(write_size == 64); //AES_cbc_encrypt((uchar*)out_buf + page_size -FIL_PAGE_DATA_END -62, tmp_buf, 63, &aeskey, iv, AES_ENCRYPT); /* copy 62 bytes from 2nd encryption to out_buf, last 2 bytes are copied later to a header field*/ @@ -194,44 +177,37 @@ fil_encrypt_page( /* free temporary buffer */ ut_free(tmp_buf); - return (buf); } - /* Set up the page header. Copied from input buffer*/ memcpy(out_buf, buf, FIL_PAGE_DATA); - /* Set up the checksum. This is only usable to verify decryption */ mach_write_to_4(out_buf + FIL_PAGE_SPACE_OR_CHKSUM, checksum); - - /* Set up the correct page type */ mach_write_to_2(out_buf + FIL_PAGE_TYPE, FIL_PAGE_PAGE_ENCRYPTED); - /* set up the trailer. The old style checksum is not used because * in case of a compressed page, this data may be compressed or unused! */ - memcpy(out_buf + (page_size -FIL_PAGE_DATA_END), buf + (page_size - FIL_PAGE_DATA_END), FIL_PAGE_DATA_END); - + memcpy(out_buf + (page_size -FIL_PAGE_DATA_END), + buf + (page_size - FIL_PAGE_DATA_END), FIL_PAGE_DATA_END); offset_ctrl_data = page_size - FIL_PAGE_DATA_END - FIL_PAGE_FILE_FLUSH_LSN; - /* Set up the encryption key. Written to the 1st byte of FIL_PAGE_FILE_FLUSH_LSN */ - mach_write_to_1(out_buf+ page_size-FIL_PAGE_DATA_END - offset_ctrl_data, key); - + mach_write_to_1(out_buf + page_size - FIL_PAGE_DATA_END - offset_ctrl_data, + key); /* store original page type. Written to 2nd and 3rd byte of the FIL_PAGE_FILE_FLUSH_LSN */ - mach_write_to_2(out_buf+page_size-FIL_PAGE_DATA_END +1 - offset_ctrl_data, orig_page_type); + mach_write_to_2( + out_buf + page_size - FIL_PAGE_DATA_END + 1 - offset_ctrl_data, + orig_page_type); /* write remaining bytes to FIL_PAGE_FILE_FLUSH_LSN byte 4 and 5 */ - memcpy(out_buf+ page_size -FIL_PAGE_DATA_END + 3 - offset_ctrl_data, tmp_buf+ 62, 2); - - - + memcpy(out_buf+ page_size -FIL_PAGE_DATA_END + 3 - offset_ctrl_data, + tmp_buf + 62, 2); #ifdef UNIV_DEBUG /* Verify */ @@ -241,7 +217,6 @@ fil_encrypt_page( #endif /* UNIV_DEBUG */ - srv_stats.pages_page_encrypted.inc(); *out_len = page_size; @@ -251,36 +226,28 @@ fil_encrypt_page( return (out_buf); } - - /****************************************************************//** -For page encrypted pages decrypt the page after actual read -operation. -@return decrypted page */ -void -fil_decrypt_page( + For page encrypted pages decrypt the page after actual read + operation. + @return decrypted page */ +ulint fil_decrypt_page( /*================*/ - byte* page_buf, /*!< in: preallocated buffer or NULL */ - byte* buf, /*!< in/out: buffer from which to read; in aio - this must be appropriately aligned */ - ulint len, /*!< in: length of output buffer.*/ - ulint* write_size, /*!< in/out: Actual payload size of the decrypted data. */ - ulint unit_test) -{ +byte* page_buf, /*!< in: preallocated buffer or NULL */ +byte* buf, /*!< in/out: buffer from which to read; in aio + this must be appropriately aligned */ +ulint len, /*!< in: length of output buffer.*/ +ulint* write_size, /*!< in/out: Actual payload size of the decrypted data. */ +ibool* page_compressed /*!(ut_malloc(len+16)); + in_buf = static_cast(ut_malloc(len + 16)); //in_buf = static_cast(ut_malloc(UNIV_PAGE_SIZE)); } else { in_buf = page_buf; } - data_size = ((page_size - header_len - FIL_PAGE_DATA_END) / 16) * 16; remainder = (page_size - header_len - FIL_PAGE_DATA_END) - (data_size - 1); - - - #ifdef UNIV_PAGEENCRIPTION_DEBUG fprintf(stderr, "InnoDB: Note: Preparing for decrypt for len %lu\n", actual_size); fflush(stderr); #endif /* UNIV_PAGEENCRIPTION_DEBUG */ + tmp_buf = static_cast(ut_malloc(64)); + tmp_page_buf = static_cast(ut_malloc(page_size)); + memset(tmp_page_buf, 0, page_size); + char *stringkey = "BDE472A295675CA92E0467EADBC0E023"; + uint8 key_len = 16; + + char *stringiv = "2D1AF8D3974E0BD3EFED5A6F82594F5E"; + uint8 iv_len = 16; + /* 1st decryption: 64 bytes */ + /* 62 bytes from data area and 2 bytes from header are copied to temporary buffer */ + memcpy(tmp_buf, buf + page_size - FIL_PAGE_DATA_END -62, 62); + memcpy(tmp_buf + 62, buf + FIL_PAGE_FILE_FLUSH_LSN +3, 2); + err = my_aes_decrypt_cbc((const char*) tmp_buf, 64, + (char *) tmp_page_buf + page_size - FIL_PAGE_DATA_END - 62, + &tmp_write_size, stringkey, key_len, stringiv, iv_len); - tmp_buf= static_cast(ut_malloc(64)); - tmp_page_buf = static_cast(ut_malloc(page_size)); - memset(tmp_page_buf,0, page_size); - char *stringkey = "BDE472A295675CA92E0467EADBC0E023"; - uint8 key_len = 16; - - char *stringiv = "2D1AF8D3974E0BD3EFED5A6F82594F5E"; - uint8 iv_len = 16; - - /* 1st decryption: 64 bytes */ - /* 62 bytes from data area and 2 bytes from header are copied to temporary buffer */ - memcpy(tmp_buf, buf + page_size - FIL_PAGE_DATA_END -62, 62); - memcpy(tmp_buf + 62, buf + FIL_PAGE_FILE_FLUSH_LSN +3, 2); - err = my_aes_decrypt_cbc((const char*) tmp_buf, - 64, - (char *) tmp_page_buf + page_size -FIL_PAGE_DATA_END -62, - &tmp_write_size, - stringkey, - key_len, - stringiv, - iv_len - ); - - - /* If decrypt fails it means that page is corrupted or has an unknown key */ - if (err != AES_OK) { - fprintf(stderr, "InnoDB: Corruption: Page is marked as encrypted\n" - "InnoDB: but decrypt failed with error %d.\n" - "InnoDB: size %lu len %lu, key%d\n", err, data_size, - len, page_encryption_key); - fflush(stderr); - ut_error; - } + /* If decrypt fails it means that page is corrupted or has an unknown key */ + if (err != AES_OK) { + fprintf(stderr, "InnoDB: Corruption: Page is marked as encrypted\n" + "InnoDB: but decrypt failed with error %d.\n" + "InnoDB: size %lu len %lu, key %d\n", err, data_size, len, + page_encryption_key); + fflush(stderr); - ut_ad(tmp_write_size == 63); + ut_free(tmp_page_buf); + ut_free(tmp_buf); - /* copy 1st part from buf to tmp_page_buf */ - /* do not override result of 1st decryption */ - memcpy(tmp_page_buf + FIL_PAGE_DATA, buf + FIL_PAGE_DATA, data_size - 60); - memset(in_buf, 0, page_size); + // Need to free temporal buffer if no buffer was given + if (NULL == page_buf) { + ut_free(in_buf); + } + return err; + } + ut_ad(tmp_write_size == 63); - err = my_aes_decrypt_cbc((char*) tmp_page_buf + FIL_PAGE_DATA, - data_size, - (char *) in_buf + FIL_PAGE_DATA, - &tmp_write_size, - stringkey, - key_len, - stringiv, - iv_len - ); - ut_ad(tmp_write_size = data_size-1); - memcpy(in_buf + FIL_PAGE_DATA + data_size -1, tmp_page_buf + page_size - FIL_PAGE_DATA_END - 2, remainder); + /* copy 1st part from buf to tmp_page_buf */ + /* do not override result of 1st decryption */ + memcpy(tmp_page_buf + FIL_PAGE_DATA, buf + FIL_PAGE_DATA, data_size - 60); + memset(in_buf, 0, page_size); + err = my_aes_decrypt_cbc((char*) tmp_page_buf + FIL_PAGE_DATA, data_size, + (char *) in_buf + FIL_PAGE_DATA, &tmp_write_size, stringkey, + key_len, stringiv, iv_len); + ut_ad(tmp_write_size = data_size-1); + memcpy(in_buf + FIL_PAGE_DATA + data_size -1, + tmp_page_buf + page_size - FIL_PAGE_DATA_END - 2, remainder); - /* calculate a checksum to verify decryption*/ - checksum = fil_page_encryption_calc_checksum(in_buf + header_len, page_size - (FIL_PAGE_DATA_END + header_len) ); - /* compare with stored checksum */ - stored_checksum = mach_read_from_4(buf); + /* calculate a checksum to verify decryption*/ + checksum = fil_page_encryption_calc_checksum(in_buf + header_len, + page_size - (FIL_PAGE_DATA_END + header_len)); + /* compare with stored checksum */ + stored_checksum = mach_read_from_4(buf); - ut_free(tmp_page_buf); - ut_free(tmp_buf); + ut_free(tmp_page_buf); + ut_free(tmp_buf); - if (checksum != stored_checksum) { - err = PAGE_ENCRYPTION_WRONG_KEY; - } + if (checksum != stored_checksum) { + err = PAGE_ENCRYPTION_WRONG_KEY; + } /* If decrypt fails it means that page is corrupted or has an unknown key */ if (err != AES_OK) { fprintf(stderr, "InnoDB: Corruption: Page is marked as encrypted\n" "InnoDB: but decrypt failed with error %d.\n" - "InnoDB: size %lu len %lu, key%d\n", err, data_size, - len, page_encryption_key); + "InnoDB: space %lu size %lu len %lu, key %d\n", err, space_id, data_size, len, + page_encryption_key); fflush(stderr); - ut_error; + + // Need to free temporal buffer if no buffer was given + if (NULL == page_buf) { + ut_free(in_buf); + } + + return err;; } #ifdef UNIV_PAGEENCRIPTION_DEBUG - fprintf(stderr, "InnoDB: Note: Decryption succeeded for len %lu\n", len); + fprintf(stderr, "InnoDB: Note: Decryption succeeded for len %lu\n", len); fflush(stderr); -#endif /* UNIV_PAGEENCRIPTION_DEBUG */ - - +#endif /* UNIV_PAGEENCRIPTIONulint page_compressed = 0; + _DEBUG */ /* copy header */ memcpy(in_buf, buf, FIL_PAGE_DATA); /* copy trailer */ - memcpy(in_buf + page_size - FIL_PAGE_DATA_END, buf + page_size - FIL_PAGE_DATA_END, FIL_PAGE_DATA_END); + memcpy(in_buf + page_size - FIL_PAGE_DATA_END, + buf + page_size - FIL_PAGE_DATA_END, FIL_PAGE_DATA_END); /* Copy the decrypted page to the buffer pool*/ - memcpy(buf , in_buf, page_size); + memcpy(buf, in_buf, page_size); /* setting original page type */ - mach_write_to_2(buf + FIL_PAGE_TYPE , orig_page_type); - + mach_write_to_2(buf + FIL_PAGE_TYPE, orig_page_type); /* fill flush lsn bytes 1..5 with 0 because this header field was used to store control data */ - mach_write_to_4(buf+FIL_PAGE_FILE_FLUSH_LSN, 0); - mach_write_to_1(buf+FIL_PAGE_FILE_FLUSH_LSN + 4, 0); - + mach_write_to_4(buf + FIL_PAGE_FILE_FLUSH_LSN, 0); + mach_write_to_1(buf + FIL_PAGE_FILE_FLUSH_LSN + 4, 0); /* calc check sums and write to the buffer, if page was not compressed */ - if (!page_compressed) { + if (!(page_compression_flag )) { do_check_sum(page_size, buf); } else { /* page_compression uses BUF_NO_CHECKSUM_MAGIC as checksum */ - mach_write_to_4(buf+FIL_PAGE_SPACE_OR_CHKSUM, BUF_NO_CHECKSUM_MAGIC); + mach_write_to_4(buf + FIL_PAGE_SPACE_OR_CHKSUM, BUF_NO_CHECKSUM_MAGIC); } + flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + buf); + + if (!page_compression_flag) { + + page_size = fsp_flags_get_page_size(flags); + + page_num = mach_read_from_4(buf+ 4); + page_encryption_key = FSP_FLAGS_GET_PAGE_ENCRYPTION_KEY(flags); + page_encrypted = FSP_FLAGS_GET_PAGE_ENCRYPTION(flags); + page_compression_flag = FSP_FLAGS_GET_PAGE_COMPRESSION(flags); + fprintf(stderr,"Page num, page size, key, enc, compr: %lu, %lu, %lu, %lu %lu\n", page_num, page_size, page_encryption_key, page_encrypted, page_compression_flag); + } // Need to free temporal buffer if no buffer was given if (NULL == page_buf) { ut_free(in_buf); } srv_stats.pages_page_decrypted.inc(); - + return err; } - - - - -void do_check_sum( ulint page_size, byte* buf) { +void do_check_sum(ulint page_size, byte* buf) { ib_uint32_t checksum; /* recalculate check sum - from buf0flu.cc*/ switch ((srv_checksum_algorithm_t) srv_checksum_algorithm) { @@ -493,6 +472,7 @@ void do_check_sum( ulint page_size, byte* buf) { break; case SRV_CHECKSUM_ALGORITHM_NONE: case SRV_CHECKSUM_ALGORITHM_STRICT_NONE: + checksum = BUF_NO_CHECKSUM_MAGIC; break; /* no default so the compiler will emit a warning if new enum diff --git a/storage/xtradb/include/fil0pageencryption.h b/storage/xtradb/include/fil0pageencryption.h index 30b0d8ee78ba6..24bd00017c95f 100644 --- a/storage/xtradb/include/fil0pageencryption.h +++ b/storage/xtradb/include/fil0pageencryption.h @@ -1,18 +1,8 @@ /***************************************************************************** -Copyright (C) 2013 SkySQL Ab. All Rights Reserved. +Copyright (C) 2014 -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., -51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +eperi GmbH *****************************************************************************/ @@ -25,6 +15,10 @@ this program; if not, write to the Free Software Foundation, Inc., #endif #define PAGE_ENCRYPTION_WRONG_KEY 1 +#define PAGE_ENCRYPTION_WRONG_PAGE_TYPE 2 +#define PAGE_ENCRYPTION_ERROR 3 + + /******************************************************************//** @file include/fil0pageencryption.h @@ -68,14 +62,14 @@ fil_encrypt_page( byte* out_buf, /*!< out: compressed buffer */ ulint len, /*!< in: length of input buffer.*/ ulint compression_level, /*!< in: compression level */ - ulint* out_len, /*!< out: actual length of compressed page */ - ulint unit_test); + ulint* out_len /*!< out: actual length of compressed page */ + ); /****************************************************************//** For page encrypted pages decrypt the page after actual read operation. @return decrypted page */ -void +ulint fil_decrypt_page( /*================*/ byte* page_buf, /*!< in: preallocated buffer or NULL */ @@ -83,7 +77,8 @@ fil_decrypt_page( this must be appropriately aligned */ ulint len, /*!< in: length of output buffer.*/ ulint* write_size, /*!< in/out: Actual payload size of the decrypted data. */ - ulint unit_test); + ibool* page_compressed /*!page_buf2); - tmp = fil_encrypt_page(fil_node_get_space_id(slot->message1), (byte *)buf, slot->page_buf2, len, page_encryption_key, &real_len, 0); + tmp = fil_encrypt_page(fil_node_get_space_id(slot->message1), (byte *)buf, slot->page_buf2, len, page_encryption_key, &real_len); /* If encryption succeeded, set up the length and buffer */ if (tmp != buf) { @@ -5556,6 +5561,33 @@ os_aio_linux_collect( /* We have not overstepped to next segment. */ ut_a(slot->pos < end_pos); + + + /* page encryption */ + if (slot->message1 && slot->page_encryption) { + if (slot->page_buf2==NULL) { + os_slot_alloc_page_buf2(slot); + } + + ut_ad(slot->page_buf2); + + if (slot->type == OS_FILE_READ) { + if (fil_page_is_encrypted(slot->buf)) { + fil_decrypt_page(slot->page_buf2, slot->buf, slot->len, slot->write_size, NULL); + } + } else { + if (slot->page_encryption_success && + fil_page_is_encrypted(slot->page_buf2)) { + ut_ad(slot->page_encryption_page); + if (srv_use_trim && os_fallocate_failed == FALSE) { + // Deallocate unused blocks from file system + os_file_trim(slot->file, slot, slot->len); + } + } + } + } + + /* If the table is page compressed and this is read, we decompress before we annouce the read is complete. For writes, we free the compressed page. */ @@ -5585,29 +5617,6 @@ os_aio_linux_collect( } } - /* page encryption */ - if (slot->message1 && slot->page_encryption) { - if (slot->page_buf2==NULL) { - os_slot_alloc_page_buf2(slot); - } - - ut_ad(slot->page_buf2); - - if (slot->type == OS_FILE_READ) { - if (fil_page_is_encrypted(slot->buf)) { - fil_decrypt_page(slot->page_buf2, slot->buf, slot->len, slot->write_size, 0); - } - } else { - if (slot->page_encryption_success && - fil_page_is_encrypted(slot->page_buf2)) { - ut_ad(slot->page_encryption_page); - if (srv_use_trim && os_fallocate_failed == FALSE) { - // Deallocate unused blocks from file system - os_file_trim(slot->file, slot, slot->len); - } - } - } - } /* Mark this request as completed. The error handling will be done in the calling function. */ diff --git a/unittest/eperi/eperi_aes-t.c b/unittest/eperi/eperi_aes-t.c index 8f12d150cee55..98ac57af42e8e 100644 --- a/unittest/eperi/eperi_aes-t.c +++ b/unittest/eperi/eperi_aes-t.c @@ -12,33 +12,6 @@ typedef unsigned long int ibool; #include "../../storage/xtradb/include/fil0pageencryption.h" -extern byte* -fil_encrypt_page( -/*==============*/ - ulint space_id, /*!< in: tablespace id of the - table. */ - byte* buf, /*!< in: buffer from which to write; in aio - this must be appropriately aligned */ - byte* out_buf, /*!< out: compressed buffer */ - ulint len, /*!< in: length of input buffer.*/ - ulint compression_level, /*!< in: compression level */ - ulint* out_len, /*!< out: actual length of compressed page */ - ulint unit_test); - -/****************************************************************//** -For page encrypted pages decrypt the page after actual read -operation. -@return decrypted page */ -extern void -fil_decrypt_page( -/*================*/ - byte* page_buf, /*!< in: preallocated buffer or NULL */ - byte* buf, /*!< out: buffer from which to read; in aio - this must be appropriately aligned */ - ulint len, /*!< in: length of output buffer.*/ - ulint* write_size, /*!< in/out: Actual payload size of the decrypted data. */ - ulint unit_test); - diff --git a/unittest/eperi/pageenc-t.cc b/unittest/eperi/pageenc-t.cc index 456b15bcaa74c..a165d41d8c5dd 100644 --- a/unittest/eperi/pageenc-t.cc +++ b/unittest/eperi/pageenc-t.cc @@ -31,14 +31,14 @@ fil_encrypt_page( byte* out_buf, /*!< out: compressed buffer */ ulint len, /*!< in: length of input buffer.*/ ulint compression_level, /*!< in: compression level */ - ulint* out_len, /*!< out: actual length of compressed page */ - ulint unit_test); + ulint* out_len /*!< out: actual length of compressed page */ + ); /****************************************************************//** For page encrypted pages decrypt the page after actual read operation. @return decrypted page */ -extern void +extern ulint fil_decrypt_page( /*================*/ byte* page_buf, /*!< in: preallocated buffer or NULL */ @@ -46,7 +46,8 @@ fil_decrypt_page( this must be appropriately aligned */ ulint len, /*!< in: length of output buffer.*/ ulint* write_size, /*!< in/out: Actual payload size of the decrypted data. */ - ulint unit_test); + ibool* page_compressed + ); @@ -77,9 +78,9 @@ void testIt(char* filename, ulint cmp_checksum) { byte* buf = readFile(filename); byte* dest = (byte *) malloc(16384*sizeof(byte)); ulint out_len; - fil_encrypt_page(0,buf,dest,0,255,&out_len,1); + fil_encrypt_page(0,buf,dest,0,255,&out_len); - fil_decrypt_page(NULL, dest, 16384 ,NULL,1); + fil_decrypt_page(NULL, dest, 16384 ,NULL,NULL); ulint a = 0; ulint b = 0; if (cmp_checksum) { From 8831a6dbea97dbfc528365f17913ef592e366a11 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludger=20G=C3=B6ckel?= Date: Fri, 26 Sep 2014 12:46:52 +0200 Subject: [PATCH 07/70] fixed merged error --- storage/xtradb/fil/fil0pageencryption.cc | 130 ++++++++--------------- 1 file changed, 42 insertions(+), 88 deletions(-) diff --git a/storage/xtradb/fil/fil0pageencryption.cc b/storage/xtradb/fil/fil0pageencryption.cc index df8416b7a44c9..c3349e9f01727 100644 --- a/storage/xtradb/fil/fil0pageencryption.cc +++ b/storage/xtradb/fil/fil0pageencryption.cc @@ -326,75 +326,40 @@ ibool* page_compressed /*!(ut_malloc(64)); tmp_page_buf = static_cast(ut_malloc(page_size)); memset(tmp_page_buf,0, page_size); - char *stringkey = "BDE472A295675CA92E0467EADBC0E023"; - uint8 key_len = 16; - - char *stringiv = "2D1AF8D3974E0BD3EFED5A6F82594F5E"; - uint8 iv_len = 16; - - /* 1st decryption: 64 bytes */ - /* 62 bytes from data area and 2 bytes from header are copied to temporary buffer */ - memcpy(tmp_buf, buf + page_size - FIL_PAGE_DATA_END -62, 62); - memcpy(tmp_buf + 62, buf + FIL_PAGE_FILE_FLUSH_LSN +3, 2); - err = my_aes_decrypt_cbc((const char*) tmp_buf, - 64, - (char *) tmp_page_buf + page_size -FIL_PAGE_DATA_END -62, - &tmp_write_size, - stringkey, - key_len, - stringiv, - iv_len - ); - - - /* If decrypt fails it means that page is corrupted or has an unknown key */ - if (err != AES_OK) { - fprintf(stderr, "InnoDB: Corruption: Page is marked as encrypted\n" - "InnoDB: but decrypt failed with error %d.\n" - "InnoDB: size %lu len %lu, key%d\n", err, data_size, - len, page_encryption_key); - fflush(stderr); - ut_error; - } - - ut_ad(tmp_write_size == 63); - - /* copy 1st part from buf to tmp_page_buf */ - /* do not override result of 1st decryption */ - memcpy(tmp_page_buf + FIL_PAGE_DATA, buf + FIL_PAGE_DATA, data_size - 60); - memset(in_buf, 0, page_size); - - - - err = my_aes_decrypt_cbc((char*) tmp_page_buf + FIL_PAGE_DATA, - data_size, - (char *) in_buf + FIL_PAGE_DATA, - &tmp_write_size, - stringkey, - key_len, - stringiv, - iv_len - ); - ut_ad(tmp_write_size = data_size-1); - memcpy(in_buf + FIL_PAGE_DATA + data_size -1, tmp_page_buf + page_size - FIL_PAGE_DATA_END - 2, remainder); + char *stringkey = "BDE472A295675CA92E0467EADBC0E023"; + uint8 key_len = 16; + char *stringiv = "2D1AF8D3974E0BD3EFED5A6F82594F5E"; + uint8 iv_len = 16; - /* calculate a checksum to verify decryption*/ - checksum = fil_page_encryption_calc_checksum(in_buf + header_len, page_size - (FIL_PAGE_DATA_END + header_len) ); - /* compare with stored checksum */ - stored_checksum = mach_read_from_4(buf); + /* 1st decryption: 64 bytes */ + /* 62 bytes from data area and 2 bytes from header are copied to temporary buffer */ + memcpy(tmp_buf, buf + page_size - FIL_PAGE_DATA_END -62, 62); + memcpy(tmp_buf + 62, buf + FIL_PAGE_FILE_FLUSH_LSN +3, 2); + err = my_aes_decrypt_cbc((const char*) tmp_buf, + 64, + (char *) tmp_page_buf + page_size -FIL_PAGE_DATA_END -62, + &tmp_write_size, + stringkey, + key_len, + stringiv, + iv_len + ); - ut_free(tmp_page_buf); - ut_free(tmp_buf); - // Need to free temporal buffer if no buffer was given + /* If decrypt fails it means that page is corrupted or has an unknown key */ + if (err != AES_OK) { + fprintf(stderr, "InnoDB: Corruption: Page is marked as encrypted\n" + "InnoDB: but decrypt failed with error %d.\n" + "InnoDB: size %lu len %lu, key%d\n", err, data_size, + len, page_encryption_key); + fflush(stderr); if (NULL == page_buf) { ut_free(in_buf); } return err; } - ut_ad(tmp_write_size == 63); /* copy 1st part from buf to tmp_page_buf */ @@ -402,42 +367,39 @@ ibool* page_compressed /*!>>>>>> 71eac244841f8d7d56eec91853bcecb2a8d80bad ib_uint32_t checksum; /* recalculate check sum - from buf0flu.cc*/ switch ((srv_checksum_algorithm_t) srv_checksum_algorithm) { From 2e26a4e24b29fe26d1f3d80cd942205a5afa1537 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Clemens=20D=C3=B6rrh=C3=B6fer?= Date: Wed, 24 Sep 2014 17:32:47 +0200 Subject: [PATCH 08/70] Key file parser can handle encrypted keyfiles Keyfiles can now be encrypted the parser will expect a keyfile encrypted with a shared secret, using openssl commandline. i.e. 'openssl enc -aes-256-cbc -md sha1 -k secret -in keys.txt -out keys.enc' Make sure to use sha1 as message digest and aes-256-cbc. --- include/keyfile.h | 4 +- include/my_aes.h | 23 +++++-- mysys/CMakeLists.txt | 2 +- mysys/keyfile.c | 76 +++++++++++++++++++---- mysys_ssl/my_aes.cc | 59 +++++++++++------- storage/xtradb/fil/fil0pageencryption.cc | 55 +++++++--------- unittest/eperi/CMakeLists.txt | 3 + unittest/eperi/eperi_aes-t.c | 24 +++++++ unittest/eperi/eperi_keyfile-t.c | 60 ++++++++++++++---- unittest/eperi/keys.enc | Bin 0 -> 1952 bytes unittest/eperi/keys.txt | 4 ++ 11 files changed, 220 insertions(+), 90 deletions(-) create mode 100644 unittest/eperi/keys.enc diff --git a/include/keyfile.h b/include/keyfile.h index 01d9b0a1d451d..f4ee664e9d26f 100644 --- a/include/keyfile.h +++ b/include/keyfile.h @@ -7,10 +7,10 @@ struct keyentry { }; int -parseFile(FILE * fp, struct keyentry **allKeys, const int k_len); +parseFile(FILE * fp, struct keyentry **allKeys, const int k_len, const char *secret); int -parseLine(const char *line, struct keyentry *entry); +parseLine(const char *line, struct keyentry *entry, const int k_len); int isComment(char *line); diff --git a/include/my_aes.h b/include/my_aes.h index 0d4486448a1da..c73ad4e40ef95 100644 --- a/include/my_aes.h +++ b/include/my_aes.h @@ -46,8 +46,22 @@ typedef unsigned long int ulint; */ int my_aes_encrypt_cbc(const char* source, ulint source_length, char* dest, ulint *dest_length, - const char* key, uint8 key_length, - const char* iv, uint8 iv_length); + const unsigned char* key, uint8 key_length, + const unsigned char* iv, uint8 iv_length); + +/** + * Calculate key and iv from a given salt and secret as it is handled in openssl encrypted files via console + * + * SYNOPSIS + * my_Bytes_To_Key() + * @param salt [in] the given salt as extracted from the encrypted file + * @param secret [in] the given secret as String, provided by the user + * @param key [out] 32 Bytes of key are written to this pointer + * @param iv [out] 16 Bytes of iv are written to this pointer + */ +void my_bytes_to_key(const unsigned char *salt, const char *secret, unsigned char *key, unsigned char *iv); + +void my_aes_hexToUint(const char* in, unsigned char *out, int dest_length); /* my_aes_encrypt - Crypt buffer with AES encryption algorithm. @@ -74,10 +88,11 @@ int my_aes_encrypt(const char *source, int source_length, char *dest, returns - size of original data, or negative in case of error. */ + int my_aes_decrypt_cbc(const char* source, ulint source_length, char* dest, ulint *dest_length, - const char* key, uint8 key_length, - const char* iv, uint8 iv_length); + const unsigned char* key, uint8 key_length, + const unsigned char* iv, uint8 iv_length); /* my_aes_decrypt - DeCrypt buffer with AES encryption algorithm. diff --git a/mysys/CMakeLists.txt b/mysys/CMakeLists.txt index 6dff5736fddf4..21d36fa759e02 100644 --- a/mysys/CMakeLists.txt +++ b/mysys/CMakeLists.txt @@ -72,7 +72,7 @@ IF(HAVE_MLOCK) ENDIF() ADD_CONVENIENCE_LIBRARY(mysys ${MYSYS_SOURCES}) -TARGET_LINK_LIBRARIES(mysys dbug strings ${ZLIB_LIBRARY} +TARGET_LINK_LIBRARIES(mysys dbug strings mysys_ssl ${ZLIB_LIBRARY} ${LIBNSL} ${LIBM} ${LIBRT} ${LIBSOCKET} ${LIBEXECINFO}) DTRACE_INSTRUMENT(mysys) diff --git a/mysys/keyfile.c b/mysys/keyfile.c index 43ba8fbaf6b02..38b773c0ebcaa 100644 --- a/mysys/keyfile.c +++ b/mysys/keyfile.c @@ -1,9 +1,21 @@ +//Author Clemens Doerrhoefer + #include #include #include +#include #include #define E_WRONG_NUMBER_OF_MATCHES 10 +#define MAX_KEY_FILE_SIZE 1048576 +#define MAX_BUFFER_LENGTH 512 + +#define KEY_FILE_PARSE_OK 0 +#define KEY_FILE_TOO_BIG 100 +#define KEY_BUFFER_TOO_BIG 200 +#define KEY_FILE_PARSE_NULL 300 +#define KEY_FILE_TOO_MANY_KEYS 400 + int isComment(char *line) @@ -38,32 +50,67 @@ isComment(char *line) } int -parseFile(FILE * fp, struct keyentry **allKeys, const int k_len) +parseFile(FILE * fp, struct keyentry **allKeys, const int k_len, const char *secret) { + const char *MAGIC = "Salted__"; + long file_size = 0; + char *buffer, *decrypted; + char *line = NULL; if(NULL == fp) { + fprintf(stderr, "Key file not found.\n"); return 100; } - char *line = NULL; - size_t len = 0; - ssize_t read; - while(( read = getline(&line, &len, fp)) != -1 ) { + //get size of file + fseek(fp, 0L, SEEK_END); + file_size = ftell(fp); + fseek(fp, 0L, SEEK_SET); + + if(file_size > MAX_KEY_FILE_SIZE) { + return KEY_FILE_TOO_BIG; + } + + //Read file into buffer + buffer = (char*) malloc((file_size+1)*sizeof(char)); + fread(buffer, file_size, 1, fp); + + //Check for file encryption + if(memcmp(buffer, MAGIC, 8) == 0) { //If file is encrypted, decrypt it first. + unsigned char salt[8]; + unsigned char *key = malloc(32 * sizeof(char)); + unsigned char *iv = malloc(16 * sizeof(char)); + decrypted = malloc(file_size * sizeof(char)); + memcpy(&salt, buffer+8, 8); + my_bytes_to_key(&salt, secret, key, iv); + unsigned long int d_size = 0; + my_aes_decrypt_cbc(buffer + 16, file_size -16, decrypted, &d_size, key, 32, iv, 16); + memcpy(buffer, decrypted, d_size); + + free(decrypted); + free(key); + free(iv); + } + + line = strtok(buffer, "\n"); + while(line != NULL) { struct keyentry *entry = (struct keyentry*) malloc(sizeof(struct keyentry)); - if( parseLine(line, entry) == 0 && entry->id < k_len) { + if( parseLine(line, entry, k_len) == 0) { allKeys[entry->id] = entry; } + line = strtok(NULL, "\n"); } - return 0; + free(buffer); + return KEY_FILE_PARSE_OK; } int -parseLine(const char *line, struct keyentry *entry) +parseLine(const char *line, struct keyentry *entry, const int k_len) { const char *error_p; int offset; pcre *pattern = pcre_compile( - "([0-9]+);([0-9,a-f,A-F]+);([0-9,a-f,A-F]+)", + "([0-9]+);([0-9,a-f,A-F]{32});([0-9,a-f,A-F]{64}|[0-9,a-f,A-F]{48}|[0-9,a-f,A-F]{32})", 0, &error_p, &offset, @@ -73,7 +120,7 @@ parseLine(const char *line, struct keyentry *entry) fprintf(stderr, "Offset: %d\n", offset); } int m_len = (int) strlen(line); - char *buffer = (char*) malloc(400*sizeof(char)); + char *buffer = (char*) malloc(MAX_BUFFER_LENGTH*sizeof(char)); int rc,i; int ovector[30]; rc = pcre_exec( @@ -91,20 +138,25 @@ parseLine(const char *line, struct keyentry *entry) int substr_length = ovector[3] - ovector[2]; sprintf( buffer, "%.*s", substr_length, substring_start ); entry->id = atoi(buffer); + if(entry->id >= k_len) + return KEY_FILE_TOO_MANY_KEYS; substring_start = line + ovector[4]; substr_length = ovector[5] - ovector[4]; entry->iv = malloc(substr_length*sizeof(char)); + sprintf( entry->iv, "%.*s", substr_length, substring_start ); substring_start = line + ovector[6]; substr_length = ovector[7] - ovector[6]; entry->key = malloc(substr_length*sizeof(char)); sprintf( entry->key, "%.*s", substr_length, substring_start ); - return 0; } else { return E_WRONG_NUMBER_OF_MATCHES; } - return 0; + if(entry->id == NULL || entry->iv == NULL || entry->key == NULL) { + return KEY_FILE_PARSE_NULL; + } + return KEY_FILE_PARSE_OK; } diff --git a/mysys_ssl/my_aes.cc b/mysys_ssl/my_aes.cc index 7cb068df02fe8..2a792dbf6f12e 100644 --- a/mysys_ssl/my_aes.cc +++ b/mysys_ssl/my_aes.cc @@ -24,6 +24,8 @@ #elif defined(HAVE_OPENSSL) #include #include +#include +#include // Wrap C struct, to ensure resources are released. struct MyCipherCtx @@ -109,17 +111,36 @@ static int my_aes_create_key(const char *key, int key_length, uint8 *rkey) @param iv_length [in] Size of destination array. */ void -my_aes_hexToUint(const char* iv, unsigned char *dest, int dest_length) +my_aes_hexToUint(const char* in, unsigned char *out, int dest_length) { - const char *pos = iv; + const char *pos = in; int count = 0; for(count = 0; count < dest_length; count++) { - sscanf(pos, "%2hhx", &dest[count]); + sscanf(pos, "%2hhx", &out[count]); pos += 2 * sizeof(char); } } + +/** + * Calculate key and iv from a given salt and secret as it is handled in openssl encrypted files via console + * + * SYNOPSIS + * my_Bytes_To_Key() + * @param salt [in] the given salt as extracted from the encrypted file + * @param secret [in] the given secret as String, provided by the user + * @param key [out] 32 Bytes of key are written to this pointer + * @param iv [out] 16 Bytes of iv are written to this pointer + */ +void +my_bytes_to_key(const unsigned char *salt, const char *secret, unsigned char *key, unsigned char *iv) +{ + const EVP_CIPHER *type = EVP_aes_256_cbc(); + const EVP_MD *digest = EVP_sha1(); + EVP_BytesToKey(type, digest, salt, (unsigned char*) secret, strlen(secret), 1, key, iv); +} + /** Crypt buffer with AES encryption algorithm. @@ -138,18 +159,13 @@ my_aes_hexToUint(const char* iv, unsigned char *dest, int dest_length) int my_aes_encrypt_cbc(const char* source, ulint source_length, char* dest, ulint *dest_length, - const char* key, uint8 key_length, - const char* iv, uint8 iv_length) + const unsigned char* key, uint8 key_length, + const unsigned char* iv, uint8 iv_length) { #if defined(HAVE_OPENSSL) MyCipherCtx ctx; int u_len, f_len; /* The real key to be used for encryption */ - unsigned char rkey[key_length]; - my_aes_hexToUint(key, rkey, key_length); - - unsigned char riv[iv_length]; - my_aes_hexToUint(iv, riv, iv_length); const EVP_CIPHER* cipher; switch(key_length) { case 16: @@ -167,9 +183,9 @@ int my_aes_encrypt_cbc(const char* source, ulint source_length, //Initialize Encryption Engine here, default software Engine is default ENGINE *engine = NULL; - if (! EVP_EncryptInit_ex(&ctx.ctx, cipher, engine, rkey, riv)) + if (! EVP_EncryptInit_ex(&ctx.ctx, cipher, engine, key, iv)) return AES_BAD_DATA; /* Error */ - int test = EVP_CIPHER_CTX_key_length(&ctx.ctx); + EVP_CIPHER_CTX_key_length(&ctx.ctx); OPENSSL_assert(EVP_CIPHER_CTX_key_length(&ctx.ctx) == key_length); OPENSSL_assert(EVP_CIPHER_CTX_iv_length(&ctx.ctx) == iv_length); OPENSSL_assert(EVP_CIPHER_CTX_block_size(&ctx.ctx) == 16); @@ -186,20 +202,13 @@ int my_aes_encrypt_cbc(const char* source, ulint source_length, int my_aes_decrypt_cbc(const char* source, ulint source_length, char* dest, ulint *dest_length, - const char* key, uint8 key_length, - const char* iv, uint8 iv_length) + const unsigned char* key, uint8 key_length, + const unsigned char* iv, uint8 iv_length) { #if defined(HAVE_OPENSSL) MyCipherCtx ctx; int u_len, f_len; - /* The real key to be used for decryption */ - unsigned char rkey[key_length]; - my_aes_hexToUint(key, rkey, key_length); - - unsigned char riv[iv_length]; - my_aes_hexToUint(iv, riv, iv_length); - const EVP_CIPHER* cipher; switch(key_length) { case 16: @@ -217,7 +226,7 @@ int my_aes_decrypt_cbc(const char* source, ulint source_length, //Initialize Encryption Engine here, default software Engine is default ENGINE *engine = NULL; - if (! EVP_DecryptInit_ex(&ctx.ctx, cipher, engine, rkey, riv)) + if (! EVP_DecryptInit_ex(&ctx.ctx, cipher, engine, key, iv)) return AES_BAD_DATA; /* Error */ OPENSSL_assert(EVP_CIPHER_CTX_key_length(&ctx.ctx) == key_length); OPENSSL_assert(EVP_CIPHER_CTX_iv_length(&ctx.ctx) == iv_length); @@ -225,8 +234,10 @@ int my_aes_decrypt_cbc(const char* source, ulint source_length, if (! EVP_DecryptUpdate(&ctx.ctx, (unsigned char *) dest, &u_len, (unsigned char *)source, source_length)) return AES_BAD_DATA; /* Error */ - if (! EVP_DecryptFinal_ex(&ctx.ctx, (unsigned char *) dest + u_len, &f_len)) - return AES_BAD_DATA; /* Error */ + if (! EVP_DecryptFinal_ex(&ctx.ctx, (unsigned char *) dest + u_len, &f_len)) { + *dest_length = (ulint) u_len; + return AES_BAD_DATA; + } *dest_length = (ulint) (u_len + f_len); #endif return AES_OK; diff --git a/storage/xtradb/fil/fil0pageencryption.cc b/storage/xtradb/fil/fil0pageencryption.cc index b062098afa94f..e4907bfc28233 100644 --- a/storage/xtradb/fil/fil0pageencryption.cc +++ b/storage/xtradb/fil/fil0pageencryption.cc @@ -74,7 +74,7 @@ fil_encrypt_page( ulint write_size = 0; ib_uint64_t flush_lsn = 0; ib_uint32_t checksum = 0; - ulint page_compressed = 0; + ulint offset_ctrl_data = 0; fil_space_t* space = NULL; byte* tmp_buf; @@ -135,36 +135,22 @@ fil_encrypt_page( checksum = fil_page_encryption_calc_checksum(buf + header_len, page_size - (FIL_PAGE_DATA_END + header_len)); - char *stringkey = "BDE472A295675CA92E0467EADBC0E023"; + const unsigned char rkey[] = {0xbd, 0xe4, 0x72, 0xa2, 0x95, 0x67, 0x5c, 0xa9, + 0x2e, 0x04, 0x67, 0xea, 0xdb, 0xc0,0xe0, 0x23}; uint8 key_len = 16; - char *stringiv = "2D1AF8D3974E0BD3EFED5A6F82594F5E"; + const unsigned char iv[] = {0x2d, 0x1a, 0xf8, 0xd3, 0x97, 0x4e, 0x0b, 0xd3, 0xef, 0xed, + 0x5a, 0x6f, 0x82, 0x59, 0x4f,0x5e}; uint8 iv_len = 16; - - //AES_KEY aeskey; - //AES_set_encrypt_key(hexkey, key_len*8, &aeskey); - //AES_cbc_encrypt((uchar*)buf +header_len + offset, (uchar *)out_buf + header_len, data_size-offset, &aeskey, iv, AES_ENCRYPT); - write_size = data_size; - /* - err = my_aes_encrypt_cbc((char*)buf + header_len + offset, - data_size - offset, - (char *) out_buf + header_len, - &write_size, - stringkey, - key_len, - stringiv, - iv_len); - -*/ - + write_size = data_size; /* 1st encryption: data_size -1 bytes starting from FIL_PAGE_DATA */ - err = my_aes_encrypt_cbc((char*)buf +header_len, + err = my_aes_encrypt_cbc((char*)buf + header_len, data_size-1, (char *)out_buf + header_len, &write_size, - stringkey, + (const unsigned char *)&rkey, key_len, - stringiv, + (const unsigned char *)&iv, iv_len);; ut_ad(write_size == data_size); @@ -182,9 +168,9 @@ fil_encrypt_page( 63, (char*)tmp_buf, &write_size, - stringkey, + (const unsigned char *)&rkey, key_len, - stringiv, + (const unsigned char *)&iv, iv_len); ut_ad(write_size == 64); //AES_cbc_encrypt((uchar*)out_buf + page_size -FIL_PAGE_DATA_END -62, tmp_buf, 63, &aeskey, iv, AES_ENCRYPT); @@ -284,13 +270,11 @@ fil_decrypt_page( ulint data_size = 0; ulint page_size = UNIV_PAGE_SIZE; ulint orig_page_type=0; - ulint remaining_bytes = 0; + ulint header_len = FIL_PAGE_DATA; ulint remainder = 0; - ulint offset = 1; ulint offset_ctrl_data = 0; - ulint flush_lsn = 0; ulint page_compressed = 0; ulint checksum = 0; ulint stored_checksum = 0; @@ -365,12 +349,15 @@ fil_decrypt_page( tmp_buf= static_cast(ut_malloc(64)); tmp_page_buf = static_cast(ut_malloc(page_size)); memset(tmp_page_buf,0, page_size); - char *stringkey = "BDE472A295675CA92E0467EADBC0E023"; + const unsigned char rkey[] = {0xbd, 0xe4, 0x72, 0xa2, 0x95, 0x67, 0x5c, 0xa9, + 0x2e, 0x04, 0x67, 0xea, 0xdb, 0xc0,0xe0, 0x23}; uint8 key_len = 16; - char *stringiv = "2D1AF8D3974E0BD3EFED5A6F82594F5E"; + const unsigned char iv[] = {0x2d, 0x1a, 0xf8, 0xd3, 0x97, 0x4e, 0x0b, 0xd3, 0xef, 0xed, + 0x5a, 0x6f, 0x82, 0x59, 0x4f,0x5e}; uint8 iv_len = 16; + /* 1st decryption: 64 bytes */ /* 62 bytes from data area and 2 bytes from header are copied to temporary buffer */ memcpy(tmp_buf, buf + page_size - FIL_PAGE_DATA_END -62, 62); @@ -379,9 +366,9 @@ fil_decrypt_page( 64, (char *) tmp_page_buf + page_size -FIL_PAGE_DATA_END -62, &tmp_write_size, - stringkey, + (const unsigned char *)&rkey, key_len, - stringiv, + (const unsigned char *)&iv, iv_len ); ut_ad(tmp_write_size == 63); @@ -397,9 +384,9 @@ fil_decrypt_page( data_size, (char *) in_buf + FIL_PAGE_DATA, &tmp_write_size, - stringkey, + (const unsigned char *)&rkey, key_len, - stringiv, + (const unsigned char *)&iv, iv_len ); ut_ad(tmp_write_size = data_size-1); diff --git a/unittest/eperi/CMakeLists.txt b/unittest/eperi/CMakeLists.txt index 9f8745bbf7227..93cc141c15143 100644 --- a/unittest/eperi/CMakeLists.txt +++ b/unittest/eperi/CMakeLists.txt @@ -40,3 +40,6 @@ MY_ADD_TESTS(enc0enc file(COPY ${CMAKE_CURRENT_LIST_DIR}/keys.txt DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) +file(COPY + ${CMAKE_CURRENT_LIST_DIR}/keys.enc + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) diff --git a/unittest/eperi/eperi_aes-t.c b/unittest/eperi/eperi_aes-t.c index 8f12d150cee55..a37114fdec05a 100644 --- a/unittest/eperi/eperi_aes-t.c +++ b/unittest/eperi/eperi_aes-t.c @@ -345,6 +345,29 @@ void test_page_enc_dec() { ok (i==0, "in==out"); } +/* + * Test if bytes for AES Key and IV are generated in the same way as in openssl commandline. + */ +void +test_bytes_to_key() +{ + plan(2); + char salt[] = {0x0c, 0x3b, 0x72, 0x1b, 0xfe, 0x07, 0xe2, 0xb3}; + char *secret = "secret"; + char key[32]; + char iv[16]; + char keyresult[32] = {0x2E, 0xFF, 0xB7, 0x1D, 0xDB, 0x97, 0xA8, 0x3A, + 0x03, 0x5A, 0x06, 0xDF, 0xB0, 0xDD, 0x72, 0x29, + 0xA6, 0xD9, 0x1F, 0xFB, 0xE6, 0x06, 0x3B, 0x4B, + 0x81, 0x23, 0x85, 0x45, 0x71, 0x28, 0xFF, 0x1F}; + char ivresult[16] = {0x61, 0xFF, 0xC8, 0x27, 0x5B, 0x46, 0x4C, 0xBD, + 0x55, 0x82, 0x0E, 0x54, 0x8F, 0xE4, 0x44, 0xD9}; + + my_bytes_to_key(&salt, secret, &key, &iv); + + ok(memcmp(key, &keyresult, 32) == 0, "BytesToKey key generated successfully."); + ok(memcmp(iv, &ivresult, 16) == 0, "BytesToKey iv generated successfully."); +} int @@ -357,5 +380,6 @@ main(int argc __attribute__((unused)),char *argv[]) test_cbc_resultsize(); test_cbc_enc_dec(); test_wrong_key(); + test_bytes_to_key(); return 0; } diff --git a/unittest/eperi/eperi_keyfile-t.c b/unittest/eperi/eperi_keyfile-t.c index 30077490d35ae..37384ce447db1 100644 --- a/unittest/eperi/eperi_keyfile-t.c +++ b/unittest/eperi/eperi_keyfile-t.c @@ -1,5 +1,6 @@ #include #include +#include #include void @@ -18,21 +19,54 @@ printEntry(struct keyentry *entry) { printf("\nid:%d \niv:%s \nkey:%s", entry->id, entry->iv, entry->key); } +void +test_parseFile_ciphertext(){ + plan(6); + char *secret = "secret"; + struct keyentry **allKeys = (struct keyentry**) malloc( 256 * sizeof(struct keyentry*)); + FILE *fp = fopen("keys.enc", "r"); + if(0 == parseFile(fp, allKeys, 256, secret)) { + fclose(fp); + ok(allKeys[1]->id == 1, "Key id 1 is present"); + ok(!strcmp(allKeys[2]->iv,"35B2FF0795FB84BBD666DB8430CA214E"), "Testing IV value of key 2"); + ok(!strcmp(allKeys[15]->key, "B374A26A71490437AA024E4FADD5B497FDFF1A8EA6FF12F6FB65AF2720B59CCF"),"Testing key value of key 15"); + ok((allKeys[47] == 0), "Key id 47 should be null."); + ok(allKeys[255]->id == 255, "Last key inserted"); + ok((allKeys[256] == 0), "Cannot insert more keys than defined."); + ok(0 == strcmp(allKeys[4]->key, "770A8A65DA156D24EE2A093277530142770A8A65DA156D24EE2A093277530142"), "Parser ignores entries that are too long."); + } else + { + ok(0, "Cannot open testfile"); + } + free(allKeys); +} + +void +test_parseFile_plaintext() +{ + plan(6); + struct keyentry **allKeys = (struct keyentry**) malloc( 256 * sizeof(struct keyentry*)); + FILE *fp = fopen("keys.txt", "r"); + if(0 == parseFile(fp, allKeys, 256, NULL)) { + //printAll(allKeys, 256); + fclose(fp); + ok(allKeys[1]->id == 1, "Key id 1 is present"); + ok(0 == strcmp(allKeys[2]->iv,"35B2FF0795FB84BBD666DB8430CA214E"), "Testing IV value of key 2"); + ok(0 == strcmp(allKeys[15]->key, "B374A26A71490437AA024E4FADD5B497FDFF1A8EA6FF12F6FB65AF2720B59CCF"),"Testing key value of key 15"); + ok((allKeys[47] == 0), "Key id 47 should be null."); + ok(allKeys[255]->id == 255, "Last key inserted"); + ok((allKeys[256] == 0), "Cannot insert more keys than defined."); + ok(0 == strcmp(allKeys[4]->key, "770A8A65DA156D24EE2A093277530142770A8A65DA156D24EE2A093277530142"), "Parser ignores entries that are too long."); + } else { + ok(0, "Cannot open testfile"); + } + free(allKeys); +} int main(int argc __attribute__((unused)),char *argv[]) { - plan(6); - struct keyentry **allKeys = (struct keyentry**) malloc( 256 * sizeof(struct keyentry*)); - FILE *fp = fopen("keys.txt", "r"); - parseFile(fp, allKeys, 256); - //printAll(allKeys, 256); - fclose(fp); - ok(allKeys[1]->id == 1, "Key id 1 is present"); - ok(!strcmp(allKeys[2]->iv,"35B2FF0795FB84BBD666DB8430CA214E"), "Testing IV value of key 2"); - ok(!strcmp(allKeys[15]->key, "B374A26A71490437AA024E4FADD5B497FDFF1A8EA6FF12F6FB65AF2720B59CCF"),"Testing key value of key 15"); - ok((allKeys[47] == 0), "Key id 47 should be null."); - ok(allKeys[255]->id == 255, "Last key inserted"); - ok((allKeys[256] == 0), "Cannot insert more keys than defined."); - return 0; + test_parseFile_ciphertext(); + test_parseFile_plaintext(); + return 0; } diff --git a/unittest/eperi/keys.enc b/unittest/eperi/keys.enc new file mode 100644 index 0000000000000000000000000000000000000000..2774d58404cdd1e831a1e01e729b53a0bbb9d4e2 GIT binary patch literal 1952 zcmV;R2VeM8VQh3|WM5z5ibXTJ?st^^TGL>foK^@G$O*DmifRLa8$S5i3LJJ~^=my= z<=TtU1zu-k5gA-G>X<9Q_*ii}`6-~B4)|%QS|6cHVVQG?#&RDq@d#-00 zxx6#4*>tTTt}zicImm1W^KRu%cy2!gU_?%4@kQX40A~|;zoBl+i?ruo?8Q_}Y5Zem zf+}p*F;NgV^B~j&oZoL9?Wzxw0v->A(GW9O_q;Nvkp733F?kr=kqM-|xIKZtOonrJ zp-vPwToRr?S)v$>0L*(j!3!iHu&NQvFq{=sFJ3tbUzk)RaXw)Zlh#^EcP=XudCW%Y z(=3$yaYj)$&M>O`QeT5}bXfKo>UvnnZFju;)HD0V(jDrwjXZ*+swhSG#HqRFe9{SV z`j?webzB5{{m@WzU&8DwYMH#Xq3GqF(twC>6T(HE#&xZr55|OY2FARAv3Da_Hb-zI z7W!!h*gS$)3?H%6vujSd2N&nvDU`nz(XsZa2XT_lseX8xO5Kr`F|tN^Kn2iNMzJU; zHV&Kcl&4PS7yvCmVYLv=FxQJFO<6{gB&8L3bwri}wMB@|RuZ{-P2vO3kd!d{vyc{2 zY{OnRfHdhy;=DMO_#9R`a`JPqe5$%%TE{UhHfFVkpFZ!F^3ShG7WP8p5m~?er1I?Z zNn|zeBi8=p9ku5JCqARg_0&x`%2#*L?CX1`dY5qfFVne$Gr&HjPRio)RNZH2ew^${ zY<}$(b-zMhAyfJ0*O8{-H&;qo-2u0&TK9KLf}5DKOM_gIWKWLb%N@}?KR6c05u)`@|c6DM9u~HOZ=WPbhwR$-f1Xq#!1V#ydJZc zIRaP~z{e8AH&sB;wFm&yf-lt|C4D^lz#bgN{6VIv-0w)~BKGW0r?j_5Qo+~I3t(;4 zlEblCx0AEp)iCGb>r_}PdhzYQCT^K-^|mbPU$}pm=B1(kZ$ziNG*t~b9B}?As?kXdS(7dWlPnjOy>SgIt|LbcN{`kYUPa;VhMXhHPYxRDh170yT{gzMEEyOb= zIrHgd2)c#Q7|29t``=BXx@n8HLBoWaH3FC7ri~s&c*O9@z|*}LjUT9PmSTPMs8?>x zG6Gf!g^&r7k%Jo-06jXvFaq%n2FcCI?~iHY5}DMVT&D8ib}NeGWF z-wfcLj0oE}pTqPQ*JIQUY9m8sMV1|mj$!;Dps>9^Lu zQO7~+5}o5Nv2`$Q=s0T@Dgk3-JPjKc#4&I8h)@E3L(7GqZ#>c13|hCVk^GTC8fdVl zK=E#c(iT(_jw(10mg%2VN)bF6aBLzI#F9rZDb?ccIm%=|YPZL+OpIJoxrZh7sr{(t zVBOo_7%$0x@;o~l+ck9fnB^?>UcqwoB4u-dc>NCtex~--zAm-ab>;gFq!y^tJ+bi! z&o5_dv`_CSPTyDAGknCwMm|)q{TZ3-5uBZb;j8@^mczg&Qrr-V)ZkdKk7}2>K}uNp zg~rr*iwd0t4rm=}r%j{P^5iHviS{adpgU~zL60ymLBHvP-`kaap1P5mLfAQB<%T10 zsL8ZXG5xBRPiAbHzt-nwiCzt8Tx49hyOT%3Gwht4NGMs?~*|}TALLYcJXZA?4L+-76cr^i7y{tv$ zaYx>;udUNhhFBJpNlUzj0NwS?!xvHIzjpGJC~f8P{QPgG-`M$?=aj+087)sIF{erF zCDA}bp_Z)whtq#z{;PvekA~B9=-6dK!|tF;;=jd%qn3 literal 0 HcmV?d00001 diff --git a/unittest/eperi/keys.txt b/unittest/eperi/keys.txt index 9d0a279a895d3..b2608073b004f 100644 --- a/unittest/eperi/keys.txt +++ b/unittest/eperi/keys.txt @@ -9,3 +9,7 @@ 3;7E892875A52C59A3B5883z6B13C31FBD;B374A26A71490437AA024E4FADD5B497FDFF1A8EA6FF12F6FB65AF2720B59CCF 255;F5502320F8429037B8DAEF761B189D12;770A8A65DA156D24EE2A093277530142 256;F5502320F8429037B8DAEF761B189D12;770A8A65DA156D24EE2A093277530142 +4;F5502320F8429037B8DAEF761B189D12;770A8A65DA156D24EE2A093277530142770A8A65DA156D24EE2A093277530142770A8A65DA156D24EE2A093277530142770A8A65DA156D24EE2A093277530142770A8A65DA156D24EE2A093277530142770A8A65DA156D24EE2A093277530142770A8A65DA156D24EE2A093277530142770A8A65DA156D24EE2A093277530142770A8A65DA156D24EE2A093277530142770A8A65DA156D24EE2A093277530142770A8A65DA156D24EE2A093277530142770A8A65DA156D24EE2A093277530142770A8A65DA156D24EE2A093277530142770A8A65DA156D24EE2A093277530142770A8A65DA156D24EE2A093277530142770A8A65DA156D24EE2A093277530142770A8A65DA156D24EE2A093277530142770A8A65DA156D24EE2A093277530142770A8A65DA156D24EE2A093277530142770A8A65DA156D24EE2A093277530142770A8A65DA156D24EE2A093277530142770A8A65DA156D24EE2A093277530142770A8A65DA156D24EE2A093277530142770A8A65DA156D24EE2A093277530142 +5;021B0663D4DD7B54E2EBC852677E40BD;18420B5CBA31CCDFFE9716E91EB61374D05914F3ADE23E03 +6;9BF92CEA026CE732DA80821122A8CE97;966050D7777350B6FD5CCB3E5F648DA45C63BEFB6DEDDFA13443F156B7D35C84 +7;BC44D4AFD2D9FCD82A679E4DC6700D06;B5EA210C8C09EF20DB95EC584714A89F From ae6ea133547af6606973306c16687ae0a6dcdaee Mon Sep 17 00:00:00 2001 From: florinfugaciu Date: Fri, 26 Sep 2014 16:07:31 +0200 Subject: [PATCH 09/70] florin key auslesen --- sql/mysqld.cc | 13 +- storage/xtradb/CMakeLists.txt | 3 + storage/xtradb/enc/EncKeys.cc | 203 ++++++++++++++++++++++++++ storage/xtradb/enc/KeySingleton.cc | 58 ++++++++ storage/xtradb/enc/keyfile.c | 119 +++++++++++++++ storage/xtradb/handler/ha_innodb.cc | 31 +++- storage/xtradb/handler/ha_innodb.h | 4 +- storage/xtradb/include/EncKeys.h | 69 +++++++++ storage/xtradb/include/KeySingleton.h | 56 +++++++ unittest/eperi/CMakeLists.txt | 4 + unittest/eperi/EperiKeySingleton-t.cc | 73 +++++++++ unittest/eperi/EperiKeySingleton-t.h | 17 +++ unittest/eperi/my.cnf | 94 ++++++++++++ 13 files changed, 741 insertions(+), 3 deletions(-) create mode 100644 storage/xtradb/enc/EncKeys.cc create mode 100644 storage/xtradb/enc/KeySingleton.cc create mode 100644 storage/xtradb/enc/keyfile.c create mode 100644 storage/xtradb/include/EncKeys.h create mode 100644 storage/xtradb/include/KeySingleton.h create mode 100644 unittest/eperi/EperiKeySingleton-t.cc create mode 100644 unittest/eperi/EperiKeySingleton-t.h create mode 100644 unittest/eperi/my.cnf diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 0794d7fee6159..c70044d66cf39 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -14,6 +14,8 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA */ +#include "../storage/xtradb/include/KeySingleton.h" + #include "sql_plugin.h" #include "sql_priv.h" #include "unireg.h" @@ -696,7 +698,7 @@ MY_LOCALE *my_default_lc_time_names; SHOW_COMP_OPTION have_ssl, have_symlink, have_dlopen, have_query_cache; SHOW_COMP_OPTION have_geometry, have_rtree_keys; -SHOW_COMP_OPTION have_crypt, have_compress; +SHOW_COMP_OPTION have_crypt, have_datacrypt, have_compress; SHOW_COMP_OPTION have_profiling; SHOW_COMP_OPTION have_openssl; @@ -5833,6 +5835,10 @@ int mysqld_main(int argc, char **argv) mysql_cond_signal(&COND_server_started); mysql_mutex_unlock(&LOCK_server_started); + KeySingleton& ksp2 = KeySingleton::getInstance(); + struct keyentry *entry = ksp2.getKeys(2); + printf("id:%3u \tiv:%s \tkey:%s\n", entry->id, entry->iv, entry->key); + #if defined(_WIN32) || defined(HAVE_SMEM) handle_connections_methods(); #else @@ -8609,6 +8615,11 @@ static int mysql_init_variables(void) #else have_crypt=SHOW_OPTION_NO; #endif +#ifdef HAVE_DATACRYPT + have_datacrypt=SHOW_OPTION_YES; +#else + have_datacrypt=SHOW_OPTION_NO; +#endif #ifdef HAVE_COMPRESS have_compress= SHOW_OPTION_YES; #else diff --git a/storage/xtradb/CMakeLists.txt b/storage/xtradb/CMakeLists.txt index d1df187fb84d6..cf5da4061b550 100644 --- a/storage/xtradb/CMakeLists.txt +++ b/storage/xtradb/CMakeLists.txt @@ -333,6 +333,9 @@ SET(INNOBASE_SOURCES dict/dict0stats.cc dict/dict0stats_bg.cc dyn/dyn0dyn.cc + enc/EncKeys.cc + enc/KeySingleton.cc + enc/keyfile.c eval/eval0eval.cc eval/eval0proc.cc fil/fil0fil.cc diff --git a/storage/xtradb/enc/EncKeys.cc b/storage/xtradb/enc/EncKeys.cc new file mode 100644 index 0000000000000..3bd038808f3cd --- /dev/null +++ b/storage/xtradb/enc/EncKeys.cc @@ -0,0 +1,203 @@ +/* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + + +/******************************************************************//** +@file EncKeys.cc +A class to keep keys for encryption/decryption. + +Created 09/15/2014 Florin Fugaciu +***********************************************************************/ + + +#include "EncKeys.h" +#include +#include +#include + + + +EncKeys::EncKeys() { + lenKey = sizeof(keyentry); + for( int ii=0; ii < MAX_KEYS; ii++) { + keys[ii].id = ii+1; + keys[ii].iv = keys[ii].key = NULL; + } + oneKey.iv = new char[MAX_IVLEN + 1]; + oneKey.key = new char[MAX_KEYLEN + 1]; +} + + +EncKeys::~EncKeys() { + for( int ii=0; ii < MAX_KEYS; ii++) { + delete[] keys[ii].iv; keys[ii].iv = NULL; + delete[] keys[ii].key; keys[ii].key = NULL; + } + delete[] oneKey.iv; oneKey.iv = NULL; + delete[] oneKey.key; oneKey.key = NULL; +} + +bool EncKeys::initKeysThroughFile( const char *name, const char *path) { + size_t len1 = strlen(path); + size_t len2 = strlen(name); + bool ret = true, isSlash = ('/' == path[len1-1]); + char *filename = (char *) malloc( len1 + len2 + isSlash ? 1 : 2); + + sprintf( filename, "%s%s%s", path, isSlash ? "" : "/", name); + ret = parseFile( filename, 256); + free(filename); + return ret; +} + + +bool EncKeys::initKeys( const char *name, const char *url, const int initType) { + if( KEYINITTYPE_FILE == initType) // url == path && name == filename + return initKeysThroughFile( name, url); + else if( KEYINITTYPE_SERVER == initType) { + printf("Not yet implemented. I'll exit now.\n\n"); + exit(ERROR_NOINITIALIZEDKEYS); + } + return false; +} + + +keyentry *EncKeys::getKeys( int id) { + if( KEY_MIN <= id && KEY_MAX >= id) { + memcpy(&oneKey, &keys[id-1], lenKey); + return &oneKey; + } + else + return NULL; +} + + +bool EncKeys::parseFile( const char* filename, const uint k_len) +{ + char *line, *buf = line = NULL; + size_t len = 0; + ssize_t read; + bool ret = true; + + printf("Reading %s\n\n", filename); + FILE *fp = fopen( filename, "r"); + if(NULL == fp) { + printf("Could not open %s for reading. I'll exit.\n\n", filename); + return false; + } + while( -1 != (read = getline( &line, &len, fp)) ) { + line[read - 1] = '\0'; + if( true == parseLine(line) && oneKey.id > 0 && oneKey.id < k_len) { + keys[oneKey.id - 1].id = oneKey.id; + buf = (char *) calloc( len = strlen(oneKey.iv) + 1, 1); + memcpy( buf, oneKey.iv, len); + keys[oneKey.id - 1].iv = buf; + buf = (char *) calloc( len = strlen(oneKey.key) + 1, 1); + memcpy( buf, oneKey.key, len); + keys[oneKey.id - 1].key = buf; + } + else if( 0 == oneKey.id) + oneKey.id = -1; + else { + printf("Could not read key with ID = %d. Exit now!\n\n", oneKey.id); + ret = false; + } + } + free(line); + fclose(fp); + return ret; +} + +bool EncKeys::parseLine( const char *line) +{ + if( isComment(line)) { + oneKey.id = 0; + return false; + } + + const char *error_p; + int offset; + + pcre *pattern = pcre_compile( + "([0-9]+);([0-9,a-f,A-F]+);([0-9,a-f,A-F]+)", + 0, + &error_p, + &offset, + NULL); + if( NULL != error_p ) { + fprintf(stderr, "Error: %s\n", error_p); + fprintf(stderr, "Offset: %d\n", offset); + } + + char *buf = (char*) malloc(400*sizeof(char)); + sprintf( buf, "%s", line); + int rc, m_len = (int) strlen(buf); + int ovector[30]; + rc = pcre_exec( + pattern, + NULL, + buf, + m_len, + 0, + 0, + ovector, + 30 + ); + if( 4 == rc) { + char *substring_start = buf + ovector[2]; + int substr_length = ovector[3] - ovector[2]; + char buffer[4]; + sprintf( buffer, "%.*s", substr_length, substring_start ); + oneKey.id = atoi(buffer); + + substring_start = buf + ovector[4]; + substr_length = ovector[5] - ovector[4]; + sprintf( oneKey.iv, "%.*s", substr_length, substring_start ); + + substring_start = buf + ovector[6]; + substr_length = ovector[7] - ovector[6]; + sprintf( oneKey.key, "%.*s", substr_length, substring_start ); + } + else + return false; //E_WRONG_NUMBER_OF_MATCHES; + + return true; +} + +bool EncKeys::isComment( const char *line) +{ + const char *error_p; + int offset; + int m_len = (int) strlen(line); + + pcre *pattern = pcre_compile( + "\\s*#.*", + 0, + &error_p, + &offset, + NULL); + int rc, ovector[30]; + rc = pcre_exec( + pattern, + NULL, + line, + m_len, + 0, + 0, + ovector, + 30 + ); + if( 0 > rc) return false; + else return true; +} diff --git a/storage/xtradb/enc/KeySingleton.cc b/storage/xtradb/enc/KeySingleton.cc new file mode 100644 index 0000000000000..26671a1a965fa --- /dev/null +++ b/storage/xtradb/enc/KeySingleton.cc @@ -0,0 +1,58 @@ +/* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + + +/******************************************************************//** +@file KeySingleton.cc +Implementation of single pattern to keep keys for encrypting/decrypting pages. + +Created 09/13/2014 Florin Fugaciu +***********************************************************************/ + + +#include "KeySingleton.h" +#include + + +bool KeySingleton::instanceInited = false; +KeySingleton KeySingleton::theInstance; +EncKeys KeySingleton::encKeys; + + + +KeySingleton & KeySingleton::getInstance() { + if( !instanceInited) { + printf("Encryption / decryption keys were not initialized. I'll exit.\n\n"); + exit(ERROR_NOINITIALIZEDKEYS); + } + return theInstance; +} + +KeySingleton & KeySingleton::getInstance(const char *name, const char *url, const int initType) { + if(instanceInited) return theInstance; + + instanceInited = encKeys.initKeys(name, url, initType); + if( !instanceInited) { + printf("Could not initialize the encryption / decryption keys. I'll exit.\n\n"); + exit(ERROR_NOINITIALIZEDKEYS); + } + + return theInstance; +} + +keyentry *KeySingleton::getKeys(int id) { + return encKeys.getKeys(id); +} + diff --git a/storage/xtradb/enc/keyfile.c b/storage/xtradb/enc/keyfile.c new file mode 100644 index 0000000000000..7af5d00e0a766 --- /dev/null +++ b/storage/xtradb/enc/keyfile.c @@ -0,0 +1,119 @@ +#include +#include +#include +#include + +#define E_WRONG_NUMBER_OF_MATCHES 10 + +int +isComment(char *line) +{ + const char *error_p; + int offset; + int m_len = (int) strlen(line); + + pcre *pattern = pcre_compile( + "\\s*#.*", + 0, + &error_p, + &offset, + NULL); + int rc,i; + int ovector[30]; + rc = pcre_exec( + pattern, + NULL, + line, + m_len, + 0, + 0, + ovector, + 30 + ); + if(rc < 0) { + return 0; + } else { + return 1; + } +} + +/** + * Liest höchstens k_len Schüssel im Strukturarray allKeys ein + */ +int +parseFile(FILE * fp, struct keyentry **allKeys, const int k_len) +{ + if(NULL == fp) { + return 100; + } + char *line = NULL; + size_t len = 0; + ssize_t read; + struct keyentry *entry = NULL; + int skip = FALSE; + + while( -1 != (read = getline(&line, &len, fp)) ) { + if( FALSE == skip ) + entry = (struct keyentry*) malloc(sizeof(struct keyentry)); + if( 0 == parseLine(line, entry) && entry->id > 0 && entry->id < k_len) { + allKeys[entry->id] = entry; + skip = FALSE; + } + else + skip = TRUE; + } + return 0; +} + +int +parseLine(const char *line, struct keyentry *entry) +{ + const char *error_p; + int offset; + + pcre *pattern = pcre_compile( + "([0-9]+);([0-9,a-f,A-F]+);([0-9,a-f,A-F]+)", + 0, + &error_p, + &offset, + NULL); + if( error_p != NULL ) { + fprintf(stderr, "Error: %s\n", error_p); + fprintf(stderr, "Offset: %d\n", offset); + } + int m_len = (int) strlen(line); + char *buffer = (char*) malloc(400*sizeof(char)); + int rc,i; + int ovector[30]; + rc = pcre_exec( + pattern, + NULL, + line, + m_len, + 0, + 0, + ovector, + 30 + ); + if( 4 == rc && !isComment(line) ) { + char *substring_start = line + ovector[2]; + int substr_length = ovector[3] - ovector[2]; + sprintf( buffer, "%.*s", substr_length, substring_start ); + entry->id = atoi(buffer); + + substring_start = line + ovector[4]; + substr_length = ovector[5] - ovector[4]; + entry->iv = malloc(substr_length*sizeof(char)); + sprintf( entry->iv, "%.*s", substr_length, substring_start ); + + substring_start = line + ovector[6]; + substr_length = ovector[7] - ovector[6]; + entry->key = malloc(substr_length*sizeof(char)); + sprintf( entry->key, "%.*s", substr_length, substring_start ); + return 0; + } else + { + return E_WRONG_NUMBER_OF_MATCHES; + } + return 0; +} diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 7fc525717a2fd..61c53f19263e7 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -213,6 +213,11 @@ static char* innobase_disable_monitor_counter = NULL; static char* innobase_reset_monitor_counter = NULL; static char* innobase_reset_all_monitor_counter = NULL; +/* Encryption for tables and columns */ +static char* innobase_data_encryption_providername = "keys.txt";//FF +static char* innobase_data_encryption_providerurl = "/home/florin/w/cxx/build-mariadb/unittest/eperi/";//FF +static uint innobase_data_encryption_providertype = 1; // 1 == file, 2 == server + /* The highest file format being used in the database. The value can be set by user, however, it will be adjusted to the newer file format if a table of such format is created/opened. */ @@ -3405,6 +3410,12 @@ innobase_init( ut_a(DATA_MYSQL_TRUE_VARCHAR == (ulint)MYSQL_TYPE_VARCHAR); + //FF + KeySingleton& keysingleton = KeySingleton::getInstance( innobase_data_encryption_providername, + innobase_data_encryption_providerurl, innobase_data_encryption_providertype); + struct keyentry *entry = keysingleton.getKeys(1); + printf("id:%3u \tiv:%s \tkey:%s\n", entry->id, entry->iv, entry->key); + #ifndef DBUG_OFF static const char test_filename[] = "-@"; char test_tablename[sizeof test_filename @@ -19950,6 +19961,22 @@ static MYSQL_SYSVAR_BOOL(use_mtflush, srv_use_mtflush, "Use multi-threaded flush. Default FALSE.", NULL, NULL, FALSE); +static MYSQL_SYSVAR_UINT(encryption_providertype, innobase_data_encryption_providertype, + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + "Use table or column encryption / decryption. Default is 0 for no use, 1 for keyfile and 2 for keyserver.", + NULL, NULL, 1, 0, 2, 0); + +static MYSQL_SYSVAR_STR(encryption_providername, innobase_data_encryption_providername, + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + "Name of keyfile or keyserver.", + NULL, NULL, NULL); + +static MYSQL_SYSVAR_STR(encryption_providerurl, innobase_data_encryption_providerurl, + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + "Path or URL for keyfile or keyserver.", + NULL, NULL, NULL); + + static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(log_block_size), MYSQL_SYSVAR(additional_mem_pool_size), @@ -20159,7 +20186,9 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(compression_algorithm), MYSQL_SYSVAR(mtflush_threads), MYSQL_SYSVAR(use_mtflush), - + MYSQL_SYSVAR(encryption_providertype), + MYSQL_SYSVAR(encryption_providername), + MYSQL_SYSVAR(encryption_providerurl), NULL }; diff --git a/storage/xtradb/handler/ha_innodb.h b/storage/xtradb/handler/ha_innodb.h index ea752b312d831..ebc566dc78ace 100644 --- a/storage/xtradb/handler/ha_innodb.h +++ b/storage/xtradb/handler/ha_innodb.h @@ -26,6 +26,8 @@ this program; if not, write to the Free Software Foundation, Inc., #include "dict0stats.h" +#include "KeySingleton.h" + /* Structure defines translation table between mysql index and innodb index structures */ struct innodb_idx_translate_t { @@ -58,7 +60,7 @@ typedef struct st_innobase_share { /** Prebuilt structures in an InnoDB table handle used within MySQL */ struct row_prebuilt_t; -/** Engine specific table options are definined using this struct */ +/** Engine specific table options are defined using this struct */ struct ha_table_option_struct { bool page_compressed; /*!< Table is using page compression diff --git a/storage/xtradb/include/EncKeys.h b/storage/xtradb/include/EncKeys.h new file mode 100644 index 0000000000000..bb501487ba8f1 --- /dev/null +++ b/storage/xtradb/include/EncKeys.h @@ -0,0 +1,69 @@ +/* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + + +/******************************************************************//** +@file EncKeys.h +A structure and class to keep keys for encryption/decryption. + +Created 09/15/2014 Florin Fugaciu +***********************************************************************/ + +#ifndef ENCKEYS_H_ +#define ENCKEYS_H_ + +#include +#include + +#define MAX_KEYS 255 +#define KEY_MIN 1 +#define KEY_MAX MAX_KEYS + +#define MAX_KEYLEN 512 +#define MAX_IVLEN 256 + +#define KEYINITTYPE_FILE 1 +#define KEYINITTYPE_SERVER 2 + +#define ERROR_NOINITIALIZEDKEYS 1 + +#define E_WRONG_NUMBER_OF_MATCHES 10 + + +struct keyentry { + uint id; + char *iv; + char *key; +}; + + +class EncKeys { + uint lenKey; + keyentry keys[MAX_KEYS]; + keyentry oneKey; + + bool initKeysThroughFile( const char *name, const char *path); + bool isComment( const char *line); + bool parseFile( const char* filename, const uint k_len); + bool parseLine( const char *line); + +public: + EncKeys(); + virtual ~EncKeys(); + bool initKeys( const char *name, const char *url, const int initType); + keyentry *getKeys( int id); +}; + +#endif /* ENCKEYS_H_ */ diff --git a/storage/xtradb/include/KeySingleton.h b/storage/xtradb/include/KeySingleton.h new file mode 100644 index 0000000000000..d5817fca7ee17 --- /dev/null +++ b/storage/xtradb/include/KeySingleton.h @@ -0,0 +1,56 @@ +/* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + + +/******************************************************************//** +@file KeySingletonPattern.h +Implementation of single pattern to keep keys for encrypting/decrypting pages. + +Created 09/13/2014 Florin Fugaciu +***********************************************************************/ + + +#ifndef KEYSINGLETON_H_ +#define KEYSINGLETON_H_ + +#include "EncKeys.h" + + +class KeySingleton { +private: + + + static bool instanceInited; + static KeySingleton theInstance; + static EncKeys encKeys; + + // No new instance or object possible + KeySingleton() {} + + // No new instance possible through copy constructor + KeySingleton( const KeySingleton&) {} + + // No new instance possible through copy + KeySingleton & operator = (const KeySingleton&); + +public: + virtual ~KeySingleton() {} //encKeys.~EncKeys(); + static KeySingleton& getInstance(); + // Init the instance for only one time + static KeySingleton& getInstance(const char *name, const char *url, const int initType); + keyentry *getKeys(int id); +}; + +#endif /* KEYSINGLETON_H_ */ diff --git a/unittest/eperi/CMakeLists.txt b/unittest/eperi/CMakeLists.txt index 0e605c7a4e66e..9e709e11f0e79 100644 --- a/unittest/eperi/CMakeLists.txt +++ b/unittest/eperi/CMakeLists.txt @@ -30,6 +30,10 @@ MY_ADD_TESTS(eperi) MY_ADD_TESTS(eperi_aes LINK_LIBRARIES mysys_ssl dbug) +MY_ADD_TESTS(EperiKeySingleton + EXT "cc" + LINK_LIBRARIES xtradb pcre) + MY_ADD_TESTS( pageenc EXT "cc" diff --git a/unittest/eperi/EperiKeySingleton-t.cc b/unittest/eperi/EperiKeySingleton-t.cc new file mode 100644 index 0000000000000..a4a83fdeac1a3 --- /dev/null +++ b/unittest/eperi/EperiKeySingleton-t.cc @@ -0,0 +1,73 @@ +/******************************************************************//** +@file EperiKeySingleton-t.cc +Implementation of single pattern to keep keys for encrypting/decrypting pages. + +Created 09/15/2014 Florin Fugaciu +***********************************************************************/ + + +#include "EperiKeySingleton-t.h" +#include +#include +#include +#include +#include +#include + + +EperiKeySingleton::EperiKeySingleton() { +} + +EperiKeySingleton::~EperiKeySingleton() { +} + + + + +void printEntry(struct keyentry *entry, uint id) +{ + if( NULL == entry) + printf("No such keyID = %d\n", id); + else + printf("%3u. id:%3u \tiv:%s \tkey:%s\n", id, entry->id, entry->iv, entry->key); +} + +void printAll(KeySingleton& ksp, uint length) +{ + int len = (MAX_KEYS <= length ? MAX_KEYS : length); + for( int ii=1; ii<=len; ii++) + printEntry(ksp.getKeys(ii), ii); +} + + +int main() +{ +// plan(9); + printf("%s\n", "main() EperiKeySingleton.cc"); + + KeySingleton& ksp = KeySingleton::getInstance( "keys.txt", "/home/florin/w/cxx/build-mariadb/unittest/eperi/", KEYINITTYPE_FILE); + printEntry(ksp.getKeys(0), 0); + +/* + EncKeys encKeys; + encKeys.initKeys("keys.txt", "/home/florin/w/cxx/build-mariadb/unittest/eperi/", KEYINITTYPE_FILE); + printEntry(encKeys.getKeys(0), 0); +*/ + + + printAll(ksp, 256); + ok(ksp.getKeys(1)->id == 1, "Key id 1 is present"); + ok(!strcmp(ksp.getKeys(2)->iv,"35B2FF0795FB84BBD666DB8430CA214E"), "Testing IV value of key 2"); + ok(!strcmp(ksp.getKeys(15)->key, "B374A26A71490437AA024E4FADD5B497FDFF1A8EA6FF12F6FB65AF2720B59CCF"),"Testing key value of key 15"); + ok((NULL == ksp.getKeys(47)->key), "Key id 47 should be null."); + ok(ksp.getKeys(255)->id == 255, "Last possible key to insert"); + ok((NULL == ksp.getKeys(256)), "Cannot insert more keys than defined."); + + KeySingleton& ksp1 = KeySingleton::getInstance("keys.txt", "/home/florin/w/cxx/build-mariadb/unittest/eperi", KEYINITTYPE_FILE); + printEntry(ksp1.getKeys(1), 1); + + KeySingleton& ksp2 = KeySingleton::getInstance(); + printEntry(ksp2.getKeys(2), 2); + + return EXIT_SUCCESS; +} diff --git a/unittest/eperi/EperiKeySingleton-t.h b/unittest/eperi/EperiKeySingleton-t.h new file mode 100644 index 0000000000000..92939b2706c42 --- /dev/null +++ b/unittest/eperi/EperiKeySingleton-t.h @@ -0,0 +1,17 @@ +/******************************************************************//** +@file EperiKeySingleton-t.h +Implementation of single pattern to keep keys for encrypting/decrypting pages. + +Created 09/15/2014 Florin Fugaciu +***********************************************************************/ + +#ifndef EPERIKEYSINGLETON_T_H_ +#define EPERIKEYSINGLETONPATTERN_T_H_ + +class EperiKeySingleton { +public: + EperiKeySingleton(); + virtual ~EperiKeySingleton(); +}; + +#endif /* EPERIKEYSINGLETON_T_H_ */ diff --git a/unittest/eperi/my.cnf b/unittest/eperi/my.cnf new file mode 100644 index 0000000000000..09dd8d0449d2e --- /dev/null +++ b/unittest/eperi/my.cnf @@ -0,0 +1,94 @@ +# Example MySQL config file for small systems. +# +# This is for a system with little memory (<= 64M) where MySQL is only used +# from time to time and it's important that the mysqld daemon +# doesn't use much resources. +# +# MySQL programs look for option files in a set of +# locations which depend on the deployment platform. +# You can copy this option file to one of those +# locations. For information about these locations, see: +# http://dev.mysql.com/doc/mysql/en/option-files.html +# +# In this file, you can use all long options that a program supports. +# If you want to know which options a program supports, run the program +# with the "--help" option. + +# The following options will be passed to all MySQL clients +[client] +#password = your_password +port = 3306 +socket = /tmp/mysql.sock + +# Here follows entries for some specific programs + +# The MySQL server +[mysqld] +port = 3306 +socket = /tmp/mysql.sock +skip-external-locking +key_buffer_size = 16K +max_allowed_packet = 1M +table_open_cache = 4 +sort_buffer_size = 64K +read_buffer_size = 256K +read_rnd_buffer_size = 256K +net_buffer_length = 2K +thread_stack = 240K + +# Don't listen on a TCP/IP port at all. This can be a security enhancement, +# if all processes that need to connect to mysqld run on the same host. +# All interaction with mysqld must be made via Unix sockets or named pipes. +# Note that using this option without enabling named pipes on Windows +# (using the "enable-named-pipe" option) will render mysqld useless! +# +#skip-networking +server-id = 1 + +# Uncomment the following if you want to log updates +#log-bin=mysql-bin + +# binary logging format - mixed recommended +#binlog_format=mixed + +# Causes updates to non-transactional engines using statement format to be +# written directly to binary log. Before using this option make sure that +# there are no dependencies between transactional and non-transactional +# tables such as in the statement INSERT INTO t_myisam SELECT * FROM +# t_innodb; otherwise, slaves may diverge from the master. +#binlog_direct_non_transactional_updates=TRUE + +# Uncomment the following if you are using InnoDB tables +innodb_data_home_dir = /home/florin/w/cxx/build-mariadb/db/mysql/data +innodb_data_file_path = ibdata1:10M:autoextend +innodb_log_group_home_dir = /home/florin/w/cxx/build-mariadb/db/mysql/data +# You can set .._buffer_pool_size up to 50 - 80 % +# of RAM but beware of setting memory usage too high +innodb_buffer_pool_size = 16M +#innodb_additional_mem_pool_size = 2M +# Set .._log_file_size to 25 % of buffer pool size +innodb_log_file_size = 5M +innodb_log_buffer_size = 8M +innodb_flush_log_at_trx_commit = 1 +innodb_lock_wait_timeout = 50 + +#innodb_data_encryption_providertype = 1 +#innodb_data_encryption_providername = keys.txt +#innodb_data_encryption_providerurl = /home/florin/w/cxx/build-mariadb/unittest/eperi + + +[mysqldump] +quick +max_allowed_packet = 16M + +[mysql] +no-auto-rehash +# Remove the next comment character if you are not familiar with SQL +#safe-updates + +[myisamchk] +key_buffer_size = 8M +sort_buffer_size = 8M + +[mysqlhotcopy] +interactive-timeout From 579146b9dade5f649eb46a4062c14c6d8f2674f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludger=20G=C3=B6ckel?= Date: Fri, 26 Sep 2014 16:31:40 +0200 Subject: [PATCH 10/70] minor --- storage/xtradb/fil/fil0pageencryption.cc | 9 ++++++--- storage/xtradb/os/os0file.cc | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/storage/xtradb/fil/fil0pageencryption.cc b/storage/xtradb/fil/fil0pageencryption.cc index c3349e9f01727..ec540f1f26e3d 100644 --- a/storage/xtradb/fil/fil0pageencryption.cc +++ b/storage/xtradb/fil/fil0pageencryption.cc @@ -196,7 +196,8 @@ ulint* out_len /*!< out: actual length of encrypted page */ offset_ctrl_data = page_size - FIL_PAGE_DATA_END - FIL_PAGE_FILE_FLUSH_LSN; - /* Set up the encryption key. Written to the 1st byte of FIL_PAGE_FILE_FLUSH_LSN */ + /* Set up the encryption key. Written to the 1st byte of FIL_PAGE_FILE_FLUSH_LSN. This header is currently used to store data, + * this may change. */ mach_write_to_1(out_buf + page_size - FIL_PAGE_DATA_END - offset_ctrl_data, key); @@ -281,7 +282,9 @@ ibool* page_compressed /*!page_buf2)) { ut_ad(slot->page_encryption_page); if (srv_use_trim && os_fallocate_failed == FALSE) { - // Deallocate unused blocks from file system - os_file_trim(slot->file, slot, slot->len); + // Deallocate unused blocks from file system ??? + //os_file_trim(slot->file, slot, slot->len); } } } From c4e195ee394abde8ca660894826707848fa69797 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludger=20G=C3=B6ckel?= Date: Tue, 30 Sep 2014 13:21:48 +0200 Subject: [PATCH 11/70] zwischenst --- storage/xtradb/fil/fil0pageencryption.cc | 8 +++---- storage/xtradb/include/fsp0pageencryption.ic | 22 ++++++++++++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/storage/xtradb/fil/fil0pageencryption.cc b/storage/xtradb/fil/fil0pageencryption.cc index ec540f1f26e3d..61b94d29c3fd8 100644 --- a/storage/xtradb/fil/fil0pageencryption.cc +++ b/storage/xtradb/fil/fil0pageencryption.cc @@ -44,7 +44,8 @@ byte* buf, /*!< in: buffer from which to write; in aio byte* out_buf, /*!< out: encrypted buffer */ ulint len, /*!< in: length of input buffer.*/ ulint encryption_key,/*!< in: encryption key */ -ulint* out_len /*!< out: actual length of encrypted page */ +ulint* out_len, /*!< out: actual length of encrypted page */ +ulint mode ) { int err = AES_OK; @@ -64,9 +65,8 @@ ulint* out_len /*!< out: actual length of encrypted page */ ut_ad(buf);ut_ad(out_buf); key = encryption_key; -#ifndef EP_UNIT_TEST - unit_test = 1; -#endif + unit_test = 0x01 & mode; + //TODO encryption default key /* If no encryption key was provided to this table, use system default key diff --git a/storage/xtradb/include/fsp0pageencryption.ic b/storage/xtradb/include/fsp0pageencryption.ic index c86f1c371b004..8945e17dc66d9 100644 --- a/storage/xtradb/include/fsp0pageencryption.ic +++ b/storage/xtradb/include/fsp0pageencryption.ic @@ -79,6 +79,28 @@ fil_space_is_page_encrypted( return(flags); } +/*******************************************************************//** +Returns the page encryption key of the space, or 0 if the space +is not encrypted. The tablespace must be cached in the memory cache. +@return page compression level, ULINT_UNDEFINED if space not found */ +UNIV_INLINE +ulint +fil_space_get_page_encryption_key( +/*=================================*/ + ulint id) /*!< in: space id */ +{ + ulint flags; + + flags = fil_space_get_flags(id); + + if (flags && flags != ULINT_UNDEFINED) { + + return(fsp_flags_get_page_encryption_key(flags)); + } + + return(flags); +} + /********************************************************************//** Determine the tablespace is using atomic writes from dict_table_t::flags. From dbe784fabb92cc1f2a472639fe20ef32fcaaae48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludger=20G=C3=B6ckel?= Date: Tue, 30 Sep 2014 13:22:24 +0200 Subject: [PATCH 12/70] zwischens --- storage/xtradb/include/fil0pageencryption.h | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/storage/xtradb/include/fil0pageencryption.h b/storage/xtradb/include/fil0pageencryption.h index 24bd00017c95f..c01fa9325f3bd 100644 --- a/storage/xtradb/include/fil0pageencryption.h +++ b/storage/xtradb/include/fil0pageencryption.h @@ -62,7 +62,8 @@ fil_encrypt_page( byte* out_buf, /*!< out: compressed buffer */ ulint len, /*!< in: length of input buffer.*/ ulint compression_level, /*!< in: compression level */ - ulint* out_len /*!< out: actual length of compressed page */ + ulint* out_len, /*!< out: actual length of compressed page */ + ulint mode /*!< in: calling mode. Should be 0. */ ); /****************************************************************//** @@ -77,7 +78,9 @@ fil_decrypt_page( this must be appropriately aligned */ ulint len, /*!< in: length of output buffer.*/ ulint* write_size, /*!< in/out: Actual payload size of the decrypted data. */ - ibool* page_compressed /*! Date: Tue, 30 Sep 2014 17:41:03 +0200 Subject: [PATCH 13/70] working with PAGE_COMPRESSED PAGE_ENCRYPTION and no use of flush_lsn header --- sql/mysqld.cc | 6 +- storage/xtradb/fil/fil0pageencryption.cc | 331 ++++++++++++------- storage/xtradb/handler/ha_innodb.cc | 8 +- storage/xtradb/include/fil0pageencryption.h | 1 - storage/xtradb/include/fsp0pageencryption.ic | 23 -- storage/xtradb/os/os0file.cc | 8 +- unittest/eperi/CMakeLists.txt | 7 + unittest/eperi/compressed | Bin 0 -> 16384 bytes unittest/eperi/compressed_full | Bin 0 -> 16384 bytes unittest/eperi/pageenc-t.cc | 24 +- 10 files changed, 253 insertions(+), 155 deletions(-) create mode 100644 unittest/eperi/compressed create mode 100644 unittest/eperi/compressed_full diff --git a/sql/mysqld.cc b/sql/mysqld.cc index c70044d66cf39..be67882271a4c 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -5835,9 +5835,9 @@ int mysqld_main(int argc, char **argv) mysql_cond_signal(&COND_server_started); mysql_mutex_unlock(&LOCK_server_started); - KeySingleton& ksp2 = KeySingleton::getInstance(); - struct keyentry *entry = ksp2.getKeys(2); - printf("id:%3u \tiv:%s \tkey:%s\n", entry->id, entry->iv, entry->key); +// KeySingleton& ksp2 = KeySingleton::getInstance(); +// struct keyentry *entry = ksp2.getKeys(2); +// printf("id:%3u \tiv:%s \tkey:%s\n", entry->id, entry->iv, entry->key); #if defined(_WIN32) || defined(HAVE_SMEM) handle_connections_methods(); diff --git a/storage/xtradb/fil/fil0pageencryption.cc b/storage/xtradb/fil/fil0pageencryption.cc index 61b94d29c3fd8..c8b332e1c6845 100644 --- a/storage/xtradb/fil/fil0pageencryption.cc +++ b/storage/xtradb/fil/fil0pageencryption.cc @@ -14,6 +14,7 @@ #include "fil0fil.h" #include "fil0pageencryption.h" +#include "fsp0pageencryption.h" #include "my_dbug.h" #include "buf0checksum.h" @@ -23,11 +24,14 @@ //#define UNIV_PAGEENCRIPTION_DEBUG //#define CRYPT_FF -/* calculate a checksum to verify decryption */ +/* calculate a 3 byte checksum to verify decryption. One byte is needed for other things */ ulint fil_page_encryption_calc_checksum(unsigned char* buf, size_t size) { ulint checksum = 0; checksum = ut_fold_binary(buf, size); - checksum = checksum & 0xFFFFFFFFUL; + checksum = checksum & 0x0FFFFFF0UL; + checksum = checksum >> 8; + checksum = checksum & 0x00FFFFFFUL; + return checksum; } @@ -56,14 +60,17 @@ ulint mode ulint page_size = UNIV_PAGE_SIZE; ulint orig_page_type = 0; ulint write_size = 0; - ib_uint64_t flush_lsn = 0; - ib_uint32_t checksum = 0; + + ib_uint32_t checksum = 0; ulint offset_ctrl_data = 0; fil_space_t* space = NULL; - byte* tmp_buf; + byte* tmp_buf = NULL; ulint unit_test = 0; + ibool page_compressed = 0 ; ut_ad(buf);ut_ad(out_buf); key = encryption_key; + ulint offset = 0; + byte remaining_byte = 0 ; unit_test = 0x01 & mode; @@ -95,29 +102,16 @@ ulint mode /* following number of bytes are not encrypted at first */ remainder = (page_size - header_len - FIL_PAGE_DATA_END) - (data_size - 1); - /* read bytes 1..5 of FIL_PAGE_FILE_FLUSH_LSN */ - flush_lsn = ut_ull_create(mach_read_from_3(buf + FIL_PAGE_FILE_FLUSH_LSN), - mach_read_from_2(buf + FIL_PAGE_FILE_FLUSH_LSN + 3)); - /* read original page type */ orig_page_type = mach_read_from_2(buf + FIL_PAGE_TYPE); - if (flush_lsn != 0x0) { - /** currently no support because FIL_PAGE_FILE_FLUSH_LSN field is used to store control data */ - fprintf(stderr, - "InnoDB: Warning: Encryption failed for space %lu name %s. Unsupported or unknown page type.\n", - space_id, fil_space_name(space), len, err, write_size); - - srv_stats.pages_page_encryption_error.inc(); - *out_len = len; - - return (buf); + if (orig_page_type == FIL_PAGE_PAGE_COMPRESSED) { + page_compressed = 1; } /* calculate a checksum, can be used to verify decryption */ checksum = fil_page_encryption_calc_checksum(buf + header_len, page_size - (FIL_PAGE_DATA_END + header_len)); - char *stringkey = "BDE472A295675CA92E0467EADBC0E023"; uint8 key_len = 16; char *stringiv = "2D1AF8D3974E0BD3EFED5A6F82594F5E"; @@ -143,25 +137,40 @@ ulint mode err = my_aes_encrypt_cbc((char*) buf + header_len, data_size - 1, (char *) out_buf + header_len, &write_size, stringkey, key_len, stringiv, iv_len); - ; + ut_ad(write_size == data_size); + if (page_compressed) { + /* page compressed pages: only one encryption. 3 bytes remain unencrypted. 2 bytes are appended to the encrypted buffer. + * one byte is later written to the checksum header. + */ + offset = 1; + } + /* copy remaining bytes to output buffer */ memcpy(out_buf + header_len + data_size, buf + header_len + data_size - 1, - remainder); + remainder - offset); + + if (page_compressed) { + remaining_byte = mach_read_from_1(buf + header_len + data_size +1); + } else { - /* create temporary buffer for 2nd encryption */ - tmp_buf = static_cast(ut_malloc(64)); + /* create temporary buffer for 2nd encryption */ + tmp_buf = static_cast(ut_malloc(64)); - /* 2nd encryption: 63 bytes from out_buf, result length is 64 bytes */ - err = my_aes_encrypt_cbc( - (char*) out_buf + page_size - FIL_PAGE_DATA_END - 62, 63, - (char*) tmp_buf, &write_size, stringkey, key_len, stringiv, iv_len); - ut_ad(write_size == 64); - //AES_cbc_encrypt((uchar*)out_buf + page_size -FIL_PAGE_DATA_END -62, tmp_buf, 63, &aeskey, iv, AES_ENCRYPT); - /* copy 62 bytes from 2nd encryption to out_buf, last 2 bytes are copied later to a header field*/ - memcpy(out_buf + page_size - FIL_PAGE_DATA_END -62, tmp_buf, 62); + + + /* 2nd encryption: 63 bytes from out_buf, result length is 64 bytes */ + err = my_aes_encrypt_cbc( + (char*) out_buf + page_size - FIL_PAGE_DATA_END - 62, 63, + (char*) tmp_buf, &write_size, stringkey, key_len, stringiv, iv_len); + ut_ad(write_size == 64); + //AES_cbc_encrypt((uchar*)out_buf + page_size -FIL_PAGE_DATA_END -62, tmp_buf, 63, &aeskey, iv, AES_ENCRYPT); + /* copy 62 bytes from 2nd encryption to out_buf, last 2 bytes are copied later to a header field*/ + memcpy(out_buf + page_size - FIL_PAGE_DATA_END -62, tmp_buf, 62); + + } /* error handling */ if (err != AES_OK) { @@ -175,55 +184,87 @@ ulint mode *out_len = len; /* free temporary buffer */ - ut_free(tmp_buf); + if (tmp_buf!=NULL) { + ut_free(tmp_buf); + } return (buf); } + /* set up the trailer.*/ + memcpy(out_buf + (page_size -FIL_PAGE_DATA_END), + buf + (page_size - FIL_PAGE_DATA_END), FIL_PAGE_DATA_END); + + /* Set up the page header. Copied from input buffer*/ memcpy(out_buf, buf, FIL_PAGE_DATA); - /* Set up the checksum. This is only usable to verify decryption */ - mach_write_to_4(out_buf + FIL_PAGE_SPACE_OR_CHKSUM, checksum); + + ulint compressed_size = mach_read_from_2(buf+ FIL_PAGE_DATA); + /* checksum */ + if (!page_compressed) { + /* Set up the checksum. This is only usable to verify decryption */ + mach_write_to_3(out_buf + page_size - FIL_PAGE_DATA_END, checksum); + } else { + ulint pos_checksum = page_size - FIL_PAGE_DATA_END; + if (compressed_size + FIL_PAGE_DATA > pos_checksum) { + pos_checksum = compressed_size + FIL_PAGE_DATA; + if (pos_checksum > page_size - 3) { + // checksum not supported, because no space available + } else { + /* Set up the checksum. This is only usable to verify decryption */ + mach_write_to_3(out_buf + pos_checksum, checksum); + } + } else { + mach_write_to_3(out_buf + page_size - FIL_PAGE_DATA_END, checksum); + } + } /* Set up the correct page type */ mach_write_to_2(out_buf + FIL_PAGE_TYPE, FIL_PAGE_PAGE_ENCRYPTED); - /* set up the trailer. The old style checksum is not used because - * in case of a compressed page, this data may be compressed or unused! */ - memcpy(out_buf + (page_size -FIL_PAGE_DATA_END), - buf + (page_size - FIL_PAGE_DATA_END), FIL_PAGE_DATA_END); - offset_ctrl_data = page_size - FIL_PAGE_DATA_END - FIL_PAGE_FILE_FLUSH_LSN; + offset_ctrl_data = page_size - FIL_PAGE_DATA_END; + + /* checksum fields are used to store original page type, etc. + * checksum check for page encrypted pages is omitted. PAGE_COMPRESSED pages does not seem to have a + * Old-style checksum trailer, therefore this field is only used, if there is space. Payload length is expected as + * two byte value at position FIL_PAGE_DATA */ - /* Set up the encryption key. Written to the 1st byte of FIL_PAGE_FILE_FLUSH_LSN. This header is currently used to store data, - * this may change. */ + + /* Set up the encryption key. Written to the 1st byte of the checksum header field. This header is currently used to store data. */ mach_write_to_1(out_buf + page_size - FIL_PAGE_DATA_END - offset_ctrl_data, key); - /* store original page type. Written to 2nd and 3rd byte of the FIL_PAGE_FILE_FLUSH_LSN */ + /* store original page type. Written to 2nd and 3rd byte of the checksum header field */ mach_write_to_2( out_buf + page_size - FIL_PAGE_DATA_END + 1 - offset_ctrl_data, orig_page_type); - /* write remaining bytes to FIL_PAGE_FILE_FLUSH_LSN byte 4 and 5 */ - memcpy(out_buf+ page_size -FIL_PAGE_DATA_END + 3 - offset_ctrl_data, - tmp_buf + 62, 2); + /* write remaining bytes to checksum header byte 4 and old style checksum byte 4 */ + if (!page_compressed) { + memcpy(out_buf+ page_size - FIL_PAGE_DATA_END + 3 - offset_ctrl_data, + tmp_buf + 62, 1); + memcpy(out_buf + page_size - FIL_PAGE_DATA_END +3 , tmp_buf + 63, 1); + } else { + /* if page is compressed, only one byte must be placed */ + memset(out_buf+ page_size - FIL_PAGE_DATA_END + 3 - offset_ctrl_data, + remaining_byte, 1); + } #ifdef UNIV_DEBUG /* Verify */ ut_ad(fil_page_is_encrypted(out_buf)); - //ut_ad(mach_read_from_8(out_buf+FIL_PAGE_FILE_FLUSH_LSN) == FIL_PAGE_COMPRESSION_ZLIB); - #endif /* UNIV_DEBUG */ srv_stats.pages_page_encrypted.inc(); *out_len = page_size; /* free temporary buffer */ - ut_free(tmp_buf); - + if (tmp_buf!=NULL) { + ut_free(tmp_buf); + } return (out_buf); } @@ -238,11 +279,14 @@ byte* buf, /*!< in/out: buffer from which to read; in aio this must be appropriately aligned */ ulint len, /*!< in: length of output buffer.*/ ulint* write_size, /*!< in/out: Actual payload size of the decrypted data. */ -ibool* page_compressed /*! pos_checksum) { + pos_checksum = compressed_size + FIL_PAGE_DATA; + if (pos_checksum > page_size - 3) { + // checksum not supported, because no space available + } else { + /* Set up the checksum. This is only usable to verify decryption */ + mach_write_to_3(out_buf + pos_checksum, checksum); + } + } + } +al_size); fflush(stderr); #endif /* UNIV_PAGEENCRIPTION_DEBUG */ - tmp_buf= static_cast(ut_malloc(64)); - tmp_page_buf = static_cast(ut_malloc(page_size)); - memset(tmp_page_buf,0, page_size); char *stringkey = "BDE472A295675CA92E0467EADBC0E023"; - uint8 key_len = 16; + uint8 key_len = 16; - char *stringiv = "2D1AF8D3974E0BD3EFED5A6F82594F5E"; - uint8 iv_len = 16; - - /* 1st decryption: 64 bytes */ - /* 62 bytes from data area and 2 bytes from header are copied to temporary buffer */ - memcpy(tmp_buf, buf + page_size - FIL_PAGE_DATA_END -62, 62); - memcpy(tmp_buf + 62, buf + FIL_PAGE_FILE_FLUSH_LSN +3, 2); - err = my_aes_decrypt_cbc((const char*) tmp_buf, - 64, - (char *) tmp_page_buf + page_size -FIL_PAGE_DATA_END -62, - &tmp_write_size, - stringkey, - key_len, - stringiv, - iv_len - ); + char *stringiv = "2D1AF8D3974E0BD3EFED5A6F82594F5E"; + uint8 iv_len = 16; - /* If decrypt fails it means that page is corrupted or has an unknown key */ - if (err != AES_OK) { - fprintf(stderr, "InnoDB: Corruption: Page is marked as encrypted\n" - "InnoDB: but decrypt failed with error %d.\n" - "InnoDB: size %lu len %lu, key%d\n", err, data_size, - len, page_encryption_key); - fflush(stderr); - if (NULL == page_buf) { - ut_free(in_buf); + if (!page_compression_flag) { + tmp_page_buf = static_cast(ut_malloc(page_size)); + tmp_buf= static_cast(ut_malloc(64)); + memset(tmp_page_buf,0, page_size); + + /* 1st decryption: 64 bytes */ + /* 62 bytes from data area and 2 bytes from header are copied to temporary buffer */ + memcpy(tmp_buf, buf + page_size - FIL_PAGE_DATA_END -62, 62); + memcpy(tmp_buf + 62, buf + FIL_PAGE_SPACE_OR_CHKSUM + 3, 1); + memcpy(tmp_buf + 63, buf + page_size - FIL_PAGE_DATA_END +3, 1); + + err = my_aes_decrypt_cbc((const char*) tmp_buf, + 64, + (char *) tmp_page_buf + page_size -FIL_PAGE_DATA_END -62, + &tmp_write_size, + stringkey, + key_len, + stringiv, + iv_len + ); + + + /* If decrypt fails it means that page is corrupted or has an unknown key */ + if (err != AES_OK) { + fprintf(stderr, "InnoDB: Corruption: Page is marked as encrypted\n" + "InnoDB: but decrypt failed with error %d.\n" + "InnoDB: size %lu len %lu, key%d\n", err, data_size, + len, page_encryption_key); + fflush(stderr); + if (NULL == page_buf) { + ut_free(in_buf); + } + return err; } - return err; - } - ut_ad(tmp_write_size == 63); + ut_ad(tmp_write_size == 63); - /* copy 1st part from buf to tmp_page_buf */ - /* do not override result of 1st decryption */ - memcpy(tmp_page_buf + FIL_PAGE_DATA, buf + FIL_PAGE_DATA, data_size - 60); - memset(in_buf, 0, page_size); + /* copy 1st part from buf to tmp_page_buf */ + /* do not override result of 1st decryption */ + memcpy(tmp_page_buf + FIL_PAGE_DATA, buf + FIL_PAGE_DATA, data_size - 60); + memset(in_buf, 0, page_size); + } else { + tmp_page_buf = buf; + } + err = my_aes_decrypt_cbc((char*) tmp_page_buf + FIL_PAGE_DATA, data_size, (char *) in_buf + FIL_PAGE_DATA, @@ -382,25 +458,58 @@ ibool* page_compressed /*! pos_checksum) { + pos_checksum = compressed_size + FIL_PAGE_DATA; + if (pos_checksum > page_size - 3) { + // checksum not supported, because no space available + no_checksum_support = 1; + } else { + /* Read the checksum. This is only usable to verify decryption */ + stored_checksum = mach_read_from_3(buf + pos_checksum); + } + } else { + /* Read the checksum. This is only usable to verify decryption */ + stored_checksum = mach_read_from_3(buf + page_size - FIL_PAGE_DATA_END); } - return err; } + if (!page_compression_flag) { + ut_free(tmp_page_buf); + ut_free(tmp_buf); + } + if (no_checksum_support) { + fprintf(stderr, "InnoDB: decrypting page can not be verified!\n"); + fflush(stderr); + + } else { + if ((stored_checksum != checksum)) { + err = PAGE_ENCRYPTION_WRONG_KEY; + // Need to free temporal buffer if no buffer was given + if (NULL == page_buf) { + ut_free(in_buf); + } + return err; + } + } #ifdef UNIV_PAGEENCRIPTION_DEBUG @@ -423,10 +532,6 @@ ibool* page_compressed /*!id, entry->iv, entry->key); +// KeySingleton& keysingleton = KeySingleton::getInstance( innobase_data_encryption_providername, +// innobase_data_encryption_providerurl, innobase_data_encryption_providertype); +// struct keyentry *entry = keysingleton.getKeys(1); +// printf("id:%3u \tiv:%s \tkey:%s\n", entry->id, entry->iv, entry->key); #ifndef DBUG_OFF static const char test_filename[] = "-@"; diff --git a/storage/xtradb/include/fil0pageencryption.h b/storage/xtradb/include/fil0pageencryption.h index c01fa9325f3bd..c59756235735c 100644 --- a/storage/xtradb/include/fil0pageencryption.h +++ b/storage/xtradb/include/fil0pageencryption.h @@ -79,7 +79,6 @@ fil_decrypt_page( ulint len, /*!< in: length of output buffer.*/ ulint* write_size, /*!< in/out: Actual payload size of the decrypted data. */ ibool* page_compressed, /*!page_buf2); - tmp = fil_encrypt_page(fil_node_get_space_id(slot->message1), (byte *)buf, slot->page_buf2, len, page_encryption_key, &real_len); + tmp = fil_encrypt_page(fil_node_get_space_id(slot->message1), (byte *)buf, slot->page_buf2, len, page_encryption_key, &real_len, 0); /* If encryption succeeded, set up the length and buffer */ if (tmp != buf) { @@ -5573,7 +5573,7 @@ os_aio_linux_collect( if (slot->type == OS_FILE_READ) { if (fil_page_is_encrypted(slot->buf)) { - fil_decrypt_page(slot->page_buf2, slot->buf, slot->len, slot->write_size, NULL); + fil_decrypt_page(slot->page_buf2, slot->buf, slot->len, slot->write_size, NULL, 0); } } else { if (slot->page_encryption_success && diff --git a/unittest/eperi/CMakeLists.txt b/unittest/eperi/CMakeLists.txt index 9e709e11f0e79..2c110f6c48f05 100644 --- a/unittest/eperi/CMakeLists.txt +++ b/unittest/eperi/CMakeLists.txt @@ -62,4 +62,11 @@ file(COPY file(COPY ${CMAKE_CURRENT_LIST_DIR}/xaf DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) + +file(COPY + ${CMAKE_CURRENT_LIST_DIR}/compressed + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) +file(COPY + ${CMAKE_CURRENT_LIST_DIR}/compressed_full + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) \ No newline at end of file diff --git a/unittest/eperi/compressed b/unittest/eperi/compressed new file mode 100644 index 0000000000000000000000000000000000000000..9ae0e192b44449ccd200ae7ffb82dce789f4d009 GIT binary patch literal 16384 zcmeIxX-Ja+90u@LN*qq-HjgkNYug*lxT;n%)LJkN*c|9t+z-|$eqKoG>1jm`Yt z1mVx?E%y7f2*QG36I~XodG%i=RZ$Mn4*iS%s`9HJmXy*XnH`?C0ae1?)xw=rc#|fv zXIu5Sv$!v3gRp03*i?nG-Gp^}?-t3PkCExtiq28ZiC1CydZwAKH5ZikgoLEd9}~UJ z?9?@S+OeZoWTZmwVwHPiuIb>VL$}qs;livKvv}A27_fX*)1&9Mi$7&19C&%S}VLXPtxT3UFJu)Nk7Ty6@hHKp|H(H79XE7|-Zw;7CqV5zN*&?~6{ZMbXqvPUz)f z+Gja8cCM9IWtnWU14ny}z^kj2Lp zs;b_&TsnI!RpQ9pyIh9Z9%NRyAPvW@s$^dP0gsT;ZWgP8d z(2tZo!BjEtyR~M`iquyyjGa7*J!Lvf=CNGG-)6fxv6jh3dw%^3>Z>j)n_`M-+`pB1 zHZb_z%b|cUVb)sUnr>rI zXdaN4pULl%rtp7i=PEYZcr#&4di%|h`k0aNHuIAYM{+h+n*2l`3;3sf$09wf&lW7c z?tZAB>zcLU+b=FR#Fj`Wg9;Rg{@o0>MI@zK<{_}Pn^*}@Z445K5-_ky!qSv+-v9dT)l z_DRY~TwF(O(K6lOz-xL=ebL0_Qw#x(^S3h1 z2Zz0VF&Y#ZlD8hbt}*YBD`sulk^fg<^UHA{009U<00Izz00bZa0SG_<0uX=z1Rwwb z2tWV=5P$##AOHafKmY;|fB*y_009U<00Izz00bZa0SG_<0uX=z1Rwwb2tWV=5P$## MAOL}XF0euT0-$j)!vFvP literal 0 HcmV?d00001 diff --git a/unittest/eperi/pageenc-t.cc b/unittest/eperi/pageenc-t.cc index a165d41d8c5dd..7dee6e8ccd8f9 100644 --- a/unittest/eperi/pageenc-t.cc +++ b/unittest/eperi/pageenc-t.cc @@ -4,12 +4,13 @@ * Created on: 23.08.2014 * Author: florin */ -#define EP_UNIT_TEST 1 -#define UNIV_INLINE +//#define UNIV_INLINE typedef unsigned char byte; typedef unsigned long int ulint; typedef unsigned long int ibool; + + #include "pageenc-t.h" #include #include @@ -31,7 +32,8 @@ fil_encrypt_page( byte* out_buf, /*!< out: compressed buffer */ ulint len, /*!< in: length of input buffer.*/ ulint compression_level, /*!< in: compression level */ - ulint* out_len /*!< out: actual length of compressed page */ + ulint* out_len, /*!< out: actual length of compressed page */ + ulint mode /*!< in: calling mode */ ); /****************************************************************//** @@ -46,7 +48,8 @@ fil_decrypt_page( this must be appropriately aligned */ ulint len, /*!< in: length of output buffer.*/ ulint* write_size, /*!< in/out: Actual payload size of the decrypted data. */ - ibool* page_compressed + ibool* page_compressed, + ulint mode /*! Date: Thu, 2 Oct 2014 09:14:22 +0200 Subject: [PATCH 14/70] Auslesen keyfile --- include/my_aes.h | 17 +- mysys_ssl/my_aes.cc | 3 +- sql/mysqld.cc | 6 +- storage/xtradb/enc/EncKeys.cc | 410 ++++++++++++++++---------- storage/xtradb/enc/KeySingleton.cc | 13 +- storage/xtradb/handler/ha_innodb.cc | 44 ++- storage/xtradb/include/EncKeys.h | 58 ++-- storage/xtradb/include/KeySingleton.h | 10 +- unittest/eperi/EperiKeySingleton-t.cc | 120 +++----- unittest/eperi/EperiKeySingleton-t.h | 34 +-- 10 files changed, 416 insertions(+), 299 deletions(-) diff --git a/include/my_aes.h b/include/my_aes.h index c73ad4e40ef95..c3dc68ce833d6 100644 --- a/include/my_aes.h +++ b/include/my_aes.h @@ -49,6 +49,7 @@ int my_aes_encrypt_cbc(const char* source, ulint source_length, const unsigned char* key, uint8 key_length, const unsigned char* iv, uint8 iv_length); + /** * Calculate key and iv from a given salt and secret as it is handled in openssl encrypted files via console * @@ -59,9 +60,19 @@ int my_aes_encrypt_cbc(const char* source, ulint source_length, * @param key [out] 32 Bytes of key are written to this pointer * @param iv [out] 16 Bytes of iv are written to this pointer */ -void my_bytes_to_key(const unsigned char *salt, const char *secret, unsigned char *key, unsigned char *iv); - -void my_aes_hexToUint(const char* in, unsigned char *out, int dest_length); +void my_bytes_to_key(const unsigned char *salt, + const char *secret, unsigned char *key, + unsigned char *iv); +/** + Decode Hexencoded String to uint8[]. + my_aes_hexToUint() + @param iv [in] Pointer to hexadecimal encoded IV String + @param dest [out] Pointer to output uint8 array. Memory needs to be allocated by caller + @param iv_length [in] Size of destination array. + */ +void my_aes_hexToUint(const char* in, + unsigned char *out, + int dest_length); /* my_aes_encrypt - Crypt buffer with AES encryption algorithm. diff --git a/mysys_ssl/my_aes.cc b/mysys_ssl/my_aes.cc index 2a792dbf6f12e..c68b56aa7f117 100644 --- a/mysys_ssl/my_aes.cc +++ b/mysys_ssl/my_aes.cc @@ -44,6 +44,7 @@ enum encrypt_dir { MY_AES_ENCRYPT, MY_AES_DECRYPT }; /* If bad data discovered during decoding */ #define AES_BAD_DATA -1 + /** This is internal function just keeps joint code of Key generation @@ -140,7 +141,6 @@ my_bytes_to_key(const unsigned char *salt, const char *secret, unsigned char *ke const EVP_MD *digest = EVP_sha1(); EVP_BytesToKey(type, digest, salt, (unsigned char*) secret, strlen(secret), 1, key, iv); } - /** Crypt buffer with AES encryption algorithm. @@ -156,7 +156,6 @@ my_bytes_to_key(const unsigned char *salt, const char *secret, unsigned char *ke >= 0 Size of encrypted data < 0 Error */ - int my_aes_encrypt_cbc(const char* source, ulint source_length, char* dest, ulint *dest_length, const unsigned char* key, uint8 key_length, diff --git a/sql/mysqld.cc b/sql/mysqld.cc index be67882271a4c..0273742e35a5a 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -5835,9 +5835,9 @@ int mysqld_main(int argc, char **argv) mysql_cond_signal(&COND_server_started); mysql_mutex_unlock(&LOCK_server_started); -// KeySingleton& ksp2 = KeySingleton::getInstance(); -// struct keyentry *entry = ksp2.getKeys(2); -// printf("id:%3u \tiv:%s \tkey:%s\n", entry->id, entry->iv, entry->key); + KeySingleton& ksp2 = KeySingleton::getInstance(); + struct keyentry *entry = ksp2.getKeys(2); + if(entry) printf("id:%3u \tiv:%s \tkey:%s\n", entry->id, entry->iv, entry->key); #if defined(_WIN32) || defined(HAVE_SMEM) handle_connections_methods(); diff --git a/storage/xtradb/enc/EncKeys.cc b/storage/xtradb/enc/EncKeys.cc index 3bd038808f3cd..badf67eaa1326 100644 --- a/storage/xtradb/enc/EncKeys.cc +++ b/storage/xtradb/enc/EncKeys.cc @@ -1,203 +1,309 @@ /* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ /******************************************************************//** -@file EncKeys.cc -A class to keep keys for encryption/decryption. - -Created 09/15/2014 Florin Fugaciu -***********************************************************************/ + @file EncKeys.cc + A class to keep keys for encryption/decryption. + Created 09/15/2014 Florin Fugaciu & Clemens Doerrhoefer + ***********************************************************************/ #include "EncKeys.h" +#include +#include #include +#include #include #include +/* + Die Schlüsseldatei kann Fehler beinhalten. Folgende Fehler werden abgefangen: + 1. Doppelte SchlüsselIDs: + 1.1. Gleiche Schlüssel --> Meldung ausgeben mit dem Hinweis auf die Zeilennummern + 1.2. Ungleiche Schlüssel --> Meldung ausgeben mit dem Hinweis auf die Zeilennummern + und Wahl des Schlüssel mit der kleinsten Zeilennummer + 2. Schlüsseldatei ist zu groß --> Meldung ausgeben und abbrechen + 3. Fehler im Schlüssel --> Meldung ausgeben und Schlüssel auf Nicht-Vorhanden setzen. Meldungen: + 3.1. Schlüssel zu groß + 3.2. Keine Übereinstimmung des Schlüssels mit den Vorgaben + 4. Schlüsselserver noch nicht implementiert --> Meldung ausgeben mit dem Hinweis auf das + Nicht-lesen-können von verschlüsselten Tabellen und Spalten. + + Anmerkung: voerst keine Hinweise auf die Zeilennummern +*/ + +const char* EncKeys::strMAGIC = "Salted__"; +const int EncKeys::magicSize = strlen(strMAGIC); // 8 byte +const char* EncKeys::newLine = "\n"; + +const char* EncKeys::errorNoKeyId = "KeyID = %u not found or with error. Check the key and the log file.\n"; +const char* EncKeys::errorInMatches = "Wrong match of the keyID in line %u, see the template.\n"; +const char* EncKeys::errorExceedKeyFileSize = "The size of the key file %s exceeds " + "the maximum allowed of %u bytes.\n"; +const char* EncKeys::errorExceedKeySize = "The key size exceeds the maximum allowed size of %u in line %u.\n"; +const char* EncKeys::errorEqualDoubleKey = "More than one identical key with keyID = %u found" + " in lines %u and %u.\nDelete one of them in the key file.\n"; +const char* EncKeys::errorUnequalDoubleKey = "More than one not identical key with keyID = %u found" + " in lines %u and %u.\nChoose the right one and delete the other in the key file.\n" + "I'll take the key from line %u\n"; +const char* EncKeys::errorNoInitializedKey = "The key could not be initialized.\n"; +const char* EncKeys::errorNotImplemented = "Initializing keys through key server is not" + " yet implemented.\nYou can not read encrypted tables or columns\n\n"; +const char* EncKeys::errorOpenFile = "Could not open %s for reading. You can not read encrypted tables or columns.\n\n"; +const char* EncKeys::errorReadingFile = "Could not read from %s. You can not read encrypted tables or columns\n\n"; +const char* EncKeys::errorFileSize = "Could not get the file size from %s. You can not read encrypted tables or columns\n\n"; +const char* EncKeys::errorFalseFileKey = "Wrong encryption / decryption key for keyfile '%s'.\n"; + EncKeys::EncKeys() { - lenKey = sizeof(keyentry); - for( int ii=0; ii < MAX_KEYS; ii++) { - keys[ii].id = ii+1; + countKeys = keyLineInKeyFile = 0; + for (int ii = 0; ii < MAX_KEYS; ii++) { + keys[ii].id = 0; keys[ii].iv = keys[ii].key = NULL; } - oneKey.iv = new char[MAX_IVLEN + 1]; - oneKey.key = new char[MAX_KEYLEN + 1]; + oneKey = NULL; } - EncKeys::~EncKeys() { - for( int ii=0; ii < MAX_KEYS; ii++) { - delete[] keys[ii].iv; keys[ii].iv = NULL; - delete[] keys[ii].key; keys[ii].key = NULL; + for (int ii = MAX_KEYS - 1; ii >= 0 ; ii--) { + delete[] keys[ii].iv; keys[ii].iv = NULL; + delete[] keys[ii].key; keys[ii].key = NULL; + + } + delete oneKey; oneKey = NULL; +} + +bool EncKeys::initKeys(const char *name, const char *url, const int initType, const char *filekey) { + if (KEYINITTYPE_FILE == initType) { // url == path && name == filename + if(ERROR_FALSE_FILE_KEY == initKeysThroughFile(name, url, filekey)) return false; + else return true; + } + else if (KEYINITTYPE_SERVER == initType) { + printf(errorNotImplemented); } - delete[] oneKey.iv; oneKey.iv = NULL; - delete[] oneKey.key; oneKey.key = NULL; + return NO_ERROR_KEY_FILE_PARSE_OK == ERROR_KEYINITTYPE_SERVER_NOT_IMPLEMENTED; } -bool EncKeys::initKeysThroughFile( const char *name, const char *path) { +int EncKeys::initKeysThroughFile(const char *name, const char *path, const char *filekey) { size_t len1 = strlen(path); size_t len2 = strlen(name); - bool ret = true, isSlash = ('/' == path[len1-1]); - char *filename = (char *) malloc( len1 + len2 + isSlash ? 1 : 2); + bool isSlash = ('/' == path[len1 - 1]); + int ret = NO_ERROR_KEY_FILE_PARSE_OK; + char *filename = new char[len1 + len2 + isSlash ? 1 : 2]; - sprintf( filename, "%s%s%s", path, isSlash ? "" : "/", name); - ret = parseFile( filename, 256); - free(filename); + sprintf(filename, "%s%s%s", path, isSlash ? "" : "/", name); + ret = parseFile((const char *)filename, 254, filekey); + delete[] filename; filename = NULL; return ret; } - -bool EncKeys::initKeys( const char *name, const char *url, const int initType) { - if( KEYINITTYPE_FILE == initType) // url == path && name == filename - return initKeysThroughFile( name, url); - else if( KEYINITTYPE_SERVER == initType) { - printf("Not yet implemented. I'll exit now.\n\n"); - exit(ERROR_NOINITIALIZEDKEYS); +/** + * Returns a struct keyentry with the asked 'id' or NULL. + */ +keyentry *EncKeys::getKeys(int id) { + if (KEY_MIN <= id && KEY_MAX >= id && (oneKey = &keys[id - 1])->iv) + return oneKey; + else { + printf(errorNoKeyId, id); + return NULL; } - return false; } +/** + * Get the keys from the key file 'filename' and decrypt it with the key 'secret'. + * Store the keys with id smaller then 'maxKeyId' in an array of structs keyentry. + * Returns NO_ERROR_PARSE_OK or an appropriate error code. + */ +int EncKeys::parseFile(const char* filename, const uint maxKeyId, const char *secret) { + int errorCode = 0; + char *buffer = decryptFile(filename, secret, &errorCode); + + if (NO_ERROR_PARSE_OK != errorCode) return errorCode; + else errorCode = NO_ERROR_KEY_FILE_PARSE_OK; -keyentry *EncKeys::getKeys( int id) { - if( KEY_MIN <= id && KEY_MAX >= id) { - memcpy(&oneKey, &keys[id-1], lenKey); - return &oneKey; + char *line = strtok(buffer, newLine); + while ( NULL != line) { + keyLineInKeyFile++; + switch (parseLine(line, maxKeyId)) { + case NO_ERROR_PARSE_OK: + keys[oneKey->id - 1] = *oneKey; + countKeys++; + printf("Line: %u --> ", keyLineInKeyFile); printKeyEntry(oneKey->id); + break; + case ERROR_ID_TOO_BIG: + printf(errorExceedKeySize, KEY_MAX, keyLineInKeyFile); + printf(" --> %s\n", line); + errorCode = ERROR_KEY_FILE_EXCEEDS_MAX_NUMBERS_OF_KEYS; + break; + case ERROR_NOINITIALIZEDKEY: + printf(errorNoInitializedKey); + printf(" --> %s\n", line); + errorCode = ERROR_KEY_FILE_PARSE_NULL; + break; + case ERROR_WRONG_NUMBER_OF_MATCHES: + printf(errorInMatches, keyLineInKeyFile); + printf(" --> %s\n", line); + errorCode = ERROR_KEY_FILE_PARSE_NULL; + break; + case NO_ERROR_KEY_GREATER_THAN_ASKED: + printf("No asked key in line %u: %s\n", keyLineInKeyFile, line); + break; + case NO_ERROR_ISCOMMENT: + printf("Is comment in line %u: %s\n", keyLineInKeyFile, line); + default: + break; + } + line = strtok(NULL, newLine); } - else - return NULL; + + free(line); line = NULL; + delete[] buffer; buffer = NULL; + return errorCode; } +int EncKeys::parseLine(const char *line, const uint maxKeyId) { + int ret = NO_ERROR_PARSE_OK; + if (isComment(line)) + ret = NO_ERROR_ISCOMMENT; + else { + const char *error_p; + int offset; + static const pcre *pattern = pcre_compile( + "([0-9]+);([0-9,a-f,A-F]{32});([0-9,a-f,A-F]{64}|[0-9,a-f,A-F]{48}|[0-9,a-f,A-F]{32})", + 0, &error_p, &offset, NULL); + if ( NULL != error_p) + fprintf(stderr, "Error: %s\nOffset: %d\n", error_p, offset); -bool EncKeys::parseFile( const char* filename, const uint k_len) -{ - char *line, *buf = line = NULL; - size_t len = 0; - ssize_t read; - bool ret = true; + int m_len = (int) strlen(line), ovector[MAX_OFFSETS_IN_PCRE_PATTERNS]; + int rc = pcre_exec(pattern, NULL, line, m_len, 0, 0, ovector, MAX_OFFSETS_IN_PCRE_PATTERNS); + if (4 == rc) { + char lin[MAX_KEY_LINE_SIZE + 1]; + strncpy( lin, line, MAX_KEY_LINE_SIZE); + lin[MAX_KEY_LINE_SIZE] = '\0'; + char *substring_start = lin + ovector[2]; + int substr_length = ovector[3] - ovector[2]; + if (3 < substr_length) + ret = ERROR_ID_TOO_BIG; + else { + char buffer[4]; + sprintf(buffer, "%.*s", substr_length, substring_start); + uint id = atoi(buffer); + if (0 == id) ret = ERROR_NOINITIALIZEDKEY; + else if (KEY_MAX < id) ret = ERROR_ID_TOO_BIG; + else if (maxKeyId < id) ret = NO_ERROR_KEY_GREATER_THAN_ASKED; + else { + oneKey = new keyentry; + oneKey->id = id; + substring_start = lin + ovector[4]; + substr_length = ovector[5] - ovector[4]; + oneKey->iv = new char[substr_length + 1]; + sprintf(oneKey->iv, "%.*s", substr_length, substring_start); + substring_start = lin + ovector[6]; + substr_length = ovector[7] - ovector[6]; + oneKey->key = new char[substr_length + 1]; + sprintf(oneKey->key, "%.*s", substr_length, substring_start); + } + } + } + else + ret = ERROR_WRONG_NUMBER_OF_MATCHES; + } + return ret; +} +/** + * Decrypt the key file 'filename' if it is encrypted with the key 'secret'. + * Store the content of the decrypted file in 'buffer'. The buffer has to be freed + * in the calling function. + */ +char* EncKeys::decryptFile(const char* filename, const char *secret, int *errorCode) { + *errorCode = NO_ERROR_PARSE_OK; printf("Reading %s\n\n", filename); - FILE *fp = fopen( filename, "r"); - if(NULL == fp) { - printf("Could not open %s for reading. I'll exit.\n\n", filename); - return false; + FILE *fp = fopen(filename, "r"); + if (NULL == fp) { + printf(errorOpenFile, filename); + *errorCode = ERROR_OPEN_FILE; + return NULL; + } + + if (fseek(fp, 0L, SEEK_END)) { + *errorCode = ERROR_READING_FILE; + return NULL; } - while( -1 != (read = getline( &line, &len, fp)) ) { - line[read - 1] = '\0'; - if( true == parseLine(line) && oneKey.id > 0 && oneKey.id < k_len) { - keys[oneKey.id - 1].id = oneKey.id; - buf = (char *) calloc( len = strlen(oneKey.iv) + 1, 1); - memcpy( buf, oneKey.iv, len); - keys[oneKey.id - 1].iv = buf; - buf = (char *) calloc( len = strlen(oneKey.key) + 1, 1); - memcpy( buf, oneKey.key, len); - keys[oneKey.id - 1].key = buf; + long file_size = ftell(fp); // get the file size + if (MAX_KEY_FILE_SIZE < file_size) { + printf(errorExceedKeyFileSize, filename, MAX_KEY_FILE_SIZE); + *errorCode = ERROR_KEY_FILE_TOO_BIG; + return NULL; + } + else if (-1L == file_size) { + printf(errorFileSize, filename); + *errorCode = ERROR_READING_FILE; + return NULL; + } + + fseek(fp, 0L, SEEK_SET); + //Read file into buffer + char *buffer = new char[file_size + 1]; + fread(buffer, file_size, 1, fp); + buffer[file_size] = '\0'; + fclose(fp); + + //Check for file encryption + if (0 == memcmp(buffer, strMAGIC, magicSize)) { //If file is encrypted, decrypt it first. + unsigned char salt[magicSize + 1]; + unsigned char *key = new unsigned char[keySize32]; + unsigned char *iv = new unsigned char[ivSize16]; + char *decrypted = new char[file_size]; + memcpy(&salt, buffer + magicSize, magicSize); + salt[magicSize] = '\0'; + my_bytes_to_key((unsigned char *) salt, secret, key, iv); + unsigned long int d_size = 0; + int res = my_aes_decrypt_cbc(buffer + 2 * magicSize, file_size - 2 * magicSize, + decrypted, &d_size, key, keySize32, iv, ivSize16); + if(0 != res) { + *errorCode = ERROR_FALSE_FILE_KEY; + delete[] buffer; buffer = NULL; + printf(errorFalseFileKey, secret); } - else if( 0 == oneKey.id) - oneKey.id = -1; else { - printf("Could not read key with ID = %d. Exit now!\n\n", oneKey.id); - ret = false; + memcpy(buffer, decrypted, d_size); + buffer[d_size] = '\0'; } + + delete[] decrypted; decrypted = NULL; + delete[] key; key = NULL; + delete[] iv; iv = NULL; } - free(line); - fclose(fp); - return ret; + return buffer; } -bool EncKeys::parseLine( const char *line) -{ - if( isComment(line)) { - oneKey.id = 0; - return false; - } - - const char *error_p; - int offset; - - pcre *pattern = pcre_compile( - "([0-9]+);([0-9,a-f,A-F]+);([0-9,a-f,A-F]+)", - 0, - &error_p, - &offset, - NULL); - if( NULL != error_p ) { - fprintf(stderr, "Error: %s\n", error_p); - fprintf(stderr, "Offset: %d\n", offset); - } - - char *buf = (char*) malloc(400*sizeof(char)); - sprintf( buf, "%s", line); - int rc, m_len = (int) strlen(buf); - int ovector[30]; - rc = pcre_exec( - pattern, - NULL, - buf, - m_len, - 0, - 0, - ovector, - 30 - ); - if( 4 == rc) { - char *substring_start = buf + ovector[2]; - int substr_length = ovector[3] - ovector[2]; - char buffer[4]; - sprintf( buffer, "%.*s", substr_length, substring_start ); - oneKey.id = atoi(buffer); - - substring_start = buf + ovector[4]; - substr_length = ovector[5] - ovector[4]; - sprintf( oneKey.iv, "%.*s", substr_length, substring_start ); - - substring_start = buf + ovector[6]; - substr_length = ovector[7] - ovector[6]; - sprintf( oneKey.key, "%.*s", substr_length, substring_start ); - } - else - return false; //E_WRONG_NUMBER_OF_MATCHES; - - return true; +bool EncKeys::isComment(const char *line) { + const char *error_p; + int offset, m_len = (int) strlen(line), ovector[MAX_OFFSETS_IN_PCRE_PATTERNS]; + static const pcre *pattern = pcre_compile("\\s*#.*", 0, &error_p, &offset, NULL); + int rc = pcre_exec( pattern, NULL, line, m_len, 0, 0, ovector, MAX_OFFSETS_IN_PCRE_PATTERNS); + if (0 > rc) return false; + else return true; } -bool EncKeys::isComment( const char *line) + +void EncKeys::printKeyEntry( uint id) { - const char *error_p; - int offset; - int m_len = (int) strlen(line); - - pcre *pattern = pcre_compile( - "\\s*#.*", - 0, - &error_p, - &offset, - NULL); - int rc, ovector[30]; - rc = pcre_exec( - pattern, - NULL, - line, - m_len, - 0, - 0, - ovector, - 30 - ); - if( 0 > rc) return false; - else return true; + keyentry *entry = getKeys(id); + if( NULL == entry) printf("No such keyID = %u\n", id); + else printf("Key: id:%3u \tiv:%s \tkey:%s\n", entry->id, entry->iv, entry->key); } diff --git a/storage/xtradb/enc/KeySingleton.cc b/storage/xtradb/enc/KeySingleton.cc index 26671a1a965fa..5dab2a5434c31 100644 --- a/storage/xtradb/enc/KeySingleton.cc +++ b/storage/xtradb/enc/KeySingleton.cc @@ -34,19 +34,20 @@ EncKeys KeySingleton::encKeys; KeySingleton & KeySingleton::getInstance() { if( !instanceInited) { - printf("Encryption / decryption keys were not initialized. I'll exit.\n\n"); - exit(ERROR_NOINITIALIZEDKEYS); + printf("Encryption / decryption keys were not initialized. " + "You can not read encrypted tables or columns\n\n"); } return theInstance; } -KeySingleton & KeySingleton::getInstance(const char *name, const char *url, const int initType) { +KeySingleton & KeySingleton::getInstance(const char *name, const char *url, + const int initType, const char *filekey) { if(instanceInited) return theInstance; - instanceInited = encKeys.initKeys(name, url, initType); + instanceInited = encKeys.initKeys(name, url, initType, filekey); if( !instanceInited) { - printf("Could not initialize the encryption / decryption keys. I'll exit.\n\n"); - exit(ERROR_NOINITIALIZEDKEYS); + printf("Could not initialize any of the encryption / decryption keys. " + "You can not read encrypted tables or columns\n\n"); } return theInstance; diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 0b9922940434e..a0de249dfb0bc 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -214,9 +214,10 @@ static char* innobase_reset_monitor_counter = NULL; static char* innobase_reset_all_monitor_counter = NULL; /* Encryption for tables and columns */ -static char* innobase_data_encryption_providername = "keys.txt";//FF -static char* innobase_data_encryption_providerurl = "/home/florin/w/cxx/build-mariadb/unittest/eperi/";//FF -static uint innobase_data_encryption_providertype = 1; // 1 == file, 2 == server +static char* innobase_data_encryption_providername = NULL; +static char* innobase_data_encryption_providerurl = NULL; +static uint innobase_data_encryption_providertype = 0; // 1 == file, 2 == server +static char* innobase_data_encryption_filekey = NULL; /* The highest file format being used in the database. The value can be set by user, however, it will be adjusted to the newer file format if @@ -3411,10 +3412,11 @@ innobase_init( ut_a(DATA_MYSQL_TRUE_VARCHAR == (ulint)MYSQL_TYPE_VARCHAR); //FF -// KeySingleton& keysingleton = KeySingleton::getInstance( innobase_data_encryption_providername, -// innobase_data_encryption_providerurl, innobase_data_encryption_providertype); -// struct keyentry *entry = keysingleton.getKeys(1); -// printf("id:%3u \tiv:%s \tkey:%s\n", entry->id, entry->iv, entry->key); + KeySingleton& keysingleton = KeySingleton::getInstance( + innobase_data_encryption_providername, innobase_data_encryption_providerurl, + innobase_data_encryption_providertype, innobase_data_encryption_filekey); + struct keyentry *entry = keysingleton.getKeys(3); + if(entry) printf("id:%3u \tiv:%s \tkey:%s\n", entry->id, entry->iv, entry->key); #ifndef DBUG_OFF static const char test_filename[] = "-@"; @@ -3566,6 +3568,14 @@ innobase_init( srv_data_home = (innobase_data_home_dir ? innobase_data_home_dir : default_path); + printf("\ninnobase_data_home_dir = %s,\n innobase_data_encryption_providerurl = %s," + "\n Type = %u, innobase_data_encryption_providername = %s\n\n", + innobase_data_home_dir, innobase_data_encryption_providerurl + ? innobase_data_encryption_providerurl : "ist NULL", + innobase_data_encryption_providertype, innobase_data_encryption_providername + ? innobase_data_encryption_providername : "ist NULL"); +// fflush(stdout); + /* Set default InnoDB data file size to 12 MB and let it be auto-extending. Thus users can use InnoDB in >= 4.0 without having to specify any startup options. */ @@ -19961,21 +19971,26 @@ static MYSQL_SYSVAR_BOOL(use_mtflush, srv_use_mtflush, "Use multi-threaded flush. Default FALSE.", NULL, NULL, FALSE); -static MYSQL_SYSVAR_UINT(encryption_providertype, innobase_data_encryption_providertype, +static MYSQL_SYSVAR_UINT(data_encryption_providertype, innobase_data_encryption_providertype, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, "Use table or column encryption / decryption. Default is 0 for no use, 1 for keyfile and 2 for keyserver.", NULL, NULL, 1, 0, 2, 0); -static MYSQL_SYSVAR_STR(encryption_providername, innobase_data_encryption_providername, +static MYSQL_SYSVAR_STR(data_encryption_providername, innobase_data_encryption_providername, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, "Name of keyfile or keyserver.", NULL, NULL, NULL); -static MYSQL_SYSVAR_STR(encryption_providerurl, innobase_data_encryption_providerurl, +static MYSQL_SYSVAR_STR(data_encryption_providerurl, innobase_data_encryption_providerurl, PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, "Path or URL for keyfile or keyserver.", NULL, NULL, NULL); + static MYSQL_SYSVAR_STR(data_encryption_filekey, innobase_data_encryption_filekey, + PLUGIN_VAR_RQCMDARG | PLUGIN_VAR_READONLY, + "Key to encrypt / decrypt the keyfile.", + NULL, NULL, NULL); + static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(log_block_size), @@ -20186,9 +20201,10 @@ static struct st_mysql_sys_var* innobase_system_variables[]= { MYSQL_SYSVAR(compression_algorithm), MYSQL_SYSVAR(mtflush_threads), MYSQL_SYSVAR(use_mtflush), - MYSQL_SYSVAR(encryption_providertype), - MYSQL_SYSVAR(encryption_providername), - MYSQL_SYSVAR(encryption_providerurl), + MYSQL_SYSVAR(data_encryption_providertype), + MYSQL_SYSVAR(data_encryption_providername), + MYSQL_SYSVAR(data_encryption_providerurl), + MYSQL_SYSVAR(data_encryption_filekey), NULL }; @@ -20198,7 +20214,7 @@ maria_declare_plugin(xtradb) &innobase_storage_engine, innobase_hton_name, plugin_author, - "Percona-XtraDB, Supports transactions, row-level locking, and foreign keys", + "Percona-XtraDB, Supports transactions, row-level locking, foreign keys and encryption for tables and columns", PLUGIN_LICENSE_GPL, innobase_init, /* Plugin Init */ NULL, /* Plugin Deinit */ diff --git a/storage/xtradb/include/EncKeys.h b/storage/xtradb/include/EncKeys.h index bb501487ba8f1..36d4523f16639 100644 --- a/storage/xtradb/include/EncKeys.h +++ b/storage/xtradb/include/EncKeys.h @@ -27,42 +27,52 @@ Created 09/15/2014 Florin Fugaciu #include #include -#define MAX_KEYS 255 -#define KEY_MIN 1 -#define KEY_MAX MAX_KEYS -#define MAX_KEYLEN 512 -#define MAX_IVLEN 256 - -#define KEYINITTYPE_FILE 1 -#define KEYINITTYPE_SERVER 2 - -#define ERROR_NOINITIALIZEDKEYS 1 +struct keyentry { + uint id; + char *iv; + char *key; +}; -#define E_WRONG_NUMBER_OF_MATCHES 10 +class EncKeys +{ +private: + static const char *strMAGIC, *newLine; + static const int magicSize; -struct keyentry { - uint id; - char *iv; - char *key; -}; + enum constants { MAX_OFFSETS_IN_PCRE_PATTERNS = 30 }; + enum keyAttributes { KEY_MIN = 1, KEY_MAX = 255, MAX_KEYS = 255, + MAX_IVLEN = 256, MAX_KEYLEN = 512, ivSize16 = 16, keySize32 = 32 }; + enum keyInitType { KEYINITTYPE_FILE = 1, KEYINITTYPE_SERVER = 2 }; + enum errorAttributes { MAX_KEY_LINE_SIZE = 3 * MAX_KEYLEN, MAX_KEY_FILE_SIZE = 1048576 }; + enum errorCodesLine { NO_ERROR_PARSE_OK = 0, NO_ERROR_ISCOMMENT = 10, NO_ERROR_KEY_GREATER_THAN_ASKED = 20, + ERROR_NOINITIALIZEDKEY = 30, ERROR_ID_TOO_BIG = 40, ERROR_WRONG_NUMBER_OF_MATCHES = 50, + ERROR_EQUAL_DOUBLE_KEY = 60, ERROR_UNEQUAL_DOUBLE_KEY = 70 }; + static const char *errorNoKeyId, *errorInMatches, *errorExceedKeyFileSize, + *errorExceedKeySize, *errorEqualDoubleKey, *errorUnequalDoubleKey, + *errorNoInitializedKey, *errorFalseFileKey, + *errorNotImplemented, *errorOpenFile, *errorReadingFile, *errorFileSize; -class EncKeys { - uint lenKey; - keyentry keys[MAX_KEYS]; - keyentry oneKey; + uint countKeys, keyLineInKeyFile; + keyentry keys[MAX_KEYS], *oneKey; - bool initKeysThroughFile( const char *name, const char *path); + void printKeyEntry( uint id); + int initKeysThroughFile( const char *name, const char *path, const char *filekey); bool isComment( const char *line); - bool parseFile( const char* filename, const uint k_len); - bool parseLine( const char *line); + char * decryptFile( const char* filename, const char *secret, int *errorCode); + int parseFile( const char* filename, const uint maxKeyId, const char *secret); + int parseLine( const char *line, const uint maxKeyId); public: + enum errorCodesFile { NO_ERROR_KEY_FILE_PARSE_OK = 0, ERROR_KEY_FILE_PARSE_NULL = 110, + ERROR_KEY_FILE_TOO_BIG = 120, ERROR_KEY_FILE_EXCEEDS_MAX_NUMBERS_OF_KEYS = 130, + ERROR_OPEN_FILE = 140, ERROR_READING_FILE = 150, ERROR_FALSE_FILE_KEY = 160, + ERROR_KEYINITTYPE_SERVER_NOT_IMPLEMENTED = 170 }; EncKeys(); virtual ~EncKeys(); - bool initKeys( const char *name, const char *url, const int initType); + bool initKeys( const char *name, const char *url, const int initType, const char *filekey); keyentry *getKeys( int id); }; diff --git a/storage/xtradb/include/KeySingleton.h b/storage/xtradb/include/KeySingleton.h index d5817fca7ee17..033245b56212b 100644 --- a/storage/xtradb/include/KeySingleton.h +++ b/storage/xtradb/include/KeySingleton.h @@ -28,10 +28,9 @@ Created 09/13/2014 Florin Fugaciu #include "EncKeys.h" -class KeySingleton { +class KeySingleton +{ private: - - static bool instanceInited; static KeySingleton theInstance; static EncKeys encKeys; @@ -46,10 +45,11 @@ class KeySingleton { KeySingleton & operator = (const KeySingleton&); public: - virtual ~KeySingleton() {} //encKeys.~EncKeys(); + virtual ~KeySingleton() {} static KeySingleton& getInstance(); // Init the instance for only one time - static KeySingleton& getInstance(const char *name, const char *url, const int initType); + static KeySingleton& getInstance(const char *name, const char *url, + const int initType, const char *filekey); keyentry *getKeys(int id); }; diff --git a/unittest/eperi/EperiKeySingleton-t.cc b/unittest/eperi/EperiKeySingleton-t.cc index a4a83fdeac1a3..e64d5112b73b1 100644 --- a/unittest/eperi/EperiKeySingleton-t.cc +++ b/unittest/eperi/EperiKeySingleton-t.cc @@ -1,73 +1,47 @@ -/******************************************************************//** -@file EperiKeySingleton-t.cc -Implementation of single pattern to keep keys for encrypting/decrypting pages. - -Created 09/15/2014 Florin Fugaciu -***********************************************************************/ - - -#include "EperiKeySingleton-t.h" -#include -#include -#include -#include -#include -#include - - -EperiKeySingleton::EperiKeySingleton() { -} - -EperiKeySingleton::~EperiKeySingleton() { -} - - - - -void printEntry(struct keyentry *entry, uint id) -{ - if( NULL == entry) - printf("No such keyID = %d\n", id); - else - printf("%3u. id:%3u \tiv:%s \tkey:%s\n", id, entry->id, entry->iv, entry->key); -} - -void printAll(KeySingleton& ksp, uint length) -{ - int len = (MAX_KEYS <= length ? MAX_KEYS : length); - for( int ii=1; ii<=len; ii++) - printEntry(ksp.getKeys(ii), ii); -} - - -int main() -{ -// plan(9); - printf("%s\n", "main() EperiKeySingleton.cc"); - - KeySingleton& ksp = KeySingleton::getInstance( "keys.txt", "/home/florin/w/cxx/build-mariadb/unittest/eperi/", KEYINITTYPE_FILE); - printEntry(ksp.getKeys(0), 0); - -/* - EncKeys encKeys; - encKeys.initKeys("keys.txt", "/home/florin/w/cxx/build-mariadb/unittest/eperi/", KEYINITTYPE_FILE); - printEntry(encKeys.getKeys(0), 0); -*/ - - - printAll(ksp, 256); - ok(ksp.getKeys(1)->id == 1, "Key id 1 is present"); - ok(!strcmp(ksp.getKeys(2)->iv,"35B2FF0795FB84BBD666DB8430CA214E"), "Testing IV value of key 2"); - ok(!strcmp(ksp.getKeys(15)->key, "B374A26A71490437AA024E4FADD5B497FDFF1A8EA6FF12F6FB65AF2720B59CCF"),"Testing key value of key 15"); - ok((NULL == ksp.getKeys(47)->key), "Key id 47 should be null."); - ok(ksp.getKeys(255)->id == 255, "Last possible key to insert"); - ok((NULL == ksp.getKeys(256)), "Cannot insert more keys than defined."); - - KeySingleton& ksp1 = KeySingleton::getInstance("keys.txt", "/home/florin/w/cxx/build-mariadb/unittest/eperi", KEYINITTYPE_FILE); - printEntry(ksp1.getKeys(1), 1); - - KeySingleton& ksp2 = KeySingleton::getInstance(); - printEntry(ksp2.getKeys(2), 2); - - return EXIT_SUCCESS; -} +/******************************************************************//** +@file EperiKeySingleton-t.cc +Implementation of single pattern to keep keys for encrypting/decrypting pages. + +Created 09/15/2014 Florin Fugaciu +***********************************************************************/ + + +#include "EperiKeySingleton-t.h" +#include +#include +#include +#include +#include +#include + + +EperiKeySingleton::EperiKeySingleton() { +} + +EperiKeySingleton::~EperiKeySingleton() { +} + + + + +void printEntry(struct keyentry *entry, uint id) +{ + if( NULL == entry) + printf("No such keyID = %d\n", id); + else + printf("%3u. id:%3u \tiv:%s \tkey:%s\n", id, entry->id, entry->iv, entry->key); +} + + +int main() +{ + + plan(1); + printf("%s\n", "main() EperiKeySingleton.cc"); + + KeySingleton& ksp = KeySingleton::getInstance( "keys.txt", "/home/denis", 1, "secret"); + printEntry(ksp.getKeys(0), 0); + + return EXIT_SUCCESS; + +} diff --git a/unittest/eperi/EperiKeySingleton-t.h b/unittest/eperi/EperiKeySingleton-t.h index 92939b2706c42..f50bb8b543192 100644 --- a/unittest/eperi/EperiKeySingleton-t.h +++ b/unittest/eperi/EperiKeySingleton-t.h @@ -1,17 +1,17 @@ -/******************************************************************//** -@file EperiKeySingleton-t.h -Implementation of single pattern to keep keys for encrypting/decrypting pages. - -Created 09/15/2014 Florin Fugaciu -***********************************************************************/ - -#ifndef EPERIKEYSINGLETON_T_H_ -#define EPERIKEYSINGLETONPATTERN_T_H_ - -class EperiKeySingleton { -public: - EperiKeySingleton(); - virtual ~EperiKeySingleton(); -}; - -#endif /* EPERIKEYSINGLETON_T_H_ */ +/******************************************************************//** +@file EperiKeySingleton-t.h +Implementation of single pattern to keep keys for encrypting/decrypting pages. + +Created 09/15/2014 Florin Fugaciu +***********************************************************************/ + +#ifndef EPERIKEYSINGLETON_T_H_ +#define EPERIKEYSINGLETONPATTERN_T_H_ + +class EperiKeySingleton { +public: + EperiKeySingleton(); + virtual ~EperiKeySingleton(); +}; + +#endif /* EPERIKEYSINGLETON_T_H_ */ From ece88283f2c158ead4319fb13400e33139d6afec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Clemens=20D=C3=B6rrh=C3=B6fer?= Date: Thu, 2 Oct 2014 11:22:31 +0200 Subject: [PATCH 15/70] typedef ulint entfernt --- include/my_aes.h | 9 ++++----- mysys_ssl/my_aes.cc | 14 +++++++------- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/include/my_aes.h b/include/my_aes.h index c3dc68ce833d6..c628bfa21ab82 100644 --- a/include/my_aes.h +++ b/include/my_aes.h @@ -31,7 +31,6 @@ C_MODE_START #define AES_KEY_LENGTH 128 /* Must be 128 192 or 256 */ -typedef unsigned long int ulint; /* @@ -44,8 +43,8 @@ typedef unsigned long int ulint; returns - size of encrypted data, or negative in case of error. */ -int my_aes_encrypt_cbc(const char* source, ulint source_length, - char* dest, ulint *dest_length, +int my_aes_encrypt_cbc(const char* source, unsigned long int source_length, + char* dest, unsigned long int *dest_length, const unsigned char* key, uint8 key_length, const unsigned char* iv, uint8 iv_length); @@ -100,8 +99,8 @@ int my_aes_encrypt(const char *source, int source_length, char *dest, returns - size of original data, or negative in case of error. */ -int my_aes_decrypt_cbc(const char* source, ulint source_length, - char* dest, ulint *dest_length, +int my_aes_decrypt_cbc(const char* source, unsigned long int source_length, + char* dest, unsigned long int *dest_length, const unsigned char* key, uint8 key_length, const unsigned char* iv, uint8 iv_length); diff --git a/mysys_ssl/my_aes.cc b/mysys_ssl/my_aes.cc index c68b56aa7f117..a00ddf204d8bf 100644 --- a/mysys_ssl/my_aes.cc +++ b/mysys_ssl/my_aes.cc @@ -156,8 +156,8 @@ my_bytes_to_key(const unsigned char *salt, const char *secret, unsigned char *ke >= 0 Size of encrypted data < 0 Error */ -int my_aes_encrypt_cbc(const char* source, ulint source_length, - char* dest, ulint *dest_length, +int my_aes_encrypt_cbc(const char* source, unsigned long int source_length, + char* dest, unsigned long int *dest_length, const unsigned char* key, uint8 key_length, const unsigned char* iv, uint8 iv_length) { @@ -194,13 +194,13 @@ int my_aes_encrypt_cbc(const char* source, ulint source_length, return AES_BAD_DATA; /* Error */ if (! EVP_EncryptFinal_ex(&ctx.ctx, (unsigned char *) dest + u_len, &f_len)) return AES_BAD_DATA; /* Error */ - *dest_length = (ulint) (u_len + f_len); + *dest_length = (unsigned long int) (u_len + f_len); #endif return AES_OK; } -int my_aes_decrypt_cbc(const char* source, ulint source_length, - char* dest, ulint *dest_length, +int my_aes_decrypt_cbc(const char* source, unsigned long int source_length, + char* dest, unsigned long int *dest_length, const unsigned char* key, uint8 key_length, const unsigned char* iv, uint8 iv_length) { @@ -234,10 +234,10 @@ int my_aes_decrypt_cbc(const char* source, ulint source_length, (unsigned char *)source, source_length)) return AES_BAD_DATA; /* Error */ if (! EVP_DecryptFinal_ex(&ctx.ctx, (unsigned char *) dest + u_len, &f_len)) { - *dest_length = (ulint) u_len; + *dest_length = (unsigned long int) u_len; return AES_BAD_DATA; } - *dest_length = (ulint) (u_len + f_len); + *dest_length = (unsigned long int) (u_len + f_len); #endif return AES_OK; } From c31d6e67ce5e1ef35e0c9d5c9c9d5cb70bca9053 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludger=20G=C3=B6ckel?= Date: Thu, 2 Oct 2014 15:28:20 +0200 Subject: [PATCH 16/70] fix unit test build file --- unittest/eperi/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/unittest/eperi/CMakeLists.txt b/unittest/eperi/CMakeLists.txt index c6494b687c844..b4bc4bbccf47c 100644 --- a/unittest/eperi/CMakeLists.txt +++ b/unittest/eperi/CMakeLists.txt @@ -32,7 +32,7 @@ MY_ADD_TESTS(eperi_aes MY_ADD_TESTS(EperiKeySingleton EXT "cc" - LINK_LIBRARIES xtradb pcre) + LINK_LIBRARIES xtradb pcre mysys_ssl) MY_ADD_TESTS( pageenc From 3ab88c57d0de5f9cab4e5c021cfe8b69d3faf940 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Clemens=20D=C3=B6rrh=C3=B6fer?= Date: Thu, 2 Oct 2014 16:04:38 +0200 Subject: [PATCH 17/70] key aus keyfile --- include/keyfile.h | 3 +++ storage/xtradb/CMakeLists.txt | 1 + storage/xtradb/enc/EncKeys.cc | 2 +- storage/xtradb/fil/fil0pageencryption.cc | 12 ++++++++++-- storage/xtradb/handler/ha_innodb.cc | 2 +- storage/xtradb/handler/ha_innodb.h | 1 + storage/xtradb/include/EncKeys.h | 2 ++ unittest/eperi/CMakeLists.txt | 2 +- 8 files changed, 20 insertions(+), 5 deletions(-) diff --git a/include/keyfile.h b/include/keyfile.h index f4ee664e9d26f..72cc9aeb7cdf5 100644 --- a/include/keyfile.h +++ b/include/keyfile.h @@ -1,3 +1,5 @@ +#ifndef KEYFILE_H +#define KEYFILE_H #include struct keyentry { @@ -17,3 +19,4 @@ isComment(char *line); char* trim(char *in); +#endif diff --git a/storage/xtradb/CMakeLists.txt b/storage/xtradb/CMakeLists.txt index cf5da4061b550..41b9665beacdb 100644 --- a/storage/xtradb/CMakeLists.txt +++ b/storage/xtradb/CMakeLists.txt @@ -30,6 +30,7 @@ MYSQL_CHECK_BZIP2() # OS tests IF(UNIX) + IF(CMAKE_SYSTEM_NAME STREQUAL "Linux") CHECK_INCLUDE_FILES (libaio.h HAVE_LIBAIO_H) IF (XTRADB_PREFER_STATIC_LIBAIO) diff --git a/storage/xtradb/enc/EncKeys.cc b/storage/xtradb/enc/EncKeys.cc index badf67eaa1326..5a4705d930b12 100644 --- a/storage/xtradb/enc/EncKeys.cc +++ b/storage/xtradb/enc/EncKeys.cc @@ -17,7 +17,7 @@ @file EncKeys.cc A class to keep keys for encryption/decryption. - Created 09/15/2014 Florin Fugaciu & Clemens Doerrhoefer + Created 09/15/2014 Florin Fugaciu ***********************************************************************/ #include "EncKeys.h" diff --git a/storage/xtradb/fil/fil0pageencryption.cc b/storage/xtradb/fil/fil0pageencryption.cc index 1ab6a22593916..c759c0d18ad83 100644 --- a/storage/xtradb/fil/fil0pageencryption.cc +++ b/storage/xtradb/fil/fil0pageencryption.cc @@ -20,6 +20,7 @@ #include "buf0checksum.h" #include +#include //#define UNIV_PAGEENCRIPTION_DEBUG //#define CRYPT_FF @@ -112,12 +113,15 @@ ulint mode /* calculate a checksum, can be used to verify decryption */ checksum = fil_page_encryption_calc_checksum(buf + header_len, page_size - (FIL_PAGE_DATA_END + header_len)); - + KeySingleton& keys = KeySingleton::getInstance(); const unsigned char rkey[] = {0xbd, 0xe4, 0x72, 0xa2, 0x95, 0x67, 0x5c, 0xa9, 0x2e, 0x04, 0x67, 0xea, 0xdb, 0xc0,0xe0, 0x23}; + my_aes_hexToUint(keys.getKeys(encryption_key)->key, (unsigned char*)&rkey, 16); + uint8 key_len = 16; const unsigned char iv[] = {0x2d, 0x1a, 0xf8, 0xd3, 0x97, 0x4e, 0x0b, 0xd3, 0xef, 0xed, 0x5a, 0x6f, 0x82, 0x59, 0x4f,0x5e}; + my_aes_hexToUint(keys.getKeys(encryption_key)->iv, (unsigned char*)&iv, 16); uint8 iv_len = 16; write_size = data_size; @@ -388,12 +392,16 @@ al_size); tmp_buf= static_cast(ut_malloc(64)); tmp_page_buf = static_cast(ut_malloc(page_size)); memset(tmp_page_buf,0, page_size); + + KeySingleton& keys = KeySingleton::getInstance(); const unsigned char rkey[] = {0xbd, 0xe4, 0x72, 0xa2, 0x95, 0x67, 0x5c, 0xa9, - 0x2e, 0x04, 0x67, 0xea, 0xdb, 0xc0,0xe0, 0x23}; + 0x2e, 0x04, 0x67, 0xea, 0xdb, 0xc0,0xe0, 0x23}; + my_aes_hexToUint(keys.getKeys(page_encryption_key)->key, (unsigned char*)&rkey, 16); uint8 key_len = 16; const unsigned char iv[] = {0x2d, 0x1a, 0xf8, 0xd3, 0x97, 0x4e, 0x0b, 0xd3, 0xef, 0xed, 0x5a, 0x6f, 0x82, 0x59, 0x4f,0x5e}; + my_aes_hexToUint(keys.getKeys(page_encryption_key)->iv, (unsigned char*)&rkey, 16); uint8 iv_len = 16; if (!page_compression_flag) { diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index a0de249dfb0bc..f912a946e8178 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -3415,7 +3415,7 @@ innobase_init( KeySingleton& keysingleton = KeySingleton::getInstance( innobase_data_encryption_providername, innobase_data_encryption_providerurl, innobase_data_encryption_providertype, innobase_data_encryption_filekey); - struct keyentry *entry = keysingleton.getKeys(3); + struct keyentry *entry = keysingleton.getKeys(1); if(entry) printf("id:%3u \tiv:%s \tkey:%s\n", entry->id, entry->iv, entry->key); #ifndef DBUG_OFF diff --git a/storage/xtradb/handler/ha_innodb.h b/storage/xtradb/handler/ha_innodb.h index ebc566dc78ace..7a41a41858a18 100644 --- a/storage/xtradb/handler/ha_innodb.h +++ b/storage/xtradb/handler/ha_innodb.h @@ -28,6 +28,7 @@ this program; if not, write to the Free Software Foundation, Inc., #include "KeySingleton.h" + /* Structure defines translation table between mysql index and innodb index structures */ struct innodb_idx_translate_t { diff --git a/storage/xtradb/include/EncKeys.h b/storage/xtradb/include/EncKeys.h index 36d4523f16639..f71d21c7b394c 100644 --- a/storage/xtradb/include/EncKeys.h +++ b/storage/xtradb/include/EncKeys.h @@ -28,6 +28,8 @@ Created 09/15/2014 Florin Fugaciu #include + + struct keyentry { uint id; char *iv; diff --git a/unittest/eperi/CMakeLists.txt b/unittest/eperi/CMakeLists.txt index c6494b687c844..b4bc4bbccf47c 100644 --- a/unittest/eperi/CMakeLists.txt +++ b/unittest/eperi/CMakeLists.txt @@ -32,7 +32,7 @@ MY_ADD_TESTS(eperi_aes MY_ADD_TESTS(EperiKeySingleton EXT "cc" - LINK_LIBRARIES xtradb pcre) + LINK_LIBRARIES xtradb pcre mysys_ssl) MY_ADD_TESTS( pageenc From a74dddbfd6bbf82cc681cf683f3a45c96146f543 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludger=20G=C3=B6ckel?= Date: Thu, 2 Oct 2014 16:59:35 +0200 Subject: [PATCH 18/70] respect key length and reactivate unit test --- storage/xtradb/fil/fil0pageencryption.cc | 29 +++++++++++++++++------- unittest/eperi/pageenc-t.cc | 4 ++-- 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/storage/xtradb/fil/fil0pageencryption.cc b/storage/xtradb/fil/fil0pageencryption.cc index c759c0d18ad83..49dd11fc0390c 100644 --- a/storage/xtradb/fil/fil0pageencryption.cc +++ b/storage/xtradb/fil/fil0pageencryption.cc @@ -113,17 +113,22 @@ ulint mode /* calculate a checksum, can be used to verify decryption */ checksum = fil_page_encryption_calc_checksum(buf + header_len, page_size - (FIL_PAGE_DATA_END + header_len)); - KeySingleton& keys = KeySingleton::getInstance(); const unsigned char rkey[] = {0xbd, 0xe4, 0x72, 0xa2, 0x95, 0x67, 0x5c, 0xa9, 0x2e, 0x04, 0x67, 0xea, 0xdb, 0xc0,0xe0, 0x23}; - my_aes_hexToUint(keys.getKeys(encryption_key)->key, (unsigned char*)&rkey, 16); - uint8 key_len = 16; + if (!unit_test) { + KeySingleton& keys = KeySingleton::getInstance(); + char* keyString = keys.getKeys(encryption_key)->key; + key_len = strlen(keyString)/2; + my_aes_hexToUint(keyString, (unsigned char*)&rkey, key_len); + } const unsigned char iv[] = {0x2d, 0x1a, 0xf8, 0xd3, 0x97, 0x4e, 0x0b, 0xd3, 0xef, 0xed, 0x5a, 0x6f, 0x82, 0x59, 0x4f,0x5e}; - my_aes_hexToUint(keys.getKeys(encryption_key)->iv, (unsigned char*)&iv, 16); + if (!unit_test) { + KeySingleton& keys = KeySingleton::getInstance(); + my_aes_hexToUint(keys.getKeys(encryption_key)->iv, (unsigned char*)&iv, 16); + } uint8 iv_len = 16; - write_size = data_size; /* 1st encryption: data_size -1 bytes starting from FIL_PAGE_DATA */ err = my_aes_encrypt_cbc((char*)buf + header_len, @@ -393,15 +398,23 @@ al_size); tmp_page_buf = static_cast(ut_malloc(page_size)); memset(tmp_page_buf,0, page_size); - KeySingleton& keys = KeySingleton::getInstance(); const unsigned char rkey[] = {0xbd, 0xe4, 0x72, 0xa2, 0x95, 0x67, 0x5c, 0xa9, 0x2e, 0x04, 0x67, 0xea, 0xdb, 0xc0,0xe0, 0x23}; - my_aes_hexToUint(keys.getKeys(page_encryption_key)->key, (unsigned char*)&rkey, 16); uint8 key_len = 16; + if (!unit_test) { + KeySingleton& keys = KeySingleton::getInstance(); + char* keyString = keys.getKeys(page_encryption_key)->key; + key_len = strlen(keyString)/2; + my_aes_hexToUint(keyString, (unsigned char*)&rkey, key_len); + } + const unsigned char iv[] = {0x2d, 0x1a, 0xf8, 0xd3, 0x97, 0x4e, 0x0b, 0xd3, 0xef, 0xed, 0x5a, 0x6f, 0x82, 0x59, 0x4f,0x5e}; - my_aes_hexToUint(keys.getKeys(page_encryption_key)->iv, (unsigned char*)&rkey, 16); + if (!unit_test) { + KeySingleton& keys = KeySingleton::getInstance(); + my_aes_hexToUint(keys.getKeys(page_encryption_key)->iv, (unsigned char*)&iv, 16); + } uint8 iv_len = 16; if (!page_compression_flag) { diff --git a/unittest/eperi/pageenc-t.cc b/unittest/eperi/pageenc-t.cc index efb78efc0d823..397831eeabf83 100644 --- a/unittest/eperi/pageenc-t.cc +++ b/unittest/eperi/pageenc-t.cc @@ -99,7 +99,7 @@ void testIt(char* filename, ulint cmp_checksum) { ok (i==0, str); } void test_page_enc_dec() { - /* + testIt("compressed",0); testIt("compressed_full",0); @@ -111,7 +111,7 @@ void test_page_enc_dec() { testIt("xae",1); testIt("xaf",1); - */ + } From 22d832e30476140a3ff26e711f0e4bc6b8e9e2d4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Clemens=20D=C3=B6rrh=C3=B6fer?= Date: Thu, 2 Oct 2014 17:44:57 +0200 Subject: [PATCH 19/70] Removed blocksize output --- mysys_ssl/my_aes.cc | 1 - 1 file changed, 1 deletion(-) diff --git a/mysys_ssl/my_aes.cc b/mysys_ssl/my_aes.cc index a00ddf204d8bf..012966d0626cf 100644 --- a/mysys_ssl/my_aes.cc +++ b/mysys_ssl/my_aes.cc @@ -188,7 +188,6 @@ int my_aes_encrypt_cbc(const char* source, unsigned long int source_length, OPENSSL_assert(EVP_CIPHER_CTX_key_length(&ctx.ctx) == key_length); OPENSSL_assert(EVP_CIPHER_CTX_iv_length(&ctx.ctx) == iv_length); OPENSSL_assert(EVP_CIPHER_CTX_block_size(&ctx.ctx) == 16); - printf("Blocksize: %d \n", EVP_CIPHER_CTX_block_size(&ctx.ctx)); if (! EVP_EncryptUpdate(&ctx.ctx, (unsigned char *) dest, &u_len, (unsigned const char *) source, source_length)) return AES_BAD_DATA; /* Error */ From b8a81ecdc42f2d685b07347a72f112f2ca6011e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludger=20G=C3=B6ckel?= Date: Mon, 6 Oct 2014 12:00:55 +0200 Subject: [PATCH 20/70] change header comments and removed a unit test --- include/keyfile.h | 16 +++ storage/xtradb/enc/EncKeys.cc | 2 +- storage/xtradb/enc/KeySingleton.cc | 23 ++-- storage/xtradb/enc/keyfile.c | 16 +++ storage/xtradb/fil/fil0pageencryption.cc | 17 ++- storage/xtradb/include/EncKeys.h | 23 ++-- storage/xtradb/include/KeySingleton.h | 23 ++-- storage/xtradb/include/fil0pageencryption.h | 15 ++- storage/xtradb/include/fsp0pageencryption.h | 23 ++-- storage/xtradb/include/fsp0pageencryption.ic | 23 ++-- storage/xtradb/os/os0file.cc | 4 +- unittest/eperi/CMakeLists.txt | 2 +- unittest/eperi/pfs_jim-t.cc | 126 ------------------- 13 files changed, 119 insertions(+), 194 deletions(-) delete mode 100644 unittest/eperi/pfs_jim-t.cc diff --git a/include/keyfile.h b/include/keyfile.h index 72cc9aeb7cdf5..d55c79086d47c 100644 --- a/include/keyfile.h +++ b/include/keyfile.h @@ -1,3 +1,19 @@ +/* Copyright (C) 2014 eperi GmbH. All Rights Reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +/******************************************************************/ #ifndef KEYFILE_H #define KEYFILE_H #include diff --git a/storage/xtradb/enc/EncKeys.cc b/storage/xtradb/enc/EncKeys.cc index 5a4705d930b12..8c3464295f5cb 100644 --- a/storage/xtradb/enc/EncKeys.cc +++ b/storage/xtradb/enc/EncKeys.cc @@ -1,4 +1,4 @@ -/* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. +/* Copyright (C) 2014 eperi GmbH. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/storage/xtradb/enc/KeySingleton.cc b/storage/xtradb/enc/KeySingleton.cc index 5dab2a5434c31..6a3ea1908c41e 100644 --- a/storage/xtradb/enc/KeySingleton.cc +++ b/storage/xtradb/enc/KeySingleton.cc @@ -1,18 +1,17 @@ -/* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. +/* Copyright (C) 2014 eperi GmbH. All Rights Reserved. - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ /******************************************************************//** @file KeySingleton.cc diff --git a/storage/xtradb/enc/keyfile.c b/storage/xtradb/enc/keyfile.c index 38b773c0ebcaa..1afed215666f2 100644 --- a/storage/xtradb/enc/keyfile.c +++ b/storage/xtradb/enc/keyfile.c @@ -1,3 +1,19 @@ +/* Copyright (C) 2014 eperi GmbH. All Rights Reserved. + + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + +/******************************************************************/ //Author Clemens Doerrhoefer #include diff --git a/storage/xtradb/fil/fil0pageencryption.cc b/storage/xtradb/fil/fil0pageencryption.cc index 49dd11fc0390c..111fd7371df46 100644 --- a/storage/xtradb/fil/fil0pageencryption.cc +++ b/storage/xtradb/fil/fil0pageencryption.cc @@ -1,9 +1,20 @@ /***************************************************************************** - Copyright (C) 2014 +Copyright (C) 2014 eperi GmbH. All Rights Reserved. - eperi GmbH - *****************************************************************************/ +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + +*****************************************************************************/ /***************************************************************** @file fil/fil0pageencryption.cc diff --git a/storage/xtradb/include/EncKeys.h b/storage/xtradb/include/EncKeys.h index f71d21c7b394c..f1f1bf2d04ed1 100644 --- a/storage/xtradb/include/EncKeys.h +++ b/storage/xtradb/include/EncKeys.h @@ -1,18 +1,17 @@ -/* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. +/* Copyright (C) 2014 eperi GmbH. All Rights Reserved. - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ /******************************************************************//** @file EncKeys.h diff --git a/storage/xtradb/include/KeySingleton.h b/storage/xtradb/include/KeySingleton.h index 033245b56212b..9fbd95e04aed1 100644 --- a/storage/xtradb/include/KeySingleton.h +++ b/storage/xtradb/include/KeySingleton.h @@ -1,18 +1,17 @@ -/* Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. +/* Copyright (C) 2014 eperi GmbH. All Rights Reserved. - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ /******************************************************************//** @file KeySingletonPattern.h diff --git a/storage/xtradb/include/fil0pageencryption.h b/storage/xtradb/include/fil0pageencryption.h index c59756235735c..4d50d5088f393 100644 --- a/storage/xtradb/include/fil0pageencryption.h +++ b/storage/xtradb/include/fil0pageencryption.h @@ -1,8 +1,18 @@ /***************************************************************************** -Copyright (C) 2014 +Copyright (C) 2014 eperi GmbH. All Rights Reserved. -eperi GmbH +This program is free software; you can redistribute it and/or modify it under +the terms of the GNU General Public License as published by the Free Software +Foundation; version 2 of the License. + +This program is distributed in the hope that it will be useful, but WITHOUT +ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS +FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + +You should have received a copy of the GNU General Public License along with +this program; if not, write to the Free Software Foundation, Inc., +51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA *****************************************************************************/ @@ -17,6 +27,7 @@ eperi GmbH #define PAGE_ENCRYPTION_WRONG_KEY 1 #define PAGE_ENCRYPTION_WRONG_PAGE_TYPE 2 #define PAGE_ENCRYPTION_ERROR 3 +#define PAGE_ENCRYPTION_OK 0 diff --git a/storage/xtradb/include/fsp0pageencryption.h b/storage/xtradb/include/fsp0pageencryption.h index fdf484ea64485..1abc38cfb1d14 100644 --- a/storage/xtradb/include/fsp0pageencryption.h +++ b/storage/xtradb/include/fsp0pageencryption.h @@ -1,20 +1,21 @@ /***************************************************************************** -Copyright (C) 2013 SkySQL Ab. All Rights Reserved. +/* Copyright (C) 2014 eperi GmbH. All Rights Reserved. -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; version 2 of the License. + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., -51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ -*****************************************************************************/ +/******************************************************************/ /******************************************************************//** @file include/fsp0pageencryption.h diff --git a/storage/xtradb/include/fsp0pageencryption.ic b/storage/xtradb/include/fsp0pageencryption.ic index 10559e7bee650..7e9b0d9eea455 100644 --- a/storage/xtradb/include/fsp0pageencryption.ic +++ b/storage/xtradb/include/fsp0pageencryption.ic @@ -1,20 +1,19 @@ /***************************************************************************** -Copyright (C) 2013,2014 SkySQL Ab. All Rights Reserved. + Copyright (C) 2014 eperi GmbH. All Rights Reserved. -This program is free software; you can redistribute it and/or modify it under -the terms of the GNU General Public License as published by the Free Software -Foundation; version 2 of the License. + This program is free software; you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation; version 2 of the License. -This program is distributed in the hope that it will be useful, but WITHOUT -ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS -FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., -51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA - -*****************************************************************************/ + You should have received a copy of the GNU General Public License + along with this program; if not, write to the Free Software + Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ /******************************************************************//** @file include/fsp0pageencryption.ic diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index b1c84a8bd85e7..0a03adaa31156 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -3148,7 +3148,7 @@ os_file_read_func( if ((ulint) ret == n) { if (fil_page_is_encrypted((byte *)buf)) { - if (fil_decrypt_page(NULL, (byte *)buf, n, NULL, &compressed, 0)) {; + if (fil_decrypt_page(NULL, (byte *)buf, n, NULL, &compressed, 0)!=PAGE_ENCRYPTION_OK) {; return FALSE; } } @@ -3274,7 +3274,7 @@ os_file_read_no_error_handling_func( if (fil_page_is_encrypted((byte *)buf)) { - fil_decrypt_page(NULL, (byte *)buf, n, NULL, &compressed, 0); + if (fil_decrypt_page(NULL, (byte *)buf, n, NULL, &compressed, 0)!=PAGE_ENCRYPTION_OK) return (FALSE); } diff --git a/unittest/eperi/CMakeLists.txt b/unittest/eperi/CMakeLists.txt index b4bc4bbccf47c..0ae9ebfa54176 100644 --- a/unittest/eperi/CMakeLists.txt +++ b/unittest/eperi/CMakeLists.txt @@ -1,4 +1,4 @@ -# Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved. +# Copyright (C) 2014 eperi GmbH. All Rights Reserved. # # This program is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by diff --git a/unittest/eperi/pfs_jim-t.cc b/unittest/eperi/pfs_jim-t.cc deleted file mode 100644 index 580adc41aef06..0000000000000 --- a/unittest/eperi/pfs_jim-t.cc +++ /dev/null @@ -1,126 +0,0 @@ -/* Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved. - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software Foundation, - 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA */ - -#include -#include -#include -#include -#include -#include -#include - -#include "stub_pfs_global.h" -#include "stub_server_misc.h" - -#include /* memset */ - -extern int summef(int a, int b); -extern int summef2(int a, int b); - - -void test_oomf() -{ - int rc; - PFS_global_param param; - - memset(& param, 0xFF, sizeof(param)); - param.m_enabled= true; - param.m_mutex_class_sizing= 0; - param.m_rwlock_class_sizing= 0; - param.m_cond_class_sizing= 0; - param.m_thread_class_sizing= 10; - param.m_table_share_sizing= 0; - param.m_file_class_sizing= 0; - param.m_mutex_sizing= 0; - param.m_rwlock_sizing= 0; - param.m_cond_sizing= 0; - param.m_thread_sizing= 1000; - param.m_table_sizing= 0; - param.m_file_sizing= 0; - param.m_file_handle_sizing= 0; - param.m_events_waits_history_sizing= 10; - param.m_events_waits_history_long_sizing= 0; - param.m_setup_actor_sizing= 0; - param.m_setup_object_sizing= 0; - param.m_host_sizing= 0; - param.m_user_sizing= 0; - param.m_account_sizing= 1000; - param.m_stage_class_sizing= 50; - param.m_events_stages_history_sizing= 0; - param.m_events_stages_history_long_sizing= 0; - param.m_statement_class_sizing= 50; - param.m_events_statements_history_sizing= 0; - param.m_events_statements_history_long_sizing= 0; - param.m_session_connect_attrs_sizing= 0; - - /* Setup */ - - stub_alloc_always_fails= false; - stub_alloc_fails_after_count= 1000; - - init_event_name_sizing(& param); - rc= init_stage_class(param.m_stage_class_sizing); - ok(rc == 0, "init stage class"); - rc= init_statement_class(param.m_statement_class_sizing); - ok(rc == 0, "init statement class"); - - /* Tests */ - - stub_alloc_fails_after_count= 1; - rc= init_account(& param); - ok(rc == 1, "oom (account)"); - cleanup_account(); - - stub_alloc_fails_after_count= 2; - rc= init_account(& param); - ok(rc == 1, "oom (account waits)"); - cleanup_account(); - - stub_alloc_fails_after_count= 3; - rc= init_account(& param); - ok(rc == 1, "oom (account stages)"); - cleanup_account(); - - stub_alloc_fails_after_count= 4; - rc= init_account(& param); - ok(rc == 1, "oom (account statements)"); - cleanup_account(); - - cleanup_statement_class(); - cleanup_stage_class(); -} - -void do_all_tests() -{ - PFS_atomic::init(); - - test_oomf(); - - PFS_atomic::cleanup(); -} - -int main(int, char **) -{ - plan(6); - MY_INIT("pfs_account-oomff-t"); - do_all_tests(); - int sum = summef(3, 9); - printf("summef meine Zahl: %d\n", sum); - sum = summef2(7, 3); - printf("summef2 meine Zahl: %d\n", sum); - my_end(0); - return exit_status(); -} - From b6bad41faeb58e43335515453ec6784409f14464 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Clemens=20D=C3=B6rrh=C3=B6fer?= Date: Mon, 6 Oct 2014 12:19:58 +0200 Subject: [PATCH 21/70] Removed keyfile tests --- mysys/CMakeLists.txt | 2 +- mysys/keyfile.c | 162 ------------------------------- unittest/eperi/CMakeLists.txt | 2 - unittest/eperi/eperi_keyfile-t.c | 72 -------------- 4 files changed, 1 insertion(+), 237 deletions(-) delete mode 100644 mysys/keyfile.c delete mode 100644 unittest/eperi/eperi_keyfile-t.c diff --git a/mysys/CMakeLists.txt b/mysys/CMakeLists.txt index 21d36fa759e02..6d9c88ccdfd84 100644 --- a/mysys/CMakeLists.txt +++ b/mysys/CMakeLists.txt @@ -43,7 +43,7 @@ SET(MYSYS_SOURCES array.c charset-def.c charset.c checksum.c my_default.c my_atomic.c my_getncpus.c my_safehash.c my_chmod.c my_rnd.c my_uuid.c wqueue.c waiting_threads.c ma_dyncol.c my_rdtsc.c my_context.c psi_noop.c - file_logger.c keyfile.c) + file_logger.c ) IF (WIN32) SET (MYSYS_SOURCES ${MYSYS_SOURCES} my_winthread.c my_wincond.c my_winerr.c my_winfile.c my_windac.c my_conio.c) diff --git a/mysys/keyfile.c b/mysys/keyfile.c deleted file mode 100644 index 38b773c0ebcaa..0000000000000 --- a/mysys/keyfile.c +++ /dev/null @@ -1,162 +0,0 @@ -//Author Clemens Doerrhoefer - -#include -#include -#include -#include -#include - -#define E_WRONG_NUMBER_OF_MATCHES 10 -#define MAX_KEY_FILE_SIZE 1048576 -#define MAX_BUFFER_LENGTH 512 - -#define KEY_FILE_PARSE_OK 0 -#define KEY_FILE_TOO_BIG 100 -#define KEY_BUFFER_TOO_BIG 200 -#define KEY_FILE_PARSE_NULL 300 -#define KEY_FILE_TOO_MANY_KEYS 400 - - -int -isComment(char *line) -{ - const char *error_p; - int offset; - int m_len = (int) strlen(line); - - pcre *pattern = pcre_compile( - "\\s*#.*", - 0, - &error_p, - &offset, - NULL); - int rc,i; - int ovector[30]; - rc = pcre_exec( - pattern, - NULL, - line, - m_len, - 0, - 0, - ovector, - 30 - ); - if(rc < 0) { - return 0; - } else { - return 1; - } -} - -int -parseFile(FILE * fp, struct keyentry **allKeys, const int k_len, const char *secret) -{ - const char *MAGIC = "Salted__"; - long file_size = 0; - char *buffer, *decrypted; - char *line = NULL; - if(NULL == fp) { - fprintf(stderr, "Key file not found.\n"); - return 100; - } - - //get size of file - fseek(fp, 0L, SEEK_END); - file_size = ftell(fp); - fseek(fp, 0L, SEEK_SET); - - if(file_size > MAX_KEY_FILE_SIZE) { - return KEY_FILE_TOO_BIG; - } - - //Read file into buffer - buffer = (char*) malloc((file_size+1)*sizeof(char)); - fread(buffer, file_size, 1, fp); - - //Check for file encryption - if(memcmp(buffer, MAGIC, 8) == 0) { //If file is encrypted, decrypt it first. - unsigned char salt[8]; - unsigned char *key = malloc(32 * sizeof(char)); - unsigned char *iv = malloc(16 * sizeof(char)); - decrypted = malloc(file_size * sizeof(char)); - memcpy(&salt, buffer+8, 8); - my_bytes_to_key(&salt, secret, key, iv); - unsigned long int d_size = 0; - my_aes_decrypt_cbc(buffer + 16, file_size -16, decrypted, &d_size, key, 32, iv, 16); - memcpy(buffer, decrypted, d_size); - - free(decrypted); - free(key); - free(iv); - } - - line = strtok(buffer, "\n"); - while(line != NULL) { - struct keyentry *entry = (struct keyentry*) malloc(sizeof(struct keyentry)); - if( parseLine(line, entry, k_len) == 0) { - allKeys[entry->id] = entry; - } - line = strtok(NULL, "\n"); - } - free(buffer); - return KEY_FILE_PARSE_OK; -} - -int -parseLine(const char *line, struct keyentry *entry, const int k_len) -{ - const char *error_p; - int offset; - - pcre *pattern = pcre_compile( - "([0-9]+);([0-9,a-f,A-F]{32});([0-9,a-f,A-F]{64}|[0-9,a-f,A-F]{48}|[0-9,a-f,A-F]{32})", - 0, - &error_p, - &offset, - NULL); - if( error_p != NULL ) { - fprintf(stderr, "Error: %s\n", error_p); - fprintf(stderr, "Offset: %d\n", offset); - } - int m_len = (int) strlen(line); - char *buffer = (char*) malloc(MAX_BUFFER_LENGTH*sizeof(char)); - int rc,i; - int ovector[30]; - rc = pcre_exec( - pattern, - NULL, - line, - m_len, - 0, - 0, - ovector, - 30 - ); - if(rc == 4 && !isComment(line)) { - char *substring_start = line + ovector[2]; - int substr_length = ovector[3] - ovector[2]; - sprintf( buffer, "%.*s", substr_length, substring_start ); - entry->id = atoi(buffer); - if(entry->id >= k_len) - return KEY_FILE_TOO_MANY_KEYS; - - substring_start = line + ovector[4]; - substr_length = ovector[5] - ovector[4]; - entry->iv = malloc(substr_length*sizeof(char)); - - sprintf( entry->iv, "%.*s", substr_length, substring_start ); - - substring_start = line + ovector[6]; - substr_length = ovector[7] - ovector[6]; - entry->key = malloc(substr_length*sizeof(char)); - sprintf( entry->key, "%.*s", substr_length, substring_start ); - } else - { - return E_WRONG_NUMBER_OF_MATCHES; - } - if(entry->id == NULL || entry->iv == NULL || entry->key == NULL) { - return KEY_FILE_PARSE_NULL; - } - return KEY_FILE_PARSE_OK; -} diff --git a/unittest/eperi/CMakeLists.txt b/unittest/eperi/CMakeLists.txt index 0ae9ebfa54176..1ace9d684e46c 100644 --- a/unittest/eperi/CMakeLists.txt +++ b/unittest/eperi/CMakeLists.txt @@ -25,8 +25,6 @@ INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql ) MY_ADD_TESTS(eperi) -MY_ADD_TESTS(eperi_keyfile - LINK_LIBRARIES mysys pcre) MY_ADD_TESTS(eperi_aes LINK_LIBRARIES mysys_ssl dbug) diff --git a/unittest/eperi/eperi_keyfile-t.c b/unittest/eperi/eperi_keyfile-t.c deleted file mode 100644 index 37384ce447db1..0000000000000 --- a/unittest/eperi/eperi_keyfile-t.c +++ /dev/null @@ -1,72 +0,0 @@ -#include -#include -#include -#include - -void -printAll(struct keyentry **all, int length) -{ - int i; - for(i=0;iid, entry->iv, entry->key); -} -void -test_parseFile_ciphertext(){ - plan(6); - char *secret = "secret"; - struct keyentry **allKeys = (struct keyentry**) malloc( 256 * sizeof(struct keyentry*)); - FILE *fp = fopen("keys.enc", "r"); - if(0 == parseFile(fp, allKeys, 256, secret)) { - fclose(fp); - ok(allKeys[1]->id == 1, "Key id 1 is present"); - ok(!strcmp(allKeys[2]->iv,"35B2FF0795FB84BBD666DB8430CA214E"), "Testing IV value of key 2"); - ok(!strcmp(allKeys[15]->key, "B374A26A71490437AA024E4FADD5B497FDFF1A8EA6FF12F6FB65AF2720B59CCF"),"Testing key value of key 15"); - ok((allKeys[47] == 0), "Key id 47 should be null."); - ok(allKeys[255]->id == 255, "Last key inserted"); - ok((allKeys[256] == 0), "Cannot insert more keys than defined."); - ok(0 == strcmp(allKeys[4]->key, "770A8A65DA156D24EE2A093277530142770A8A65DA156D24EE2A093277530142"), "Parser ignores entries that are too long."); - } else - { - ok(0, "Cannot open testfile"); - } - free(allKeys); -} - -void -test_parseFile_plaintext() -{ - plan(6); - struct keyentry **allKeys = (struct keyentry**) malloc( 256 * sizeof(struct keyentry*)); - FILE *fp = fopen("keys.txt", "r"); - if(0 == parseFile(fp, allKeys, 256, NULL)) { - //printAll(allKeys, 256); - fclose(fp); - ok(allKeys[1]->id == 1, "Key id 1 is present"); - ok(0 == strcmp(allKeys[2]->iv,"35B2FF0795FB84BBD666DB8430CA214E"), "Testing IV value of key 2"); - ok(0 == strcmp(allKeys[15]->key, "B374A26A71490437AA024E4FADD5B497FDFF1A8EA6FF12F6FB65AF2720B59CCF"),"Testing key value of key 15"); - ok((allKeys[47] == 0), "Key id 47 should be null."); - ok(allKeys[255]->id == 255, "Last key inserted"); - ok((allKeys[256] == 0), "Cannot insert more keys than defined."); - ok(0 == strcmp(allKeys[4]->key, "770A8A65DA156D24EE2A093277530142770A8A65DA156D24EE2A093277530142"), "Parser ignores entries that are too long."); - } else { - ok(0, "Cannot open testfile"); - } - free(allKeys); -} - -int -main(int argc __attribute__((unused)),char *argv[]) -{ - test_parseFile_ciphertext(); - test_parseFile_plaintext(); - return 0; -} From 050ca925409786adbc70ad8a89fea4e147768314 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Clemens=20D=C3=B6rrh=C3=B6fer?= Date: Mon, 6 Oct 2014 15:04:36 +0200 Subject: [PATCH 22/70] Using my_fopen instead of fopne --- storage/xtradb/enc/EncKeys.cc | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/storage/xtradb/enc/EncKeys.cc b/storage/xtradb/enc/EncKeys.cc index 8c3464295f5cb..d89a7a44380ea 100644 --- a/storage/xtradb/enc/EncKeys.cc +++ b/storage/xtradb/enc/EncKeys.cc @@ -27,6 +27,7 @@ #include #include #include +#include /* @@ -231,10 +232,10 @@ int EncKeys::parseLine(const char *line, const uint maxKeyId) { */ char* EncKeys::decryptFile(const char* filename, const char *secret, int *errorCode) { *errorCode = NO_ERROR_PARSE_OK; - printf("Reading %s\n\n", filename); - FILE *fp = fopen(filename, "r"); + fprintf(stderr, "Reading %s\n\n", filename); + FILE *fp = my_fopen(filename, O_RDONLY, MYF(MY_WME)); if (NULL == fp) { - printf(errorOpenFile, filename); + fprintf(stderr, errorOpenFile, filename); *errorCode = ERROR_OPEN_FILE; return NULL; } @@ -260,7 +261,7 @@ char* EncKeys::decryptFile(const char* filename, const char *secret, int *errorC char *buffer = new char[file_size + 1]; fread(buffer, file_size, 1, fp); buffer[file_size] = '\0'; - fclose(fp); + my_fclose(fp, MYF(MY_WME)); //Check for file encryption if (0 == memcmp(buffer, strMAGIC, magicSize)) { //If file is encrypted, decrypt it first. From bf6610d66a1a8c7aa5f4f7459eb3e1a35f516478 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Clemens=20D=C3=B6rrh=C3=B6fer?= Date: Mon, 6 Oct 2014 15:05:10 +0200 Subject: [PATCH 23/70] Using fprintf instead of printf for error messages --- storage/xtradb/enc/EncKeys.cc | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/storage/xtradb/enc/EncKeys.cc b/storage/xtradb/enc/EncKeys.cc index d89a7a44380ea..ae287928212fd 100644 --- a/storage/xtradb/enc/EncKeys.cc +++ b/storage/xtradb/enc/EncKeys.cc @@ -93,7 +93,7 @@ bool EncKeys::initKeys(const char *name, const char *url, const int initType, co else return true; } else if (KEYINITTYPE_SERVER == initType) { - printf(errorNotImplemented); + fprintf(stderr, errorNotImplemented); } return NO_ERROR_KEY_FILE_PARSE_OK == ERROR_KEYINITTYPE_SERVER_NOT_IMPLEMENTED; } @@ -118,7 +118,7 @@ keyentry *EncKeys::getKeys(int id) { if (KEY_MIN <= id && KEY_MAX >= id && (oneKey = &keys[id - 1])->iv) return oneKey; else { - printf(errorNoKeyId, id); + fprintf(stderr, errorNoKeyId, id); return NULL; } } @@ -142,28 +142,28 @@ int EncKeys::parseFile(const char* filename, const uint maxKeyId, const char *se case NO_ERROR_PARSE_OK: keys[oneKey->id - 1] = *oneKey; countKeys++; - printf("Line: %u --> ", keyLineInKeyFile); printKeyEntry(oneKey->id); + fprintf(stderr, "Line: %u --> ", keyLineInKeyFile); printKeyEntry(oneKey->id); break; case ERROR_ID_TOO_BIG: - printf(errorExceedKeySize, KEY_MAX, keyLineInKeyFile); - printf(" --> %s\n", line); + fprintf(stderr, errorExceedKeySize, KEY_MAX, keyLineInKeyFile); + fprintf(stderr, " --> %s\n", line); errorCode = ERROR_KEY_FILE_EXCEEDS_MAX_NUMBERS_OF_KEYS; break; case ERROR_NOINITIALIZEDKEY: - printf(errorNoInitializedKey); - printf(" --> %s\n", line); + fprintf(stderr, errorNoInitializedKey); + fprintf(stderr, " --> %s\n", line); errorCode = ERROR_KEY_FILE_PARSE_NULL; break; case ERROR_WRONG_NUMBER_OF_MATCHES: - printf(errorInMatches, keyLineInKeyFile); - printf(" --> %s\n", line); + fprintf(stderr, errorInMatches, keyLineInKeyFile); + fprintf(stderr, " --> %s\n", line); errorCode = ERROR_KEY_FILE_PARSE_NULL; break; case NO_ERROR_KEY_GREATER_THAN_ASKED: - printf("No asked key in line %u: %s\n", keyLineInKeyFile, line); + fprintf(stderr, "No asked key in line %u: %s\n", keyLineInKeyFile, line); break; case NO_ERROR_ISCOMMENT: - printf("Is comment in line %u: %s\n", keyLineInKeyFile, line); + fprintf(stderr, "Is comment in line %u: %s\n", keyLineInKeyFile, line); default: break; } @@ -246,12 +246,12 @@ char* EncKeys::decryptFile(const char* filename, const char *secret, int *errorC } long file_size = ftell(fp); // get the file size if (MAX_KEY_FILE_SIZE < file_size) { - printf(errorExceedKeyFileSize, filename, MAX_KEY_FILE_SIZE); + fprintf(stderr, errorExceedKeyFileSize, filename, MAX_KEY_FILE_SIZE); *errorCode = ERROR_KEY_FILE_TOO_BIG; return NULL; } else if (-1L == file_size) { - printf(errorFileSize, filename); + fprintf(stderr, errorFileSize, filename); *errorCode = ERROR_READING_FILE; return NULL; } @@ -278,7 +278,7 @@ char* EncKeys::decryptFile(const char* filename, const char *secret, int *errorC if(0 != res) { *errorCode = ERROR_FALSE_FILE_KEY; delete[] buffer; buffer = NULL; - printf(errorFalseFileKey, secret); + fprintf(stderr, errorFalseFileKey, secret); } else { memcpy(buffer, decrypted, d_size); @@ -305,6 +305,6 @@ bool EncKeys::isComment(const char *line) { void EncKeys::printKeyEntry( uint id) { keyentry *entry = getKeys(id); - if( NULL == entry) printf("No such keyID = %u\n", id); - else printf("Key: id:%3u \tiv:%s \tkey:%s\n", entry->id, entry->iv, entry->key); + if( NULL == entry) fprintf(stderr, "No such keyID = %u\n", id); + else fprintf(stderr, "Key: id:%3u \tiv:%s \tkey:%s\n", entry->id, entry->iv, entry->key); } From 598038d6fa53bad910ce2c5baf9c03240c2d66ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludger=20G=C3=B6ckel?= Date: Mon, 6 Oct 2014 16:34:59 +0200 Subject: [PATCH 24/70] removed name --- storage/xtradb/enc/EncKeys.cc | 18 ++---------------- storage/xtradb/enc/KeySingleton.cc | 2 +- storage/xtradb/enc/keyfile.c | 2 +- storage/xtradb/fil/fil0pageencryption.cc | 2 +- storage/xtradb/include/EncKeys.h | 2 +- storage/xtradb/include/KeySingleton.h | 2 +- storage/xtradb/include/fil0pageencryption.h | 2 +- storage/xtradb/include/fsp0pageencryption.h | 2 +- storage/xtradb/include/fsp0pageencryption.ic | 2 +- unittest/eperi/EperiKeySingleton-t.cc | 2 +- unittest/eperi/EperiKeySingleton-t.h | 2 +- unittest/eperi/pageenc-t.cc | 1 - unittest/eperi/pageenc-t.h | 1 - 13 files changed, 12 insertions(+), 28 deletions(-) diff --git a/storage/xtradb/enc/EncKeys.cc b/storage/xtradb/enc/EncKeys.cc index 8c3464295f5cb..fb4af0c3a5c10 100644 --- a/storage/xtradb/enc/EncKeys.cc +++ b/storage/xtradb/enc/EncKeys.cc @@ -17,7 +17,7 @@ @file EncKeys.cc A class to keep keys for encryption/decryption. - Created 09/15/2014 Florin Fugaciu + Created 09/15/2014 ***********************************************************************/ #include "EncKeys.h" @@ -29,21 +29,7 @@ #include -/* - Die Schlüsseldatei kann Fehler beinhalten. Folgende Fehler werden abgefangen: - 1. Doppelte SchlüsselIDs: - 1.1. Gleiche Schlüssel --> Meldung ausgeben mit dem Hinweis auf die Zeilennummern - 1.2. Ungleiche Schlüssel --> Meldung ausgeben mit dem Hinweis auf die Zeilennummern - und Wahl des Schlüssel mit der kleinsten Zeilennummer - 2. Schlüsseldatei ist zu groß --> Meldung ausgeben und abbrechen - 3. Fehler im Schlüssel --> Meldung ausgeben und Schlüssel auf Nicht-Vorhanden setzen. Meldungen: - 3.1. Schlüssel zu groß - 3.2. Keine Übereinstimmung des Schlüssels mit den Vorgaben - 4. Schlüsselserver noch nicht implementiert --> Meldung ausgeben mit dem Hinweis auf das - Nicht-lesen-können von verschlüsselten Tabellen und Spalten. - - Anmerkung: voerst keine Hinweise auf die Zeilennummern -*/ + const char* EncKeys::strMAGIC = "Salted__"; const int EncKeys::magicSize = strlen(strMAGIC); // 8 byte diff --git a/storage/xtradb/enc/KeySingleton.cc b/storage/xtradb/enc/KeySingleton.cc index 6a3ea1908c41e..1542eba24b2c9 100644 --- a/storage/xtradb/enc/KeySingleton.cc +++ b/storage/xtradb/enc/KeySingleton.cc @@ -17,7 +17,7 @@ @file KeySingleton.cc Implementation of single pattern to keep keys for encrypting/decrypting pages. -Created 09/13/2014 Florin Fugaciu +Created 09/13/2014 ***********************************************************************/ diff --git a/storage/xtradb/enc/keyfile.c b/storage/xtradb/enc/keyfile.c index 1afed215666f2..331bea360c68d 100644 --- a/storage/xtradb/enc/keyfile.c +++ b/storage/xtradb/enc/keyfile.c @@ -14,7 +14,7 @@ Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ /******************************************************************/ -//Author Clemens Doerrhoefer + #include #include diff --git a/storage/xtradb/fil/fil0pageencryption.cc b/storage/xtradb/fil/fil0pageencryption.cc index 111fd7371df46..acdbea33787d2 100644 --- a/storage/xtradb/fil/fil0pageencryption.cc +++ b/storage/xtradb/fil/fil0pageencryption.cc @@ -20,7 +20,7 @@ this program; if not, write to the Free Software Foundation, Inc., @file fil/fil0pageencryption.cc Implementation for page encryption file spaces. - Created 08/25/2014 Florin Fugaciu + Created 08/25/2014 ***********************************************************************/ #include "fil0fil.h" diff --git a/storage/xtradb/include/EncKeys.h b/storage/xtradb/include/EncKeys.h index f1f1bf2d04ed1..9dacfed9e7f23 100644 --- a/storage/xtradb/include/EncKeys.h +++ b/storage/xtradb/include/EncKeys.h @@ -17,7 +17,7 @@ @file EncKeys.h A structure and class to keep keys for encryption/decryption. -Created 09/15/2014 Florin Fugaciu +Created 09/15/2014 ***********************************************************************/ #ifndef ENCKEYS_H_ diff --git a/storage/xtradb/include/KeySingleton.h b/storage/xtradb/include/KeySingleton.h index 9fbd95e04aed1..40d73af1d2eba 100644 --- a/storage/xtradb/include/KeySingleton.h +++ b/storage/xtradb/include/KeySingleton.h @@ -17,7 +17,7 @@ @file KeySingletonPattern.h Implementation of single pattern to keep keys for encrypting/decrypting pages. -Created 09/13/2014 Florin Fugaciu +Created 09/13/2014 ***********************************************************************/ diff --git a/storage/xtradb/include/fil0pageencryption.h b/storage/xtradb/include/fil0pageencryption.h index 4d50d5088f393..bdb964cf792b0 100644 --- a/storage/xtradb/include/fil0pageencryption.h +++ b/storage/xtradb/include/fil0pageencryption.h @@ -35,7 +35,7 @@ this program; if not, write to the Free Software Foundation, Inc., @file include/fil0pageencryption.h Helper functions for encryption/decryption page data on to table space. -Created 08/25/2014 Florin Fugaciu +Created 08/25/2014 ***********************************************************************/ diff --git a/storage/xtradb/include/fsp0pageencryption.h b/storage/xtradb/include/fsp0pageencryption.h index 1abc38cfb1d14..b555454e3733c 100644 --- a/storage/xtradb/include/fsp0pageencryption.h +++ b/storage/xtradb/include/fsp0pageencryption.h @@ -21,7 +21,7 @@ @file include/fsp0pageencryption.h Helper functions for extracting/storing page encryption information to file space. -Created 08/28/2014 Florin Fugaciu +Created 08/28/2014 ***********************************************************************/ #ifndef FSP0PAGEENCRYPTION_H_ diff --git a/storage/xtradb/include/fsp0pageencryption.ic b/storage/xtradb/include/fsp0pageencryption.ic index 7e9b0d9eea455..aaf5b2b829b3e 100644 --- a/storage/xtradb/include/fsp0pageencryption.ic +++ b/storage/xtradb/include/fsp0pageencryption.ic @@ -20,7 +20,7 @@ Implementation for helper functions for encrypting/decrypting pages and atomic writes information to file space. -Created 08/28/2014 Florin Fugaciu +Created 08/28/2014 ***********************************************************************/ #include "fsp0fsp.h" diff --git a/unittest/eperi/EperiKeySingleton-t.cc b/unittest/eperi/EperiKeySingleton-t.cc index e64d5112b73b1..2fd174f8b4da0 100644 --- a/unittest/eperi/EperiKeySingleton-t.cc +++ b/unittest/eperi/EperiKeySingleton-t.cc @@ -2,7 +2,7 @@ @file EperiKeySingleton-t.cc Implementation of single pattern to keep keys for encrypting/decrypting pages. -Created 09/15/2014 Florin Fugaciu +Created 09/15/2014 ***********************************************************************/ diff --git a/unittest/eperi/EperiKeySingleton-t.h b/unittest/eperi/EperiKeySingleton-t.h index f50bb8b543192..f955686322f87 100644 --- a/unittest/eperi/EperiKeySingleton-t.h +++ b/unittest/eperi/EperiKeySingleton-t.h @@ -2,7 +2,7 @@ @file EperiKeySingleton-t.h Implementation of single pattern to keep keys for encrypting/decrypting pages. -Created 09/15/2014 Florin Fugaciu +Created 09/15/2014 ***********************************************************************/ #ifndef EPERIKEYSINGLETON_T_H_ diff --git a/unittest/eperi/pageenc-t.cc b/unittest/eperi/pageenc-t.cc index 397831eeabf83..6e26b228d953c 100644 --- a/unittest/eperi/pageenc-t.cc +++ b/unittest/eperi/pageenc-t.cc @@ -2,7 +2,6 @@ * pageenc.cc * * Created on: 23.08.2014 - * Author: florin */ //#define UNIV_INLINE typedef unsigned char byte; diff --git a/unittest/eperi/pageenc-t.h b/unittest/eperi/pageenc-t.h index aff6b99aa723a..d1bb23923a183 100644 --- a/unittest/eperi/pageenc-t.h +++ b/unittest/eperi/pageenc-t.h @@ -2,7 +2,6 @@ * johnny.h * * Created on: 23.08.2014 - * Author: florin */ #ifndef JOHNNY_H_ From 50437623abf7c21939a5da4c0c35b5b8741e3381 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludger=20G=C3=B6ckel?= Date: Tue, 7 Oct 2014 09:29:23 +0200 Subject: [PATCH 25/70] revert enabling page compression method --- storage/xtradb/fil/fil0pagecompress.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/xtradb/fil/fil0pagecompress.cc b/storage/xtradb/fil/fil0pagecompress.cc index 8c4b2cb13aa54..7e141f0ab2e79 100644 --- a/storage/xtradb/fil/fil0pagecompress.cc +++ b/storage/xtradb/fil/fil0pagecompress.cc @@ -268,7 +268,7 @@ fil_compress_page( int level = 0; ulint header_len = FIL_PAGE_DATA + FIL_PAGE_COMPRESSED_SIZE; ulint write_size=0; - ulint comp_method = 1;//innodb_compression_algorithm; + ulint comp_method = innodb_compression_algorithm; /* Cache to avoid change during function execution */ From a68f779b7b76a4b15dbd02545146d9ac9f719193 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Clemens=20D=C3=B6rrh=C3=B6fer?= Date: Wed, 8 Oct 2014 10:47:48 +0200 Subject: [PATCH 26/70] Secret can now be delivered via a File by prepending FILE: magic. Use innodb_data_encryption_filekey prepend FILE: to your option. MariaDB will then interpret the secret as Filename, which is located at the provider url. The size of your secret is limited to 256 characters. After reading the secret from the file it will be overwritten such that it cannot be read after server start. --- storage/xtradb/enc/EncKeys.cc | 50 +++++++++++++++++++++++++---- storage/xtradb/handler/ha_innodb.cc | 2 -- storage/xtradb/include/EncKeys.h | 6 ++-- 3 files changed, 48 insertions(+), 10 deletions(-) diff --git a/storage/xtradb/enc/EncKeys.cc b/storage/xtradb/enc/EncKeys.cc index ae287928212fd..55883ddcdf60b 100644 --- a/storage/xtradb/enc/EncKeys.cc +++ b/storage/xtradb/enc/EncKeys.cc @@ -99,18 +99,56 @@ bool EncKeys::initKeys(const char *name, const char *url, const int initType, co } int EncKeys::initKeysThroughFile(const char *name, const char *path, const char *filekey) { + size_t len1 = strlen(path); size_t len2 = strlen(name); - bool isSlash = ('/' == path[len1 - 1]); + const char *MAGIC = "FILE:"; + const short MAGIC_LEN = 5; int ret = NO_ERROR_KEY_FILE_PARSE_OK; - char *filename = new char[len1 + len2 + isSlash ? 1 : 2]; - - sprintf(filename, "%s%s%s", path, isSlash ? "" : "/", name); - ret = parseFile((const char *)filename, 254, filekey); - delete[] filename; filename = NULL; + #ifdef TARGET_OS_LINUX + bool isSlash = ('/' == path[len1 - 1]); + char *secret = (char*) malloc(MAX_SECRET_SIZE * sizeof(char)); + char *filename = (char*) malloc((len1 + len2 + isSlash ? 1 : 2) * sizeof(char)); + if(filekey != NULL) + { + //If secret starts with FILE: interpret the secret as filename. + if(memcmp(MAGIC, filekey, MAGIC_LEN) == 0) { + int fk_len = strlen(filekey); + char *secretfile = (char*)malloc((len1 + (fk_len - MAGIC_LEN) + isSlash ? 1 : 2)* sizeof(char)); + sprintf(secretfile, "%s%s%s", path, isSlash ? "" : "/", filekey+MAGIC_LEN); + parseSecret(secretfile, secret); + free(secretfile); + }else + { + sprintf(secret, "%s", filekey); + } + } + sprintf(filename, "%s%s%s", path, isSlash ? "" : "/", name); + ret = parseFile((const char *)filename, 254, secret); + free(filename); + free(secret); + #endif //TARGET_OS_LINUX + #ifdef __WIN__ + ut_ad(false); + #endif //__WIN__e return ret; } +void EncKeys::parseSecret( const char *secretfile, char *secret ) { + int i=0; + FILE *fp = my_fopen(secretfile, O_RDWR, MYF(MY_WME)); + fseek(fp, 0L, SEEK_END); + long file_size = ftell(fp); + fseek(fp, 0L, SEEK_SET); + fgets(secret, (MAX_SECRET_SIZE >= file_size)?file_size:MAX_SECRET_SIZE, fp); + fseek(fp, 0L, SEEK_SET); + for(i=0; iid, entry->iv, entry->key); #ifndef DBUG_OFF static const char test_filename[] = "-@"; diff --git a/storage/xtradb/include/EncKeys.h b/storage/xtradb/include/EncKeys.h index f1f1bf2d04ed1..41316616f3098 100644 --- a/storage/xtradb/include/EncKeys.h +++ b/storage/xtradb/include/EncKeys.h @@ -41,8 +41,9 @@ class EncKeys private: static const char *strMAGIC, *newLine; static const int magicSize; + static const size_t MAX_SECRET_SIZE = 256; - enum constants { MAX_OFFSETS_IN_PCRE_PATTERNS = 30 }; + enum constants { MAX_OFFSETS_IN_PCRE_PATTERNS = 30}; enum keyAttributes { KEY_MIN = 1, KEY_MAX = 255, MAX_KEYS = 255, MAX_IVLEN = 256, MAX_KEYLEN = 512, ivSize16 = 16, keySize32 = 32 }; enum keyInitType { KEYINITTYPE_FILE = 1, KEYINITTYPE_SERVER = 2 }; @@ -65,12 +66,13 @@ class EncKeys char * decryptFile( const char* filename, const char *secret, int *errorCode); int parseFile( const char* filename, const uint maxKeyId, const char *secret); int parseLine( const char *line, const uint maxKeyId); + void parseSecret( const char *filename, char *secret ); public: enum errorCodesFile { NO_ERROR_KEY_FILE_PARSE_OK = 0, ERROR_KEY_FILE_PARSE_NULL = 110, ERROR_KEY_FILE_TOO_BIG = 120, ERROR_KEY_FILE_EXCEEDS_MAX_NUMBERS_OF_KEYS = 130, ERROR_OPEN_FILE = 140, ERROR_READING_FILE = 150, ERROR_FALSE_FILE_KEY = 160, - ERROR_KEYINITTYPE_SERVER_NOT_IMPLEMENTED = 170 }; + ERROR_KEYINITTYPE_SERVER_NOT_IMPLEMENTED = 170, ERROR_ENCRYPTION_SECRET_NULL = 180 }; EncKeys(); virtual ~EncKeys(); bool initKeys( const char *name, const char *url, const int initType, const char *filekey); From efa2372f748b90c49df2952b741a8fdc0e692a0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Clemens=20D=C3=B6rrh=C3=B6fer?= Date: Wed, 8 Oct 2014 11:16:01 +0200 Subject: [PATCH 27/70] Removed storage/enc/keyfile.c --- storage/xtradb/CMakeLists.txt | 1 - storage/xtradb/enc/keyfile.c | 178 ---------------------------------- 2 files changed, 179 deletions(-) delete mode 100644 storage/xtradb/enc/keyfile.c diff --git a/storage/xtradb/CMakeLists.txt b/storage/xtradb/CMakeLists.txt index a5d3baa10fd7c..fb1eff7cc3045 100644 --- a/storage/xtradb/CMakeLists.txt +++ b/storage/xtradb/CMakeLists.txt @@ -315,7 +315,6 @@ SET(INNOBASE_SOURCES dyn/dyn0dyn.cc enc/EncKeys.cc enc/KeySingleton.cc - enc/keyfile.c eval/eval0eval.cc eval/eval0proc.cc fil/fil0fil.cc diff --git a/storage/xtradb/enc/keyfile.c b/storage/xtradb/enc/keyfile.c deleted file mode 100644 index 331bea360c68d..0000000000000 --- a/storage/xtradb/enc/keyfile.c +++ /dev/null @@ -1,178 +0,0 @@ -/* Copyright (C) 2014 eperi GmbH. All Rights Reserved. - - This program is free software; you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation; version 2 of the License. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program; if not, write to the Free Software - Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA */ - -/******************************************************************/ - - -#include -#include -#include -#include -#include - -#define E_WRONG_NUMBER_OF_MATCHES 10 -#define MAX_KEY_FILE_SIZE 1048576 -#define MAX_BUFFER_LENGTH 512 - -#define KEY_FILE_PARSE_OK 0 -#define KEY_FILE_TOO_BIG 100 -#define KEY_BUFFER_TOO_BIG 200 -#define KEY_FILE_PARSE_NULL 300 -#define KEY_FILE_TOO_MANY_KEYS 400 - - -int -isComment(char *line) -{ - const char *error_p; - int offset; - int m_len = (int) strlen(line); - - pcre *pattern = pcre_compile( - "\\s*#.*", - 0, - &error_p, - &offset, - NULL); - int rc,i; - int ovector[30]; - rc = pcre_exec( - pattern, - NULL, - line, - m_len, - 0, - 0, - ovector, - 30 - ); - if(rc < 0) { - return 0; - } else { - return 1; - } -} - -int -parseFile(FILE * fp, struct keyentry **allKeys, const int k_len, const char *secret) -{ - const char *MAGIC = "Salted__"; - long file_size = 0; - char *buffer, *decrypted; - char *line = NULL; - if(NULL == fp) { - fprintf(stderr, "Key file not found.\n"); - return 100; - } - - //get size of file - fseek(fp, 0L, SEEK_END); - file_size = ftell(fp); - fseek(fp, 0L, SEEK_SET); - - if(file_size > MAX_KEY_FILE_SIZE) { - return KEY_FILE_TOO_BIG; - } - - //Read file into buffer - buffer = (char*) malloc((file_size+1)*sizeof(char)); - fread(buffer, file_size, 1, fp); - - //Check for file encryption - if(memcmp(buffer, MAGIC, 8) == 0) { //If file is encrypted, decrypt it first. - unsigned char salt[8]; - unsigned char *key = malloc(32 * sizeof(char)); - unsigned char *iv = malloc(16 * sizeof(char)); - decrypted = malloc(file_size * sizeof(char)); - memcpy(&salt, buffer+8, 8); - my_bytes_to_key(&salt, secret, key, iv); - unsigned long int d_size = 0; - my_aes_decrypt_cbc(buffer + 16, file_size -16, decrypted, &d_size, key, 32, iv, 16); - memcpy(buffer, decrypted, d_size); - - free(decrypted); - free(key); - free(iv); - } - - line = strtok(buffer, "\n"); - while(line != NULL) { - struct keyentry *entry = (struct keyentry*) malloc(sizeof(struct keyentry)); - if( parseLine(line, entry, k_len) == 0) { - allKeys[entry->id] = entry; - } - line = strtok(NULL, "\n"); - } - free(buffer); - return KEY_FILE_PARSE_OK; -} - -int -parseLine(const char *line, struct keyentry *entry, const int k_len) -{ - const char *error_p; - int offset; - - pcre *pattern = pcre_compile( - "([0-9]+);([0-9,a-f,A-F]{32});([0-9,a-f,A-F]{64}|[0-9,a-f,A-F]{48}|[0-9,a-f,A-F]{32})", - 0, - &error_p, - &offset, - NULL); - if( error_p != NULL ) { - fprintf(stderr, "Error: %s\n", error_p); - fprintf(stderr, "Offset: %d\n", offset); - } - int m_len = (int) strlen(line); - char *buffer = (char*) malloc(MAX_BUFFER_LENGTH*sizeof(char)); - int rc,i; - int ovector[30]; - rc = pcre_exec( - pattern, - NULL, - line, - m_len, - 0, - 0, - ovector, - 30 - ); - if(rc == 4 && !isComment(line)) { - char *substring_start = line + ovector[2]; - int substr_length = ovector[3] - ovector[2]; - sprintf( buffer, "%.*s", substr_length, substring_start ); - entry->id = atoi(buffer); - if(entry->id >= k_len) - return KEY_FILE_TOO_MANY_KEYS; - - substring_start = line + ovector[4]; - substr_length = ovector[5] - ovector[4]; - entry->iv = malloc(substr_length*sizeof(char)); - - sprintf( entry->iv, "%.*s", substr_length, substring_start ); - - substring_start = line + ovector[6]; - substr_length = ovector[7] - ovector[6]; - entry->key = malloc(substr_length*sizeof(char)); - sprintf( entry->key, "%.*s", substr_length, substring_start ); - } else - { - return E_WRONG_NUMBER_OF_MATCHES; - } - if(entry->id == NULL || entry->iv == NULL || entry->key == NULL) { - return KEY_FILE_PARSE_NULL; - } - return KEY_FILE_PARSE_OK; -} From ee219c5037482dae06e0c56f1aa79fed14182965 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Clemens=20D=C3=B6rrh=C3=B6fer?= Date: Wed, 8 Oct 2014 11:52:02 +0200 Subject: [PATCH 28/70] Code cleanup removed compiler warnings --- storage/xtradb/fil/fil0pageencryption.cc | 3 +-- storage/xtradb/include/dict0dict.ic | 3 --- storage/xtradb/include/fsp0fsp.ic | 4 +--- storage/xtradb/include/fsp0pageencryption.h | 2 +- storage/xtradb/include/fsp0pageencryption.ic | 7 ++----- 5 files changed, 5 insertions(+), 14 deletions(-) diff --git a/storage/xtradb/fil/fil0pageencryption.cc b/storage/xtradb/fil/fil0pageencryption.cc index acdbea33787d2..f4e1769dd6b5a 100644 --- a/storage/xtradb/fil/fil0pageencryption.cc +++ b/storage/xtradb/fil/fil0pageencryption.cc @@ -72,7 +72,6 @@ ulint mode ulint page_size = UNIV_PAGE_SIZE; ulint orig_page_type = 0; ulint write_size = 0; - ib_uint64_t flush_lsn = 0; ib_uint32_t checksum = 0; ulint offset_ctrl_data = 0; fil_space_t* space = NULL; @@ -456,7 +455,7 @@ al_size); fprintf(stderr, "InnoDB: Corruption: Page is marked as encrypted\n" "InnoDB: but decrypt failed with error %d.\n" "InnoDB: size %lu len %lu, key%d\n", err, data_size, - len, page_encryption_key); + len, (int)page_encryption_key); fflush(stderr); if (NULL == page_buf) { ut_free(in_buf); diff --git a/storage/xtradb/include/dict0dict.ic b/storage/xtradb/include/dict0dict.ic index 6a54cf5719063..39c2f77a905ec 100644 --- a/storage/xtradb/include/dict0dict.ic +++ b/storage/xtradb/include/dict0dict.ic @@ -699,9 +699,6 @@ dict_sys_tables_type_validate( ulint page_compression_level = DICT_TF_GET_PAGE_COMPRESSION_LEVEL(type); ulint atomic_writes = DICT_TF_GET_ATOMIC_WRITES(type); - ulint page_encryption = DICT_TF_GET_PAGE_ENCRYPTION(type); - ulint page_encryption_key = DICT_TF_GET_PAGE_ENCRYPTION_KEY(type); - ut_a(atomic_writes >= 0 && atomic_writes <= ATOMIC_WRITES_OFF); /* The low order bit of SYS_TABLES.TYPE is always set to 1. diff --git a/storage/xtradb/include/fsp0fsp.ic b/storage/xtradb/include/fsp0fsp.ic index 368b2d73404c3..8352044613719 100644 --- a/storage/xtradb/include/fsp0fsp.ic +++ b/storage/xtradb/include/fsp0fsp.ic @@ -66,9 +66,7 @@ fsp_flags_is_valid( ulint unused = FSP_FLAGS_GET_UNUSED(flags); ulint page_compression = FSP_FLAGS_GET_PAGE_COMPRESSION(flags); ulint page_compression_level = FSP_FLAGS_GET_PAGE_COMPRESSION_LEVEL(flags); - ulint page_encryption = FSP_FLAGS_GET_PAGE_ENCRYPTION(flags); - ulint page_encryption_key = FSP_FLAGS_GET_PAGE_ENCRYPTION_KEY(flags); - ulint atomic_writes = FSP_FLAGS_GET_ATOMIC_WRITES(flags); + ulint atomic_writes = FSP_FLAGS_GET_ATOMIC_WRITES(flags); DBUG_EXECUTE_IF("fsp_flags_is_valid_failure", return(false);); diff --git a/storage/xtradb/include/fsp0pageencryption.h b/storage/xtradb/include/fsp0pageencryption.h index b555454e3733c..42dac18e60abf 100644 --- a/storage/xtradb/include/fsp0pageencryption.h +++ b/storage/xtradb/include/fsp0pageencryption.h @@ -1,6 +1,6 @@ /***************************************************************************** -/* Copyright (C) 2014 eperi GmbH. All Rights Reserved. + Copyright (C) 2014 eperi GmbH. All Rights Reserved. This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by diff --git a/storage/xtradb/include/fsp0pageencryption.ic b/storage/xtradb/include/fsp0pageencryption.ic index aaf5b2b829b3e..43288297972e7 100644 --- a/storage/xtradb/include/fsp0pageencryption.ic +++ b/storage/xtradb/include/fsp0pageencryption.ic @@ -110,9 +110,6 @@ ibool fil_page_is_encrypted( /*===================*/ const byte *buf) /*!< in: page */ -{ - //ibool result = FALSE; - ibool result = TRUE; - return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_ENCRYPTED); - //return(result); +{ + return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_ENCRYPTED); } From 036d52e8a9cb319f21c1ecf4593b60b50fe07330 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Clemens=20D=C3=B6rrh=C3=B6fer?= Date: Wed, 8 Oct 2014 14:25:42 +0200 Subject: [PATCH 29/70] Unittest cleanup --- unittest/eperi/eperi_aes-t.c | 89 +++++++++++++++++++++--------------- unittest/eperi/pageenc-t.cc | 35 ++++++++------ 2 files changed, 73 insertions(+), 51 deletions(-) diff --git a/unittest/eperi/eperi_aes-t.c b/unittest/eperi/eperi_aes-t.c index 06d79ef14b482..17bbfd4178fdd 100644 --- a/unittest/eperi/eperi_aes-t.c +++ b/unittest/eperi/eperi_aes-t.c @@ -61,7 +61,7 @@ fseek(fileptr, 0, SEEK_END); // Jump to the end of the file filelen = ftell(fileptr); // Get the current byte offset in the file rewind(fileptr); // Jump back to the beginning of the file -buffer = (char *)malloc((filelen+1)*sizeof(char)); // Enough memory for file + \0 +buffer = (byte *)malloc((filelen+1)*sizeof(char)); // Enough memory for file + \0 fread(buffer, filelen, 1, fileptr); // Read in the entire file fclose(fileptr); // Close the file return buffer; @@ -73,16 +73,19 @@ test_cbc_wrong_keylength() plan(2); char* source = "Joshua: Shall we play a game"; ulint s_len = (ulint)strlen(source); - char* key="899C0ECB592B2CEE46E64191B6E6DE9B97D8A8EEA43BEF78"; + unsigned char key[24] = {0x89, 0x9c, 0x0e, 0xcb, 0x59, 0x2b, 0x2c, + 0xee, 0x46, 0xe6, 0x41, 0x91, 0xb6, 0xe6, 0xde, 0x9b, 0x97, + 0xd8, 0xa8, 0xee, 0xa4, 0x3b, 0xef, 0x78 }; uint8 k_len = 6; - char* iv = "F0974007D619466B9EBF8D4F6E302AA3"; + unsigned char iv[16] = {0xf0, 0x97, 0x40, 0x07, 0xd6, 0x19, 0x46, 0x6b, + 0x9e, 0xbf, 0x8d, 0x4f, 0x6e, 0x30, 0x2a, 0xa3}; uint8 i_len = 16; char* dest = (char *) malloc(2*s_len*sizeof(char)); unsigned long int dest_len = 0; - int rc = my_aes_encrypt_cbc(source, s_len, dest, &dest_len, key, k_len, iv, i_len); + int rc = my_aes_encrypt_cbc(source, s_len, dest, &dest_len,(unsigned char*) &key, k_len,(unsigned char*) &iv, i_len); ok(rc == -5, "Encryption - wrong keylength was detected."); - rc = my_aes_decrypt_cbc(source, s_len, dest, &dest_len, key, k_len, iv, i_len); + rc = my_aes_decrypt_cbc(source, s_len, dest, &dest_len,(unsigned char*) &key, k_len,(unsigned char*) &iv, i_len); ok(rc == -5, "Decryption - wrong keylength was detected."); } @@ -92,23 +95,30 @@ test_cbc_keysizes() plan(2); char* source = MY_AES_TEST_JOSHUA; ulint s_len = (ulint)strlen(source); - char* key="899C0ECB592B2CEE46E64191B6E6DE9B97D8A8EEA43BEF78"; + unsigned char key[24] = {0x89, 0x9c, 0x0e, 0xcb, 0x59, 0x2b, 0x2c, + 0xee, 0x46, 0xe6, 0x41, 0x91, 0xb6, 0xe6, 0xde, 0x9b, 0x97, + 0xd8, 0xa8, 0xee, 0xa4, 0x3b, 0xef, 0x78 }; uint8 k_len = 24; - char* iv = "F0974007D619466B9EBF8D4F6E302AA3"; + unsigned char iv[16] = {0xf0, 0x97, 0x40, 0x07, 0xd6, 0x19, 0x46, 0x6b, + 0x9e, 0xbf, 0x8d, 0x4f, 0x6e, 0x30, 0x2a, 0xa3}; uint8 i_len = 16; char* dest = (char *) malloc(2*s_len*sizeof(char)); ulint dest_len = 0; my_aes_encrypt_cbc(source, s_len, dest, &dest_len, key, k_len, iv, i_len); source = (char *) malloc(strlen(MY_AES_TEST_TEXTBLOCK) * sizeof(char)); - my_aes_decrypt_cbc(dest , strlen(dest), source, &dest_len, key, k_len, iv, i_len); + my_aes_decrypt_cbc(dest , strlen(dest), source, &dest_len,(unsigned char*) &key, k_len,(unsigned char*) &iv, i_len); ok(strcmp(source, MY_AES_TEST_JOSHUA),"Decrypted text is identical to original text."); - key="7B3B8DA94B77F91A6E05037B21AD5F6E86BD4657C45D97BC7FF14313A781B5A3"; + unsigned char key2[32] = {0x7b, 0x3b, 0x8d, 0xa9, 0x4b, 0x77, + 0xf9, 0x1a, 0x6e, 0x05, 0x03, 0x7b, + 0x21, 0xad, 0x5f, 0x6e, 0x86, 0xbd, + 0x46, 0x57, 0xc4, 0x5d, 0x97, 0xbc, + 0xb5, 0xa3}; k_len = 32; dest = (char *) malloc(2*s_len*sizeof(char)); - my_aes_encrypt_cbc(source, s_len, dest, &dest_len, key, k_len, iv, i_len); + my_aes_encrypt_cbc(source, s_len, dest, &dest_len, (unsigned char*) &key2, k_len,(unsigned char*) &iv, i_len); source = (char *) malloc(strlen(MY_AES_TEST_TEXTBLOCK) * sizeof(char)); - my_aes_decrypt_cbc(dest , strlen(dest), source, &dest_len, key, k_len, iv, i_len); + my_aes_decrypt_cbc(dest , strlen(dest), source, &dest_len, (unsigned char*) &key2, k_len, (unsigned char*) &iv, i_len); ok(strcmp(source, MY_AES_TEST_JOSHUA),"Decrypted text is identical to original text."); free(source); free(dest); @@ -121,17 +131,17 @@ test_cbc_large() char* source = MY_AES_TEST_TEXTBLOCK; ulint s_len = (ulint)strlen(source); - char* key = "3C5DC9153A6FE5F22516E217C1603BF7"; + unsigned char key[16] = {0x3c, 0x5d, 0xc9, 0x15, 0x3a, 0x6f, 0xe5, 0xf2, + 0x25, 0x16, 0xe2, 0x17, 0xc1, 0x60, 0x3b, 0xf7}; uint8 k_len = 16; - char* iv = "F0974007D619466B9EBF8D4F6E302AA3"; + unsigned char iv[16] = {0xf0, 0x97, 0x40, 0x00, 0x7d, 0x61, 0x94, 0x66, + 0xb9, 0xeb, 0xf8, 0xd4, 0x6e, 0x30, 0x2a, 0xa3}; uint8 i_len = 16; char* dest = (char *) malloc( 2* s_len * sizeof(char)); ulint dest_len = 0; - dump_buffer(10,source); - dump_buffer(10,dest); my_aes_encrypt_cbc(source, s_len, dest, &dest_len, key, k_len, iv, i_len); source = (char *) malloc(strlen(MY_AES_TEST_TEXTBLOCK) * sizeof(char)); - my_aes_decrypt_cbc(dest , strlen(dest), source, &dest_len, key, k_len, iv, i_len); + my_aes_decrypt_cbc(dest , strlen(dest), source, &dest_len, (unsigned char*) &key, k_len, (unsigned char*) &iv, i_len); ok(strcmp(source, MY_AES_TEST_TEXTBLOCK),"Decrypted text is identical to original text."); free(source); free(dest); @@ -143,18 +153,18 @@ test_wrong_key() plan(1); char* source = MY_AES_TEST_TEXTBLOCK; ulint s_len = (ulint)strlen(source); - - char* key = "3C5DC9153A6FE5F22516E217C1603BF7"; + unsigned char key[16] = {0x3c, 0x5d, 0xc9, 0x15, 0x3a, 0x6f, 0xe5, 0xf2, + 0x25, 0x16, 0xe2, 0x17, 0xc1, 0x60, 0x3b, 0xf7}; uint8 k_len = 16; - char* iv = "F0974007D619466B9EBF8D4F6E302AA3"; + unsigned char iv[16] = {0xf0, 0x97, 0x40, 0x00, 0x7d, 0x61, 0x94, 0x66, + 0xb9, 0xeb, 0xf8, 0xd4, 0x6e, 0x30, 0x2a, 0xa3}; uint8 i_len = 16; char* dest = (char *) malloc( 2* s_len * sizeof(char)); ulint dest_len = 0; - dump_buffer(10,source); - dump_buffer(10,dest); - my_aes_encrypt_cbc(source, s_len, dest, &dest_len, key, k_len, iv, i_len); + my_aes_encrypt_cbc(source, s_len, dest, &dest_len, (unsigned char*) &key, k_len,(unsigned char*) &iv, i_len); - iv = "F1A74007D619455B9EBF8D4F6E302AA3"; + iv[0] = 0xf1; + //"F1A74007D619455B9EBF8D4F6E302AA3"; source = (char *) malloc(strlen(MY_AES_TEST_TEXTBLOCK) * sizeof(char)); my_aes_decrypt_cbc(dest , strlen(dest), source, &dest_len, key, k_len, iv, i_len); ok(strcmp(source, MY_AES_TEST_TEXTBLOCK) != 0,"Using wrong iv results in wrong decryption."); @@ -177,16 +187,18 @@ test_cbc() dest[i]=0; } ulint dest_len = 0; - char* key = "583BE7F334F85E7D9DDB362E9AC38151"; + unsigned char key[16] = {0x58, 0x3b, 0xe7, 0xf3, 0x34, 0xf8, + 0x5e, 0x7d, 0x9d, 0xdb, 0x36, 0x2e, 0x9a, 0xc3, 0x81, 0x51}; uint8 k_len = 16; - char* iv = "3325CC3F02203FB6B849990042E58BCB"; + unsigned char iv[16] = {0x33, 0x25, 0xcc, 0x3f, 0x02, 0x20, 0x3f, 0xb6, 0xb8, + 0x49, 0x99, 0x00, 0x42, 0xe5, 0x8b, 0xcb}; uint8 i_len = 16; - int ec = my_aes_encrypt_cbc(source, s_len, &dest, &dest_len, key, k_len, iv, i_len); + int ec = my_aes_encrypt_cbc(source, s_len, (char*) &dest, &dest_len, (unsigned char*) &key, k_len,(unsigned char*) &iv, i_len); ok(ec == AES_OK, "Checking return code."); for(i=0; i<20; i++) { source[i] = 0; } - my_aes_decrypt_cbc(dest , dest_len, &source, &dest_len, key, k_len, iv, i_len); + my_aes_decrypt_cbc(dest , dest_len, (char*)&source, &dest_len, (unsigned char*) &key, k_len, (unsigned char*) &iv, i_len); ok(strcmp(source, "Beam me up, Scotty."),"Decrypted text is identical to original text."); } @@ -196,15 +208,18 @@ test_cbc_resultsize() { plan(2); char *source = (char*) malloc(5000*sizeof(char)); - source = "abcdefghijklmnopqrstfjdklfkjdsljsdlkfjsaklföjsfölkdsjfölsdkjklösjsdklfjdsklöfjsdalökfjdsklöjfölksdjfklösdajfklösdaj"; + source = "abcdefghijklmnopqrstfjdklfkjdsljsdlkfjsaklföjsfölkdsjfölsd" + "kjklösjsdklfjdsklöfjsdalökfjdsklöjfölksdjfklösdajfklösdaj"; ulint s_len = (ulint) strlen(source); char* dest = (char *) malloc(2 * s_len * sizeof(char)); ulint d_len = 0; - char* key = "583BE7F334F85E7D9DDB362E9AC38151"; + unsigned char key[16] = {0x58, 0x3b, 0xe7, 0xf3, 0x34, 0xf8, + 0x5e, 0x7d, 0x9d, 0xdb, 0x36, 0x2e, 0x9a, 0xc3, 0x81, 0x51}; uint8 k_len = 16; - char* iv = "3325CC3F02203FB6B849990042E58BCB"; + unsigned char iv[16] = {0x33, 0x25, 0xcc, 0x3f, 0x02, 0x20, 0x3f, 0xb6, 0xb8, + 0x49, 0x99, 0x00, 0x42, 0xe5, 0x8b, 0xcb}; uint8 i_len = 16; - my_aes_encrypt_cbc(source, s_len, dest, &d_len, key, k_len, iv, i_len); + my_aes_encrypt_cbc(source, s_len, dest, &d_len, (unsigned char*)&key, k_len, (unsigned char *)&iv, i_len); ok(d_len==128, "Destination length ok."); } @@ -308,7 +323,7 @@ ok (i==0, "in==out"); void test_page_enc_dec() { - char* buf = readFile("xaa"); + unsigned char* buf = readFile("xaa"); char* dest = (char *) malloc(16384*sizeof(char)); //fil_encrypt_page(0,buf,dest,0,0,NULL,1); @@ -325,18 +340,18 @@ void test_bytes_to_key() { plan(2); - char salt[] = {0x0c, 0x3b, 0x72, 0x1b, 0xfe, 0x07, 0xe2, 0xb3}; + unsigned char salt[] = {0x0c, 0x3b, 0x72, 0x1b, 0xfe, 0x07, 0xe2, 0xb3}; char *secret = "secret"; char key[32]; - char iv[16]; - char keyresult[32] = {0x2E, 0xFF, 0xB7, 0x1D, 0xDB, 0x97, 0xA8, 0x3A, + unsigned char iv[16]; + unsigned char keyresult[32] = {0x2E, 0xFF, 0xB7, 0x1D, 0xDB, 0x97, 0xA8, 0x3A, 0x03, 0x5A, 0x06, 0xDF, 0xB0, 0xDD, 0x72, 0x29, 0xA6, 0xD9, 0x1F, 0xFB, 0xE6, 0x06, 0x3B, 0x4B, 0x81, 0x23, 0x85, 0x45, 0x71, 0x28, 0xFF, 0x1F}; - char ivresult[16] = {0x61, 0xFF, 0xC8, 0x27, 0x5B, 0x46, 0x4C, 0xBD, + unsigned char ivresult[16] = {0x61, 0xFF, 0xC8, 0x27, 0x5B, 0x46, 0x4C, 0xBD, 0x55, 0x82, 0x0E, 0x54, 0x8F, 0xE4, 0x44, 0xD9}; - my_bytes_to_key(&salt, secret, &key, &iv); + my_bytes_to_key((unsigned char*) &salt, secret, (unsigned char*) &key, (unsigned char*) &iv); ok(memcmp(key, &keyresult, 32) == 0, "BytesToKey key generated successfully."); ok(memcmp(iv, &ivresult, 16) == 0, "BytesToKey iv generated successfully."); diff --git a/unittest/eperi/pageenc-t.cc b/unittest/eperi/pageenc-t.cc index 6e26b228d953c..488ac0a3a2541 100644 --- a/unittest/eperi/pageenc-t.cc +++ b/unittest/eperi/pageenc-t.cc @@ -81,7 +81,6 @@ void testIt(char* filename, ulint cmp_checksum) { byte* dest = (byte *) malloc(16384*sizeof(byte)); ulint out_len; fil_encrypt_page(0,buf,dest,0,255, &out_len, 1); - ulint flags = 0; fil_decrypt_page(NULL, dest, 16384 ,NULL,NULL, 1); ulint a = 0; ulint b = 0; @@ -95,21 +94,29 @@ void testIt(char* filename, ulint cmp_checksum) { strcpy (str,"File "); strcat (str,filename ); - ok (i==0, str); + ok (i==0, "%s", (char*) str); } void test_page_enc_dec() { - - testIt("compressed",0); - testIt("compressed_full",0); - - testIt("xaa",0); - testIt("xab",0); - testIt("xac",0); - testIt("xad",0); - - - testIt("xae",1); - testIt("xaf",1); + char compressed[] = "compressed"; + char compressed_full[] = "compressed_full"; + char xaa[] = "xaa"; + char xab[] = "xab"; + char xac[] = "xac"; + char xad[] = "xad"; + char xae[] = "xae"; + char xaf[] = "xaf"; + + testIt(compressed,0); + testIt(compressed_full,0); + + testIt(xaa,0); + testIt(xab,0); + testIt(xac,0); + testIt(xad,0); + + + testIt(xae,1); + testIt(xaf,1); } From 30f03082b5140e917caf03d5d1e0acbf05572e21 Mon Sep 17 00:00:00 2001 From: unknown Date: Wed, 8 Oct 2014 15:38:34 +0200 Subject: [PATCH 30/70] Windows build essentials --- CMakeLists.txt | 1 + dbug/dbug.c | 2 +- include/mysql/plugin.h | 18 ++++++++++++++++++ sql/CMakeLists.txt | 1 + sql/mysqld.cc | 6 ++++++ sql/sql_plugin.h | 6 ++---- storage/xtradb/CMakeLists.txt | 1 - storage/xtradb/enc/EncKeys.cc | 25 +++++++++++++++---------- storage/xtradb/include/EncKeys.h | 11 ++++++----- storage/xtradb/include/os0file.h | 11 ++++++----- storage/xtradb/os/os0file.cc | 11 ++++++----- unittest/eperi/CMakeLists.txt | 5 +++++ unittest/eperi/EperiKeySingleton-t.cc | 1 - unittest/eperi/eperi_aes-t.c | 15 +++++++++++++-- 14 files changed, 80 insertions(+), 34 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 22d8569cb444b..1927147b1842a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -329,6 +329,7 @@ SET(CMAKE_INCLUDE_DIRECTORIES_PROJECT_BEFORE ON) ADD_DEFINITIONS(-DHAVE_CONFIG_H) INCLUDE_DIRECTORIES(${CMAKE_CURRENT_BINARY_DIR}/include) + # Add bundled or system zlib. MYSQL_CHECK_ZLIB_WITH_COMPRESS() # Add bundled yassl/taocrypt or system openssl. diff --git a/dbug/dbug.c b/dbug/dbug.c index b26500ee6a515..93e7957366247 100644 --- a/dbug/dbug.c +++ b/dbug/dbug.c @@ -2201,7 +2201,7 @@ while (n-- > 0) { if (on_this_line == 16 || n == 0) { int i; fprintf(stream, " "); - int cc = on_this_line; + cc = on_this_line; if (cc != 16) { diff --git a/include/mysql/plugin.h b/include/mysql/plugin.h index 03bf59ad7ac6e..d7cb0f4bdc539 100644 --- a/include/mysql/plugin.h +++ b/include/mysql/plugin.h @@ -163,6 +163,21 @@ MARIA_DECLARE_PLUGIN__(NAME, \ #define mysql_declare_plugin_end ,{0,0,0,0,0,0,0,0,0,0,0,0,0}} #define maria_declare_plugin_end ,{0,0,0,0,0,0,0,0,0,0,0,0,0}} +#ifdef _special_ + + +enum enum_mysql_show_type +{ + SHOW_UNDEF, SHOW_BOOL, SHOW_UINT, SHOW_ULONG, + SHOW_ULONGLONG, SHOW_CHAR, SHOW_CHAR_PTR, + SHOW_ARRAY, SHOW_FUNC, SHOW_DOUBLE, + SHOW_SINT, SHOW_SLONG, SHOW_SLONGLONG, SHOW_SIMPLE_FUNC, + SHOW_KEY_CACHE_LONG,SHOW_LONG_STATUS, SHOW_DOUBLE_STATUS, + SHOW_HAVE, SHOW_MY_BOOL, SHOW_HA_ROWS, SHOW_SYS,SHOW_LONG_NOFLUSH, + SHOW_LONGLONG_STATUS, SHOW_LEX_STRING +}; +#else + /* declarations for SHOW STATUS support in plugins */ @@ -174,6 +189,9 @@ enum enum_mysql_show_type SHOW_SINT, SHOW_SLONG, SHOW_SLONGLONG, SHOW_SIMPLE_FUNC, SHOW_always_last }; +#endif + + /* backward compatibility mapping. */ #define SHOW_INT SHOW_UINT diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 32499662a7c14..8aa66ac5500ca 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -13,6 +13,7 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA +add_definitions(-D_special_) IF(WITH_WSREP AND NOT EMBEDDED_LIBRARY) SET(WSREP_INCLUDES ${CMAKE_SOURCE_DIR}/wsrep) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index d24e9570ba349..bf174f960ad95 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -15,6 +15,7 @@ Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA */ #include "../storage/xtradb/include/KeySingleton.h" +#define _special_ #include "sql_plugin.h" #include "sql_priv.h" @@ -3684,6 +3685,11 @@ static bool init_global_datetime_format(timestamp_type format_type, } return false; } +//#ifndef SHOW_LONG_STATUS +//#define SHOW_LONG_STATUS 15 +//#endif + + SHOW_VAR com_status_vars[]= { {"admin_commands", (char*) offsetof(STATUS_VAR, com_other), SHOW_LONG_STATUS}, diff --git a/sql/sql_plugin.h b/sql/sql_plugin.h index a0225f4a071aa..27efbc8d87418 100644 --- a/sql/sql_plugin.h +++ b/sql/sql_plugin.h @@ -22,10 +22,8 @@ the following #define adds server-only members to enum_mysql_show_type, that is defined in plugin.h */ -#define SHOW_always_last SHOW_KEY_CACHE_LONG, \ - SHOW_LONG_STATUS, SHOW_DOUBLE_STATUS, \ - SHOW_HAVE, SHOW_MY_BOOL, SHOW_HA_ROWS, SHOW_SYS, \ - SHOW_LONG_NOFLUSH, SHOW_LONGLONG_STATUS, SHOW_LEX_STRING + +#define SHOW_always_last SHOW_KEY_CACHE_LONG,SHOW_LONG_STATUS, SHOW_DOUBLE_STATUS,SHOW_HAVE, SHOW_MY_BOOL, SHOW_HA_ROWS, SHOW_SYS,SHOW_LONG_NOFLUSH, SHOW_LONGLONG_STATUS, SHOW_LEX_STRING #include #undef SHOW_always_last diff --git a/storage/xtradb/CMakeLists.txt b/storage/xtradb/CMakeLists.txt index a5d3baa10fd7c..fb1eff7cc3045 100644 --- a/storage/xtradb/CMakeLists.txt +++ b/storage/xtradb/CMakeLists.txt @@ -315,7 +315,6 @@ SET(INNOBASE_SOURCES dyn/dyn0dyn.cc enc/EncKeys.cc enc/KeySingleton.cc - enc/keyfile.c eval/eval0eval.cc eval/eval0proc.cc fil/fil0fil.cc diff --git a/storage/xtradb/enc/EncKeys.cc b/storage/xtradb/enc/EncKeys.cc index fb4af0c3a5c10..36721bd612143 100644 --- a/storage/xtradb/enc/EncKeys.cc +++ b/storage/xtradb/enc/EncKeys.cc @@ -19,9 +19,12 @@ Created 09/15/2014 ***********************************************************************/ +#ifdef __WIN__ +#define PCRE_STATIC 1 +#endif #include "EncKeys.h" -#include +#include #include #include #include @@ -32,7 +35,7 @@ const char* EncKeys::strMAGIC = "Salted__"; -const int EncKeys::magicSize = strlen(strMAGIC); // 8 byte +const int EncKeys::magicSize = 8;//strlen(strMAGIC); // 8 byte const char* EncKeys::newLine = "\n"; const char* EncKeys::errorNoKeyId = "KeyID = %u not found or with error. Check the key and the log file.\n"; @@ -74,8 +77,8 @@ EncKeys::~EncKeys() { bool EncKeys::initKeys(const char *name, const char *url, const int initType, const char *filekey) { if (KEYINITTYPE_FILE == initType) { // url == path && name == filename - if(ERROR_FALSE_FILE_KEY == initKeysThroughFile(name, url, filekey)) return false; - else return true; + int result = initKeysThroughFile(name, url, filekey); + return NO_ERROR_KEY_FILE_PARSE_OK == result; } else if (KEYINITTYPE_SERVER == initType) { printf(errorNotImplemented); @@ -84,6 +87,7 @@ bool EncKeys::initKeys(const char *name, const char *url, const int initType, co } int EncKeys::initKeysThroughFile(const char *name, const char *path, const char *filekey) { + if (path==NULL || name==NULL) return ERROR_OPEN_FILE; size_t len1 = strlen(path); size_t len2 = strlen(name); bool isSlash = ('/' == path[len1 - 1]); @@ -113,7 +117,7 @@ keyentry *EncKeys::getKeys(int id) { * Store the keys with id smaller then 'maxKeyId' in an array of structs keyentry. * Returns NO_ERROR_PARSE_OK or an appropriate error code. */ -int EncKeys::parseFile(const char* filename, const uint maxKeyId, const char *secret) { +int EncKeys::parseFile(const char* filename, const ulint maxKeyId, const char *secret) { int errorCode = 0; char *buffer = decryptFile(filename, secret, &errorCode); @@ -160,7 +164,7 @@ int EncKeys::parseFile(const char* filename, const uint maxKeyId, const char *se return errorCode; } -int EncKeys::parseLine(const char *line, const uint maxKeyId) { +int EncKeys::parseLine(const char *line, const ulint maxKeyId) { int ret = NO_ERROR_PARSE_OK; if (isComment(line)) ret = NO_ERROR_ISCOMMENT; @@ -186,7 +190,7 @@ int EncKeys::parseLine(const char *line, const uint maxKeyId) { else { char buffer[4]; sprintf(buffer, "%.*s", substr_length, substring_start); - uint id = atoi(buffer); + ulint id = atoi(buffer); if (0 == id) ret = ERROR_NOINITIALIZEDKEY; else if (KEY_MAX < id) ret = ERROR_ID_TOO_BIG; else if (maxKeyId < id) ret = NO_ERROR_KEY_GREATER_THAN_ASKED; @@ -250,7 +254,8 @@ char* EncKeys::decryptFile(const char* filename, const char *secret, int *errorC //Check for file encryption if (0 == memcmp(buffer, strMAGIC, magicSize)) { //If file is encrypted, decrypt it first. - unsigned char salt[magicSize + 1]; + const int array_size = magicSize + 1; + unsigned char salt[array_size]; unsigned char *key = new unsigned char[keySize32]; unsigned char *iv = new unsigned char[ivSize16]; char *decrypted = new char[file_size]; @@ -263,7 +268,7 @@ char* EncKeys::decryptFile(const char* filename, const char *secret, int *errorC if(0 != res) { *errorCode = ERROR_FALSE_FILE_KEY; delete[] buffer; buffer = NULL; - printf(errorFalseFileKey, secret); + printf(errorFalseFileKey, filename); } else { memcpy(buffer, decrypted, d_size); @@ -287,7 +292,7 @@ bool EncKeys::isComment(const char *line) { } -void EncKeys::printKeyEntry( uint id) +void EncKeys::printKeyEntry( ulint id) { keyentry *entry = getKeys(id); if( NULL == entry) printf("No such keyID = %u\n", id); diff --git a/storage/xtradb/include/EncKeys.h b/storage/xtradb/include/EncKeys.h index 9dacfed9e7f23..2f52d293f3e97 100644 --- a/storage/xtradb/include/EncKeys.h +++ b/storage/xtradb/include/EncKeys.h @@ -23,6 +23,7 @@ Created 09/15/2014 #ifndef ENCKEYS_H_ #define ENCKEYS_H_ +#include "univ.i" #include #include @@ -30,7 +31,7 @@ Created 09/15/2014 struct keyentry { - uint id; + ulint id; char *iv; char *key; }; @@ -56,15 +57,15 @@ class EncKeys *errorNoInitializedKey, *errorFalseFileKey, *errorNotImplemented, *errorOpenFile, *errorReadingFile, *errorFileSize; - uint countKeys, keyLineInKeyFile; + ulint countKeys, keyLineInKeyFile; keyentry keys[MAX_KEYS], *oneKey; - void printKeyEntry( uint id); + void printKeyEntry( ulint id); int initKeysThroughFile( const char *name, const char *path, const char *filekey); bool isComment( const char *line); char * decryptFile( const char* filename, const char *secret, int *errorCode); - int parseFile( const char* filename, const uint maxKeyId, const char *secret); - int parseLine( const char *line, const uint maxKeyId); + int parseFile( const char* filename, const ulint maxKeyId, const char *secret); + int parseLine( const char *line, const ulint maxKeyId); public: enum errorCodesFile { NO_ERROR_KEY_FILE_PARSE_OK = 0, ERROR_KEY_FILE_PARSE_NULL = 110, diff --git a/storage/xtradb/include/os0file.h b/storage/xtradb/include/os0file.h index 841e1c447bded..0831bfb94d6f9 100644 --- a/storage/xtradb/include/os0file.h +++ b/storage/xtradb/include/os0file.h @@ -1191,15 +1191,16 @@ os_aio_func( on this file space */ ulint page_compression_level, /*!< page compression level to be used */ - ibool page_encryption, /*!< in: is page encryption used - on this file space */ - ulint page_encryption_key, /*!< page encryption key - to be used */ - ulint* write_size);/*!< in/out: Actual write size initialized + ulint* write_size,/*!< in/out: Actual write size initialized after fist successfull trim operation for this page and if initialized we do not trim again if actual page size does not decrease. */ + ibool page_encryption, /*!< in: is page encryption used + on this file space */ + ulint page_encryption_key); /*!< page encryption key + to be used */ + /************************************************************************//** Wakes up all async i/o threads so that they know to exit themselves in diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index 0a03adaa31156..d242fa1cbadc1 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -5125,15 +5125,16 @@ os_aio_func( on this file space */ ulint page_compression_level, /*!< page compression level to be used */ - ibool page_encryption, /*!< in: is page encryption used - on this file space */ - ulint page_encryption_key, /*!< page encryption key - to be used */ - ulint* write_size)/*!< in/out: Actual write size initialized + ulint* write_size,/*!< in/out: Actual write size initialized after fist successfull trim operation for this page and if initialized we do not trim again if actual page size does not decrease. */ + ibool page_encryption, /*!< in: is page encryption used + on this file space */ + ulint page_encryption_key) /*!< page encryption key + to be used */ + { os_aio_array_t* array; os_aio_slot_t* slot; diff --git a/unittest/eperi/CMakeLists.txt b/unittest/eperi/CMakeLists.txt index 1ace9d684e46c..332b3f40b94e9 100644 --- a/unittest/eperi/CMakeLists.txt +++ b/unittest/eperi/CMakeLists.txt @@ -24,10 +24,13 @@ INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql ${CMAKE_SOURCE_DIR}/storage/xtradb/include ) +if (WIN32) +else() MY_ADD_TESTS(eperi) MY_ADD_TESTS(eperi_aes LINK_LIBRARIES mysys_ssl dbug) + MY_ADD_TESTS(EperiKeySingleton EXT "cc" LINK_LIBRARIES xtradb pcre mysys_ssl) @@ -68,3 +71,5 @@ file(COPY file(COPY ${CMAKE_CURRENT_LIST_DIR}/compressed_full DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) + +endif() diff --git a/unittest/eperi/EperiKeySingleton-t.cc b/unittest/eperi/EperiKeySingleton-t.cc index 2fd174f8b4da0..2c4e3f41d09df 100644 --- a/unittest/eperi/EperiKeySingleton-t.cc +++ b/unittest/eperi/EperiKeySingleton-t.cc @@ -5,7 +5,6 @@ Implementation of single pattern to keep keys for encrypting/decrypting pages. Created 09/15/2014 ***********************************************************************/ - #include "EperiKeySingleton-t.h" #include #include diff --git a/unittest/eperi/eperi_aes-t.c b/unittest/eperi/eperi_aes-t.c index 06d79ef14b482..1bb9f4453c115 100644 --- a/unittest/eperi/eperi_aes-t.c +++ b/unittest/eperi/eperi_aes-t.c @@ -1,5 +1,9 @@ #define EP_UNIT_TEST 1 #define UNIV_INLINE + + +#ifndef __WIN__ + typedef unsigned char byte; typedef unsigned long int ulint; typedef unsigned long int ibool; @@ -8,7 +12,6 @@ typedef unsigned long int ibool; #include #include #include -#include #include "../../storage/xtradb/include/fil0pageencryption.h" @@ -342,7 +345,6 @@ test_bytes_to_key() ok(memcmp(iv, &ivresult, 16) == 0, "BytesToKey iv generated successfully."); } - int main(int argc __attribute__((unused)),char *argv[]) { @@ -354,5 +356,14 @@ main(int argc __attribute__((unused)),char *argv[]) test_cbc_enc_dec(); test_wrong_key(); test_bytes_to_key(); + return 0; } + +#else +int +main(int argc ,char *argv[]) { + return 0; +} + +#endif \ No newline at end of file From a9b0c4890a5cecd18d584f0e6a2edd16965b5af0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Clemens=20D=C3=B6rrh=C3=B6fer?= Date: Wed, 8 Oct 2014 17:13:45 +0200 Subject: [PATCH 31/70] EperiKeySingleton removed fix path --- unittest/eperi/CMakeLists.txt | 1 + unittest/eperi/EperiKeySingleton-t.cc | 10 +++++++--- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/unittest/eperi/CMakeLists.txt b/unittest/eperi/CMakeLists.txt index 1ace9d684e46c..f13f8365c84e4 100644 --- a/unittest/eperi/CMakeLists.txt +++ b/unittest/eperi/CMakeLists.txt @@ -24,6 +24,7 @@ INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql ${CMAKE_SOURCE_DIR}/storage/xtradb/include ) +SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D SINGLETON_TEST_DATA=\\\"${CMAKE_SOURCE_DIR}/unittest/eperi\\\" ") MY_ADD_TESTS(eperi) MY_ADD_TESTS(eperi_aes LINK_LIBRARIES mysys_ssl dbug) diff --git a/unittest/eperi/EperiKeySingleton-t.cc b/unittest/eperi/EperiKeySingleton-t.cc index 2fd174f8b4da0..53308e461c1c3 100644 --- a/unittest/eperi/EperiKeySingleton-t.cc +++ b/unittest/eperi/EperiKeySingleton-t.cc @@ -35,13 +35,17 @@ void printEntry(struct keyentry *entry, uint id) int main() { - plan(1); - printf("%s\n", "main() EperiKeySingleton.cc"); + #ifdef SINGLETON_TEST_DATA - KeySingleton& ksp = KeySingleton::getInstance( "keys.txt", "/home/denis", 1, "secret"); + printf("%s\n", "main() EperiKeySingleton.cc"); + printf("%s\n", SINGLETON_TEST_DATA); + KeySingleton& ksp = KeySingleton::getInstance( "keys.txt", SINGLETON_TEST_DATA, 1, "secret"); printEntry(ksp.getKeys(0), 0); return EXIT_SUCCESS; + #else + + #endif } From 90022b46194ab5d5618256c2f52f6233ef700560 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 9 Oct 2014 15:52:20 +0200 Subject: [PATCH 32/70] windows, impl: PAGE_ENCRYPTION --- mysys_ssl/my_aes.cc | 4 ++- sql/mysqld.cc | 7 ------ storage/xtradb/enc/EncKeys.cc | 30 +++++++++------------- storage/xtradb/fil/fil0pageencryption.cc | 8 +++--- storage/xtradb/handler/ha_innodb.cc | 1 + storage/xtradb/include/EncKeys.h | 4 +-- storage/xtradb/include/KeySingleton.h | 2 +- storage/xtradb/os/os0file.cc | 32 +++++++++++++++++++++++- 8 files changed, 55 insertions(+), 33 deletions(-) diff --git a/mysys_ssl/my_aes.cc b/mysys_ssl/my_aes.cc index 012966d0626cf..7e6b48e0bb4db 100644 --- a/mysys_ssl/my_aes.cc +++ b/mysys_ssl/my_aes.cc @@ -115,10 +115,12 @@ void my_aes_hexToUint(const char* in, unsigned char *out, int dest_length) { const char *pos = in; + int res = 0; int count = 0; for(count = 0; count < dest_length; count++) { - sscanf(pos, "%2hhx", &out[count]); + sscanf(pos, "%2hhx", &res); + out[count] = res; pos += 2 * sizeof(char); } } diff --git a/sql/mysqld.cc b/sql/mysqld.cc index bf174f960ad95..5fd24f96f1ffd 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -14,9 +14,6 @@ along with this program; if not, write to the Free Software Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA */ -#include "../storage/xtradb/include/KeySingleton.h" -#define _special_ - #include "sql_plugin.h" #include "sql_priv.h" #include "unireg.h" @@ -5725,10 +5722,6 @@ int mysqld_main(int argc, char **argv) mysql_cond_signal(&COND_server_started); mysql_mutex_unlock(&LOCK_server_started); - KeySingleton& ksp2 = KeySingleton::getInstance(); - struct keyentry *entry = ksp2.getKeys(2); - if(entry) printf("id:%3u \tiv:%s \tkey:%s\n", entry->id, entry->iv, entry->key); - #if defined(_WIN32) || defined(HAVE_SMEM) handle_connections_methods(); #else diff --git a/storage/xtradb/enc/EncKeys.cc b/storage/xtradb/enc/EncKeys.cc index b450a7b11a9c5..3d1f3eaea14ee 100644 --- a/storage/xtradb/enc/EncKeys.cc +++ b/storage/xtradb/enc/EncKeys.cc @@ -73,13 +73,12 @@ EncKeys::~EncKeys() { delete[] keys[ii].key; keys[ii].key = NULL; } - delete oneKey; oneKey = NULL; } bool EncKeys::initKeys(const char *name, const char *url, const int initType, const char *filekey) { if (KEYINITTYPE_FILE == initType) { // url == path && name == filename int result = initKeysThroughFile(name, url, filekey); - return NO_ERROR_KEY_FILE_PARSE_OK == result; + return ERROR_FALSE_FILE_KEY != result; } else if (KEYINITTYPE_SERVER == initType) { fprintf(stderr, errorNotImplemented); @@ -94,10 +93,9 @@ int EncKeys::initKeysThroughFile(const char *name, const char *path, const char const char *MAGIC = "FILE:"; const short MAGIC_LEN = 5; int ret = NO_ERROR_KEY_FILE_PARSE_OK; - #ifdef TARGET_OS_LINUX bool isSlash = ('/' == path[len1 - 1]); char *secret = (char*) malloc(MAX_SECRET_SIZE * sizeof(char)); - char *filename = (char*) malloc((len1 + len2 + isSlash ? 1 : 2) * sizeof(char)); + char *filename = (char*) malloc((len1 + len2 + (isSlash ? 1 : 2)) * sizeof(char)); if(filekey != NULL) { //If secret starts with FILE: interpret the secret as filename. @@ -107,7 +105,7 @@ int EncKeys::initKeysThroughFile(const char *name, const char *path, const char sprintf(secretfile, "%s%s%s", path, isSlash ? "" : "/", filekey+MAGIC_LEN); parseSecret(secretfile, secret); free(secretfile); - }else + } else { sprintf(secret, "%s", filekey); } @@ -116,10 +114,6 @@ int EncKeys::initKeysThroughFile(const char *name, const char *path, const char ret = parseFile((const char *)filename, 254, secret); free(filename); free(secret); - #endif //TARGET_OS_LINUX - #ifdef __WIN__ - ut_ad(false); - #endif //__WIN__e return ret; } @@ -207,7 +201,7 @@ int EncKeys::parseLine(const char *line, const ulint maxKeyId) { if (isComment(line)) ret = NO_ERROR_ISCOMMENT; else { - const char *error_p; + const char *error_p = NULL; int offset; static const pcre *pattern = pcre_compile( "([0-9]+);([0-9,a-f,A-F]{32});([0-9,a-f,A-F]{64}|[0-9,a-f,A-F]{48}|[0-9,a-f,A-F]{32})", @@ -260,7 +254,7 @@ int EncKeys::parseLine(const char *line, const ulint maxKeyId) { char* EncKeys::decryptFile(const char* filename, const char *secret, int *errorCode) { *errorCode = NO_ERROR_PARSE_OK; fprintf(stderr, "Reading %s\n\n", filename); - FILE *fp = my_fopen(filename, O_RDONLY, MYF(MY_WME)); + FILE *fp = fopen(filename, "rb"); if (NULL == fp) { fprintf(stderr, errorOpenFile, filename); *errorCode = ERROR_OPEN_FILE; @@ -275,6 +269,7 @@ char* EncKeys::decryptFile(const char* filename, const char *secret, int *errorC if (MAX_KEY_FILE_SIZE < file_size) { fprintf(stderr, errorExceedKeyFileSize, filename, MAX_KEY_FILE_SIZE); *errorCode = ERROR_KEY_FILE_TOO_BIG; + fclose(fp); return NULL; } else if (-1L == file_size) { @@ -283,13 +278,12 @@ char* EncKeys::decryptFile(const char* filename, const char *secret, int *errorC return NULL; } - fseek(fp, 0L, SEEK_SET); + rewind(fp); //Read file into buffer - char *buffer = new char[file_size + 1]; - fread(buffer, file_size, 1, fp); + uchar *buffer = new uchar[file_size + 1]; + size_t read_bytes = fread(buffer, 1, file_size, fp); buffer[file_size] = '\0'; - my_fclose(fp, MYF(MY_WME)); - + fclose(fp); //Check for file encryption if (0 == memcmp(buffer, strMAGIC, magicSize)) { //If file is encrypted, decrypt it first. const int array_size = magicSize + 1; @@ -301,7 +295,7 @@ char* EncKeys::decryptFile(const char* filename, const char *secret, int *errorC salt[magicSize] = '\0'; my_bytes_to_key((unsigned char *) salt, secret, key, iv); unsigned long int d_size = 0; - int res = my_aes_decrypt_cbc(buffer + 2 * magicSize, file_size - 2 * magicSize, + int res = my_aes_decrypt_cbc((const char*)buffer + 2 * magicSize, file_size - 2 * magicSize, decrypted, &d_size, key, keySize32, iv, ivSize16); if(0 != res) { *errorCode = ERROR_FALSE_FILE_KEY; @@ -317,7 +311,7 @@ char* EncKeys::decryptFile(const char* filename, const char *secret, int *errorC delete[] key; key = NULL; delete[] iv; iv = NULL; } - return buffer; + return (char*) buffer; } bool EncKeys::isComment(const char *line) { diff --git a/storage/xtradb/fil/fil0pageencryption.cc b/storage/xtradb/fil/fil0pageencryption.cc index acdbea33787d2..01acf1dd8adc2 100644 --- a/storage/xtradb/fil/fil0pageencryption.cc +++ b/storage/xtradb/fil/fil0pageencryption.cc @@ -125,7 +125,8 @@ ulint mode checksum = fil_page_encryption_calc_checksum(buf + header_len, page_size - (FIL_PAGE_DATA_END + header_len)); const unsigned char rkey[] = {0xbd, 0xe4, 0x72, 0xa2, 0x95, 0x67, 0x5c, 0xa9, - 0x2e, 0x04, 0x67, 0xea, 0xdb, 0xc0,0xe0, 0x23}; + 0x2e, 0x04, 0x67, 0xea, 0xdb, 0xc0,0xe0, 0x23, + 0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0}; uint8 key_len = 16; if (!unit_test) { KeySingleton& keys = KeySingleton::getInstance(); @@ -409,8 +410,9 @@ al_size); tmp_page_buf = static_cast(ut_malloc(page_size)); memset(tmp_page_buf,0, page_size); - const unsigned char rkey[] = {0xbd, 0xe4, 0x72, 0xa2, 0x95, 0x67, 0x5c, 0xa9, - 0x2e, 0x04, 0x67, 0xea, 0xdb, 0xc0,0xe0, 0x23}; + const unsigned char rkey[] = {0xbd, 0xe4, 0x72, 0xa2, 0x95, 0x67, 0x5c, 0xa9, + 0x2e, 0x04, 0x67, 0xea, 0xdb, 0xc0,0xe0, 0x23, + 0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0}; uint8 key_len = 16; if (!unit_test) { KeySingleton& keys = KeySingleton::getInstance(); diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 977a2b8cbaa74..941b56f6b20f3 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -4065,6 +4065,7 @@ innobase_end( DBUG_ENTER("innobase_end"); DBUG_ASSERT(hton == innodb_hton_ptr); + KeySingleton::getInstance().~KeySingleton(); if (innodb_inited) { srv_fast_shutdown = (ulint) innobase_fast_shutdown; diff --git a/storage/xtradb/include/EncKeys.h b/storage/xtradb/include/EncKeys.h index 3a332a7775ad3..107f6a8a3b150 100644 --- a/storage/xtradb/include/EncKeys.h +++ b/storage/xtradb/include/EncKeys.h @@ -65,8 +65,8 @@ class EncKeys int initKeysThroughFile( const char *name, const char *path, const char *filekey); bool isComment( const char *line); char * decryptFile( const char* filename, const char *secret, int *errorCode); - int parseFile( const char* filename, const uint maxKeyId, const char *secret); - int parseLine( const char *line, const uint maxKeyId); + int parseFile( const char* filename, const ulint maxKeyId, const char *secret); + int parseLine( const char *line, const ulint maxKeyId); void parseSecret( const char *filename, char *secret ); public: diff --git a/storage/xtradb/include/KeySingleton.h b/storage/xtradb/include/KeySingleton.h index 40d73af1d2eba..667ba2d5be601 100644 --- a/storage/xtradb/include/KeySingleton.h +++ b/storage/xtradb/include/KeySingleton.h @@ -44,7 +44,7 @@ class KeySingleton KeySingleton & operator = (const KeySingleton&); public: - virtual ~KeySingleton() {} + virtual ~KeySingleton() {encKeys.~EncKeys();} static KeySingleton& getInstance(); // Init the instance for only one time static KeySingleton& getInstance(const char *name, const char *url, diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index d242fa1cbadc1..63677671125be 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -3127,6 +3127,11 @@ os_file_read_func( os_mutex_exit(os_file_count_mutex); if (ret && len == n) { + if (fil_page_is_encrypted((byte *)buf)) { + if (fil_decrypt_page(NULL, (byte *)buf, n, NULL, &compressed, 0)!=PAGE_ENCRYPTION_OK) {; + return FALSE; + } + } /* Note that InnoDB writes files that are not formated as file spaces and they do not have FIL_PAGE_TYPE field, thus we must use here information is the actual @@ -3251,6 +3256,9 @@ os_file_read_no_error_handling_func( if (ret && len == n) { + if (fil_page_is_encrypted((byte *)buf)) { + if (fil_decrypt_page(NULL, (byte *)buf, n, NULL, &compressed, 0)!=PAGE_ENCRYPTION_OK) return (FALSE); + } /* Note that InnoDB writes files that are not formated as file spaces and they do not have FIL_PAGE_TYPE field, thus we must use here information is the actual @@ -5136,6 +5144,7 @@ os_aio_func( to be used */ { + void* buffer = NULL; os_aio_array_t* array; os_aio_slot_t* slot; #ifdef WIN_ASYNC_IO @@ -5255,7 +5264,12 @@ os_aio_func( if (srv_use_native_aio) { os_n_file_writes++; #ifdef WIN_ASYNC_IO - ret = WriteFile(file, buf, (DWORD) n, &len, + if (page_encryption) { + buffer = slot->page_buf2; + } else { + buffer = buf; + } + ret = WriteFile(file, buffer, (DWORD) n, &len, &(slot->control)); if(!ret && GetLastError() != ERROR_IO_PENDING) @@ -5449,6 +5463,22 @@ os_aio_windows_handle( ret_val = ret && len == slot->len; } + /* page encryption */ + if (slot->message1 && slot->page_encryption) { + if (slot->page_buf2==NULL) { + os_slot_alloc_page_buf2(slot); + } + + ut_ad(slot->page_buf2); + + if (slot->type == OS_FILE_READ) { + if (fil_page_is_encrypted(slot->buf)) { + fil_decrypt_page(slot->page_buf2, slot->buf, slot->len, slot->write_size, NULL, 0); + } + } + + } + if (slot->message1 && slot->page_compression) { // We allocate memory for page compressed buffer if and only // if it is not yet allocated. From 7cd49903f64f667c5c7716e005fa66b2ebd574bc Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 9 Oct 2014 15:54:31 +0200 Subject: [PATCH 33/70] added sthg to ignore list --- .gitignore | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index 88b42cfe566cf..9b0fc91fc0153 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,9 @@ .cproject .project Debug/* +pcre/* +pcre3/* +bld/* *-t *.a *.ctest From 5bb46146f92398c3c972f3a4924f589dc6356600 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Clemens=20D=C3=B6rrh=C3=B6fer?= Date: Thu, 9 Oct 2014 16:38:36 +0200 Subject: [PATCH 34/70] fix Linux - Windows patch --- storage/xtradb/include/os0file.ic | 4 ++-- unittest/eperi/CMakeLists.txt | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/storage/xtradb/include/os0file.ic b/storage/xtradb/include/os0file.ic index f2ff5c79351d5..59a5a149424f2 100644 --- a/storage/xtradb/include/os0file.ic +++ b/storage/xtradb/include/os0file.ic @@ -250,8 +250,8 @@ pfs_os_aio_func( result = os_aio_func(type, mode, name, file, buf, offset, n, message1, message2, space_id, trx, - page_compression, page_compression_level, - page_encryption, page_encryption_key, write_size); + page_compression, page_compression_level, write_size , + page_encryption, page_encryption_key); register_pfs_file_io_end(locker, n); diff --git a/unittest/eperi/CMakeLists.txt b/unittest/eperi/CMakeLists.txt index 332b3f40b94e9..3b37eb500b41b 100644 --- a/unittest/eperi/CMakeLists.txt +++ b/unittest/eperi/CMakeLists.txt @@ -27,8 +27,8 @@ INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql if (WIN32) else() MY_ADD_TESTS(eperi) -MY_ADD_TESTS(eperi_aes - LINK_LIBRARIES mysys_ssl dbug) +#MY_ADD_TESTS(eperi_aes +# LINK_LIBRARIES mysys_ssl dbug) MY_ADD_TESTS(EperiKeySingleton From acfa72d273688c994712d2175e1a222acf2b8ba7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Clemens=20D=C3=B6rrh=C3=B6fer?= Date: Thu, 9 Oct 2014 16:52:02 +0200 Subject: [PATCH 35/70] revert some changes --- include/mysql/plugin.h | 17 +---------------- sql/CMakeLists.txt | 1 - 2 files changed, 1 insertion(+), 17 deletions(-) diff --git a/include/mysql/plugin.h b/include/mysql/plugin.h index d7cb0f4bdc539..1a485be224991 100644 --- a/include/mysql/plugin.h +++ b/include/mysql/plugin.h @@ -163,21 +163,6 @@ MARIA_DECLARE_PLUGIN__(NAME, \ #define mysql_declare_plugin_end ,{0,0,0,0,0,0,0,0,0,0,0,0,0}} #define maria_declare_plugin_end ,{0,0,0,0,0,0,0,0,0,0,0,0,0}} -#ifdef _special_ - - -enum enum_mysql_show_type -{ - SHOW_UNDEF, SHOW_BOOL, SHOW_UINT, SHOW_ULONG, - SHOW_ULONGLONG, SHOW_CHAR, SHOW_CHAR_PTR, - SHOW_ARRAY, SHOW_FUNC, SHOW_DOUBLE, - SHOW_SINT, SHOW_SLONG, SHOW_SLONGLONG, SHOW_SIMPLE_FUNC, - SHOW_KEY_CACHE_LONG,SHOW_LONG_STATUS, SHOW_DOUBLE_STATUS, - SHOW_HAVE, SHOW_MY_BOOL, SHOW_HA_ROWS, SHOW_SYS,SHOW_LONG_NOFLUSH, - SHOW_LONGLONG_STATUS, SHOW_LEX_STRING -}; -#else - /* declarations for SHOW STATUS support in plugins */ @@ -189,7 +174,7 @@ enum enum_mysql_show_type SHOW_SINT, SHOW_SLONG, SHOW_SLONGLONG, SHOW_SIMPLE_FUNC, SHOW_always_last }; -#endif + diff --git a/sql/CMakeLists.txt b/sql/CMakeLists.txt index 8aa66ac5500ca..32499662a7c14 100644 --- a/sql/CMakeLists.txt +++ b/sql/CMakeLists.txt @@ -13,7 +13,6 @@ # along with this program; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA -add_definitions(-D_special_) IF(WITH_WSREP AND NOT EMBEDDED_LIBRARY) SET(WSREP_INCLUDES ${CMAKE_SOURCE_DIR}/wsrep) From 41ed2730e53ea170fee0e6412be025315730526d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Clemens=20D=C3=B6rrh=C3=B6fer?= Date: Fri, 10 Oct 2014 10:12:50 +0200 Subject: [PATCH 36/70] Added interface for keyserver --- storage/xtradb/enc/EncKeys.cc | 17 +++++++++++++---- storage/xtradb/enc/KeySingleton.cc | 2 +- storage/xtradb/fil/fil0pageencryption.cc | 11 ++--------- storage/xtradb/include/EncKeys.h | 1 + 4 files changed, 17 insertions(+), 14 deletions(-) diff --git a/storage/xtradb/enc/EncKeys.cc b/storage/xtradb/enc/EncKeys.cc index 3d1f3eaea14ee..ac1ea32d97c3a 100644 --- a/storage/xtradb/enc/EncKeys.cc +++ b/storage/xtradb/enc/EncKeys.cc @@ -76,14 +76,16 @@ EncKeys::~EncKeys() { } bool EncKeys::initKeys(const char *name, const char *url, const int initType, const char *filekey) { - if (KEYINITTYPE_FILE == initType) { // url == path && name == filename + if (KEYINITTYPE_FILE == initType) + { int result = initKeysThroughFile(name, url, filekey); return ERROR_FALSE_FILE_KEY != result; } - else if (KEYINITTYPE_SERVER == initType) { - fprintf(stderr, errorNotImplemented); + else if (KEYINITTYPE_SERVER == initType) + { + return NO_ERROR_KEY_FILE_PARSE_OK == initKeysThroughServer(name, url, filekey); } - return NO_ERROR_KEY_FILE_PARSE_OK == ERROR_KEYINITTYPE_SERVER_NOT_IMPLEMENTED; + return false; } int EncKeys::initKeysThroughFile(const char *name, const char *path, const char *filekey) { @@ -117,6 +119,13 @@ int EncKeys::initKeysThroughFile(const char *name, const char *path, const char return ret; } +int EncKeys::initKeysThroughServer( const char *name, const char *path, const char *filekey) +{ + //TODO + fprintf(stderr, errorNotImplemented); + return ERROR_KEYINITTYPE_SERVER_NOT_IMPLEMENTED; +} + void EncKeys::parseSecret( const char *secretfile, char *secret ) { int i=0; FILE *fp = my_fopen(secretfile, O_RDWR, MYF(MY_WME)); diff --git a/storage/xtradb/enc/KeySingleton.cc b/storage/xtradb/enc/KeySingleton.cc index 1542eba24b2c9..aea666137cce9 100644 --- a/storage/xtradb/enc/KeySingleton.cc +++ b/storage/xtradb/enc/KeySingleton.cc @@ -46,7 +46,7 @@ KeySingleton & KeySingleton::getInstance(const char *name, const char *url, instanceInited = encKeys.initKeys(name, url, initType, filekey); if( !instanceInited) { printf("Could not initialize any of the encryption / decryption keys. " - "You can not read encrypted tables or columns\n\n"); + "You can not read encrypted tables\n\n"); } return theInstance; diff --git a/storage/xtradb/fil/fil0pageencryption.cc b/storage/xtradb/fil/fil0pageencryption.cc index bcc7aad492656..4cbacebba4eee 100644 --- a/storage/xtradb/fil/fil0pageencryption.cc +++ b/storage/xtradb/fil/fil0pageencryption.cc @@ -85,26 +85,19 @@ ulint mode unit_test = 0x01 & mode; - //TODO encryption default key - /* If no encryption key was provided to this table, use system - default key - if (key == 0) { - key = 0; - }*/ - if (!unit_test) { ut_ad(fil_space_is_page_encrypted(space_id)); fil_system_enter(); space = fil_space_get_by_id(space_id); fil_system_exit(); - + } #ifdef UNIV_DEBUG fprintf(stderr, "InnoDB: Note: Preparing for encryption for space %lu name %s len %lu\n", space_id, fil_space_name(space), len); #endif /* UNIV_DEBUG */ - } + /* data_size -1 bytes will be encrypted at first. * data_size is the length of the cipher text.*/ diff --git a/storage/xtradb/include/EncKeys.h b/storage/xtradb/include/EncKeys.h index 107f6a8a3b150..c29b58617b0bc 100644 --- a/storage/xtradb/include/EncKeys.h +++ b/storage/xtradb/include/EncKeys.h @@ -63,6 +63,7 @@ class EncKeys void printKeyEntry( ulint id); int initKeysThroughFile( const char *name, const char *path, const char *filekey); + int initKeysThroughServer( const char *name, const char *path, const char *filekey); bool isComment( const char *line); char * decryptFile( const char* filename, const char *secret, int *errorCode); int parseFile( const char* filename, const ulint maxKeyId, const char *secret); From 6ca7fc3eb363e90e31d6b0e42bb512ffe3c69a68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Clemens=20D=C3=B6rrh=C3=B6fer?= Date: Fri, 10 Oct 2014 13:47:33 +0200 Subject: [PATCH 37/70] Check for NULLpointer in keystore. Code cleanup - restructured and removed a lot of code in fil_page_encrypt and file_page_decrypt. --- storage/xtradb/enc/EncKeys.cc | 4 +- storage/xtradb/fil/fil0pageencryption.cc | 152 +- storage/xtradb/os/os0file.cc | 1 + storage/xtradb/os/os0file.cc.orig | 6683 ---------------------- 4 files changed, 66 insertions(+), 6774 deletions(-) delete mode 100644 storage/xtradb/os/os0file.cc.orig diff --git a/storage/xtradb/enc/EncKeys.cc b/storage/xtradb/enc/EncKeys.cc index ac1ea32d97c3a..361c3b77d4823 100644 --- a/storage/xtradb/enc/EncKeys.cc +++ b/storage/xtradb/enc/EncKeys.cc @@ -154,8 +154,8 @@ keyentry *EncKeys::getKeys(int id) { } /** - * Get the keys from the key file 'filename' and decrypt it with the key 'secret'. - * Store the keys with id smaller then 'maxKeyId' in an array of structs keyentry. + * Get the keys from the key file and decrypt it with the key . + * Store the keys with id smaller then in an array of structs keyentry. * Returns NO_ERROR_PARSE_OK or an appropriate error code. */ int EncKeys::parseFile(const char* filename, const ulint maxKeyId, const char *secret) { diff --git a/storage/xtradb/fil/fil0pageencryption.cc b/storage/xtradb/fil/fil0pageencryption.cc index 4cbacebba4eee..d89c344a4b753 100644 --- a/storage/xtradb/fil/fil0pageencryption.cc +++ b/storage/xtradb/fil/fil0pageencryption.cc @@ -117,33 +117,35 @@ ulint mode checksum = fil_page_encryption_calc_checksum(buf + header_len, page_size - (FIL_PAGE_DATA_END + header_len)); const unsigned char rkey[] = {0xbd, 0xe4, 0x72, 0xa2, 0x95, 0x67, 0x5c, 0xa9, - 0x2e, 0x04, 0x67, 0xea, 0xdb, 0xc0,0xe0, 0x23, - 0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0}; + 0x2e, 0x04, 0x67, 0xea, 0xdb, 0xc0, 0xe0, 0x23, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; uint8 key_len = 16; - if (!unit_test) { - KeySingleton& keys = KeySingleton::getInstance(); - char* keyString = keys.getKeys(encryption_key)->key; - key_len = strlen(keyString)/2; - my_aes_hexToUint(keyString, (unsigned char*)&rkey, key_len); - } - const unsigned char iv[] = {0x2d, 0x1a, 0xf8, 0xd3, 0x97, 0x4e, 0x0b, 0xd3, 0xef, 0xed, - 0x5a, 0x6f, 0x82, 0x59, 0x4f,0x5e}; - if (!unit_test) { + const unsigned char iv[] = {0x2d, 0x1a, 0xf8, 0xd3, 0x97, 0x4e, 0x0b, 0xd3, + 0xef, 0xed, 0x5a, 0x6f, 0x82, 0x59, 0x4f, 0x5e}; + uint8 iv_len = 16; + if (!unit_test) + { KeySingleton& keys = KeySingleton::getInstance(); - my_aes_hexToUint(keys.getKeys(encryption_key)->iv, (unsigned char*)&iv, 16); + keyentry *key = keys.getKeys(encryption_key); + if(key != NULL) { + key_len = strlen(key->key)/2; + my_aes_hexToUint(key->key, (unsigned char*)&rkey, key_len); + my_aes_hexToUint(key->iv, (unsigned char*)&iv, iv_len); + } else { + err = AES_KEY_CREATION_FAILED; + } } - uint8 iv_len = 16; + write_size = data_size; - /* 1st encryption: data_size -1 bytes starting from FIL_PAGE_DATA */ - err = my_aes_encrypt_cbc((char*)buf + header_len, - data_size-1, - (char *)out_buf + header_len, - &write_size, - (const unsigned char *)&rkey, - key_len, - (const unsigned char *)&iv, - iv_len);; - ut_ad(write_size == data_size); + if (err == AES_OK) { + /* 1st encryption: data_size -1 bytes starting from FIL_PAGE_DATA */ + err = my_aes_encrypt_cbc((char*) buf + header_len, data_size - 1, + (char *) out_buf + header_len, &write_size, + (const unsigned char *) &rkey, key_len, + (const unsigned char *) &iv, iv_len); + ; ut_ad(write_size == data_size); + } if (page_compressed) { /* page compressed pages: only one encryption. 3 bytes remain unencrypted. 2 bytes are appended to the encrypted buffer. @@ -156,15 +158,13 @@ ulint mode memcpy(out_buf + header_len + data_size, buf + header_len + data_size - 1, remainder - offset); - if (page_compressed) { + if (page_compressed && err == AES_OK) { remaining_byte = mach_read_from_1(buf + header_len + data_size +1); - } else { - - /* create temporary buffer for 2nd encryption */ + } else { + //create temporary buffer for 2nd encryption tmp_buf = static_cast(ut_malloc(64)); - - /* 2nd encryption: 63 bytes from out_buf, result length is 64 bytes */ - err = my_aes_encrypt_cbc((char*)out_buf + page_size -FIL_PAGE_DATA_END -62, + /* 2nd encryption: 63 bytes from out_buf, result length is 64 bytes */ + err = my_aes_encrypt_cbc((char*)out_buf + page_size -FIL_PAGE_DATA_END -62, 63, (char*)tmp_buf, &write_size, @@ -289,7 +289,7 @@ ibool* page_compressed, /*! pos_checksum) { - pos_checksum = compressed_size + FIL_PAGE_DATA; - if (pos_checksum > page_size - 3) { - // checksum not supported, because no space available - } else { - /* Set up the checksum. This is only usable to verify decryption */ - mach_write_to_3(out_buf + pos_checksum, checksum); - } - } - } -al_size); - fflush(stderr); -#endif /* UNIV_PAGEENCRIPTION_DEBUG */ - tmp_buf= static_cast(ut_malloc(64)); tmp_page_buf = static_cast(ut_malloc(page_size)); memset(tmp_page_buf,0, page_size); - const unsigned char rkey[] = {0xbd, 0xe4, 0x72, 0xa2, 0x95, 0x67, 0x5c, 0xa9, - 0x2e, 0x04, 0x67, 0xea, 0xdb, 0xc0,0xe0, 0x23, - 0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0}; - uint8 key_len = 16; - if (!unit_test) { - KeySingleton& keys = KeySingleton::getInstance(); - char* keyString = keys.getKeys(page_encryption_key)->key; - key_len = strlen(keyString)/2; - my_aes_hexToUint(keyString, (unsigned char*)&rkey, key_len); + const unsigned char rkey[] = { 0xbd, 0xe4, 0x72, 0xa2, 0x95, 0x67, 0x5c, + 0xa9, 0x2e, 0x04, 0x67, 0xea, 0xdb, 0xc0, 0xe0, 0x23, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00 }; + uint8 key_len = 16; + const unsigned char iv[] = { 0x2d, 0x1a, 0xf8, 0xd3, 0x97, 0x4e, 0x0b, 0xd3, + 0xef, 0xed, 0x5a, 0x6f, 0x82, 0x59, 0x4f, 0x5e }; + uint8 iv_len = 16; + if (!unit_test) { + KeySingleton& keys = KeySingleton::getInstance(); + keyentry *key = keys.getKeys(page_decryption_key); + if (key != NULL) { + key_len = strlen(key->key) / 2; + my_aes_hexToUint(key->key, (unsigned char*) &rkey, key_len); + my_aes_hexToUint(key->iv, (unsigned char*) &iv, iv_len); + } else { + err = AES_KEY_CREATION_FAILED; } + } - const unsigned char iv[] = {0x2d, 0x1a, 0xf8, 0xd3, 0x97, 0x4e, 0x0b, 0xd3, 0xef, 0xed, - 0x5a, 0x6f, 0x82, 0x59, 0x4f,0x5e}; - if (!unit_test) { - KeySingleton& keys = KeySingleton::getInstance(); - my_aes_hexToUint(keys.getKeys(page_encryption_key)->iv, (unsigned char*)&iv, 16); - } - uint8 iv_len = 16; - if (!page_compression_flag) { tmp_page_buf = static_cast(ut_malloc(page_size)); tmp_buf= static_cast(ut_malloc(64)); @@ -434,23 +413,19 @@ al_size); memcpy(tmp_buf + 62, buf + FIL_PAGE_SPACE_OR_CHKSUM + 3, 1); memcpy(tmp_buf + 63, buf + page_size - FIL_PAGE_DATA_END +3, 1); - err = my_aes_decrypt_cbc((const char*) tmp_buf, - 64, - (char *) tmp_page_buf + page_size -FIL_PAGE_DATA_END -62, - &tmp_write_size, - (const unsigned char *)&rkey, - key_len, - (const unsigned char *)&iv, - iv_len - ); - + if (err == AES_OK) { + err = my_aes_decrypt_cbc((const char*) tmp_buf, 64, + (char *) tmp_page_buf + page_size - FIL_PAGE_DATA_END - 62, + &tmp_write_size, (const unsigned char *) &rkey, key_len, + (const unsigned char *) &iv, iv_len); + } /* If decrypt fails it means that page is corrupted or has an unknown key */ if (err != AES_OK) { fprintf(stderr, "InnoDB: Corruption: Page is marked as encrypted\n" "InnoDB: but decrypt failed with error %d.\n" "InnoDB: size %lu len %lu, key%d\n", err, data_size, - len, (int)page_encryption_key); + len, (int)page_decryption_key); fflush(stderr); if (NULL == page_buf) { ut_free(in_buf); @@ -538,8 +513,7 @@ al_size); #ifdef UNIV_PAGEENCRIPTION_DEBUG fprintf(stderr, "InnoDB: Note: Decryption succeeded for len %lu\n", len); fflush(stderr); -#endif /* UNIV_PAGEENCRIPTIONulint page_compressed = 0; - _DEBUG */ +#endif /* copy header */ memcpy(in_buf, buf, FIL_PAGE_DATA); @@ -577,7 +551,7 @@ al_size); page_size = fsp_flags_get_page_size(flags); page_num = mach_read_from_4(buf+ 4); - page_encryption_key = FSP_FLAGS_GET_PAGE_ENCRYPTION_KEY(flags); + page_decryption_key = FSP_FLAGS_GET_PAGE_ENCRYPTION_KEY(flags); page_encrypted = FSP_FLAGS_GET_PAGE_ENCRYPTION(flags); page_compression_flag = FSP_FLAGS_GET_PAGE_COMPRESSION(flags); // fprintf(stderr,"Page num, page size, key, enc, compr: %lu, %lu, %lu, %lu %lu\n", page_num, page_size, page_encryption_key, page_encrypted, page_compression_flag); diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index 63677671125be..957bc60d48ff5 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -4840,6 +4840,7 @@ os_aio_array_reserve_slot( tmp = fil_encrypt_page(fil_node_get_space_id(slot->message1), (byte *)buf, slot->page_buf2, len, page_encryption_key, &real_len, 0); /* If encryption succeeded, set up the length and buffer */ + //TODO we do not need to reset len since we do not alter any content size if (tmp != buf) { len = real_len; buf = slot->page_buf2; diff --git a/storage/xtradb/os/os0file.cc.orig b/storage/xtradb/os/os0file.cc.orig deleted file mode 100644 index c954c16a6fb10..0000000000000 --- a/storage/xtradb/os/os0file.cc.orig +++ /dev/null @@ -1,6683 +0,0 @@ -/*********************************************************************** - -Copyright (c) 1995, 2013, Oracle and/or its affiliates. All Rights Reserved. -Copyright (c) 2009, Percona Inc. -Copyright (c) 2013, 2014, SkySQL Ab. All Rights Reserved. - -Portions of this file contain modifications contributed and copyrighted -by Percona Inc.. Those modifications are -gratefully acknowledged and are described briefly in the InnoDB -documentation. The contributions by Percona Inc. are incorporated with -their permission, and subject to the conditions contained in the file -COPYING.Percona. - -This program is free software; you can redistribute it and/or modify it -under the terms of the GNU General Public License as published by the -Free Software Foundation; version 2 of the License. - -This program is distributed in the hope that it will be useful, but -WITHOUT ANY WARRANTY; without even the implied warranty of -MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General -Public License for more details. - -You should have received a copy of the GNU General Public License along with -this program; if not, write to the Free Software Foundation, Inc., -51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA - -***********************************************************************/ - -/**************************************************//** -@file os/os0file.cc -The interface to the operating system file i/o primitives - -Created 10/21/1995 Heikki Tuuri -*******************************************************/ - -#include "os0file.h" - -#ifdef UNIV_NONINL -#include "os0file.ic" -#endif -#include "ha_prototypes.h" -#include "ut0mem.h" -#include "srv0srv.h" -#include "srv0start.h" -#include "fil0fil.h" -#include "fsp0fsp.h" -#include "fil0pagecompress.h" -#include "fsp0pageencryption.h" -#include "fil0pageencryption.h" -#include "buf0buf.h" -#include "btr0types.h" -#include "trx0trx.h" -#include "srv0mon.h" -#include "srv0srv.h" -#ifdef HAVE_POSIX_FALLOCATE -#include "fcntl.h" -#endif -#ifndef UNIV_HOTBACKUP -# include "os0sync.h" -# include "os0thread.h" -#else /* !UNIV_HOTBACKUP */ -# ifdef __WIN__ -/* Add includes for the _stat() call to compile on Windows */ -# include -# include -# include -# endif /* __WIN__ */ -#endif /* !UNIV_HOTBACKUP */ - -#if defined(LINUX_NATIVE_AIO) -#include -#endif - -#ifdef _WIN32 -#define IOCP_SHUTDOWN_KEY (ULONG_PTR)-1 -#endif - -#if defined(UNIV_LINUX) && defined(HAVE_SYS_IOCTL_H) -# include -# ifndef DFS_IOCTL_ATOMIC_WRITE_SET -# define DFS_IOCTL_ATOMIC_WRITE_SET _IOW(0x95, 2, uint) -# endif -#endif - -#ifdef HAVE_LZO -#include "lzo/lzo1x.h" -#endif - -/** Insert buffer segment id */ -static const ulint IO_IBUF_SEGMENT = 0; - -/** Log segment id */ -static const ulint IO_LOG_SEGMENT = 1; - -/* This specifies the file permissions InnoDB uses when it creates files in -Unix; the value of os_innodb_umask is initialized in ha_innodb.cc to -my_umask */ - -#ifndef __WIN__ -/** Umask for creating files */ -UNIV_INTERN ulint os_innodb_umask = S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP; -#else -/** Umask for creating files */ -UNIV_INTERN ulint os_innodb_umask = 0; -#endif /* __WIN__ */ - -#ifndef UNIV_HOTBACKUP -/* We use these mutexes to protect lseek + file i/o operation, if the -OS does not provide an atomic pread or pwrite, or similar */ -#define OS_FILE_N_SEEK_MUTEXES 16 -UNIV_INTERN os_ib_mutex_t os_file_seek_mutexes[OS_FILE_N_SEEK_MUTEXES]; - -/* In simulated aio, merge at most this many consecutive i/os */ -#define OS_AIO_MERGE_N_CONSECUTIVE 64 - -#ifdef WITH_INNODB_DISALLOW_WRITES -#define WAIT_ALLOW_WRITES() os_event_wait(srv_allow_writes_event) -#else -#define WAIT_ALLOW_WRITES() do { } while (0) -#endif /* WITH_INNODB_DISALLOW_WRITES */ - -/********************************************************************** - -InnoDB AIO Implementation: -========================= - -We support native AIO for windows and linux. For rest of the platforms -we simulate AIO by special io-threads servicing the IO-requests. - -Simulated AIO: -============== - -In platforms where we 'simulate' AIO following is a rough explanation -of the high level design. -There are four io-threads (for ibuf, log, read, write). -All synchronous IO requests are serviced by the calling thread using -os_file_write/os_file_read. The Asynchronous requests are queued up -in an array (there are four such arrays) by the calling thread. -Later these requests are picked up by the io-thread and are serviced -synchronously. - -Windows native AIO: -================== - -If srv_use_native_aio is not set then windows follow the same -code as simulated AIO. If the flag is set then native AIO interface -is used. On windows, one of the limitation is that if a file is opened -for AIO no synchronous IO can be done on it. Therefore we have an -extra fifth array to queue up synchronous IO requests. -There are innodb_file_io_threads helper threads. These threads work -on the four arrays mentioned above in Simulated AIO. No thread is -required for the sync array. -If a synchronous IO request is made, it is first queued in the sync -array. Then the calling thread itself waits on the request, thus -making the call synchronous. -If an AIO request is made the calling thread not only queues it in the -array but also submits the requests. The helper thread then collects -the completed IO request and calls completion routine on it. - -Linux native AIO: -================= - -If we have libaio installed on the system and innodb_use_native_aio -is set to TRUE we follow the code path of native AIO, otherwise we -do simulated AIO. -There are innodb_file_io_threads helper threads. These threads work -on the four arrays mentioned above in Simulated AIO. -If a synchronous IO request is made, it is handled by calling -os_file_write/os_file_read. -If an AIO request is made the calling thread not only queues it in the -array but also submits the requests. The helper thread then collects -the completed IO request and calls completion routine on it. - -**********************************************************************/ - -/** Flag: enable debug printout for asynchronous i/o */ -UNIV_INTERN ibool os_aio_print_debug = FALSE; - -#ifdef UNIV_PFS_IO -/* Keys to register InnoDB I/O with performance schema */ -UNIV_INTERN mysql_pfs_key_t innodb_file_data_key; -UNIV_INTERN mysql_pfs_key_t innodb_file_log_key; -UNIV_INTERN mysql_pfs_key_t innodb_file_temp_key; -UNIV_INTERN mysql_pfs_key_t innodb_file_bmp_key; -#endif /* UNIV_PFS_IO */ - -/** The asynchronous i/o array slot structure */ -struct os_aio_slot_t{ -#ifdef WIN_ASYNC_IO - OVERLAPPED control; /*!< Windows control block for the - aio request, MUST be first element in the structure*/ - void *arr; /*!< Array this slot belongs to*/ -#endif - - ibool is_read; /*!< TRUE if a read operation */ - ulint pos; /*!< index of the slot in the aio - array */ - ibool reserved; /*!< TRUE if this slot is reserved */ - time_t reservation_time;/*!< time when reserved */ - ulint len; /*!< length of the block to read or - write */ - byte* buf; /*!< buffer used in i/o */ - ulint type; /*!< OS_FILE_READ or OS_FILE_WRITE */ - os_offset_t offset; /*!< file offset in bytes */ - os_file_t file; /*!< file where to read or write */ - const char* name; /*!< file name or path */ - ibool io_already_done;/*!< used only in simulated aio: - TRUE if the physical i/o already - made and only the slot message - needs to be passed to the caller - of os_aio_simulated_handle */ - ulint space_id; - fil_node_t* message1; /*!< message which is given by the */ - void* message2; /*!< the requester of an aio operation - and which can be used to identify - which pending aio operation was - completed */ - ulint bitmap; - - byte* page_compression_page; /*!< Memory allocated for - page compressed page and - freed after the write - has been completed */ - - byte* page_encryption_page; /*!< Memory allocated for - page encrypted page and - freed after the write - has been completed */ - - - ibool page_compression; - ulint page_compression_level; - - ibool page_encryption; - ulint page_encryption_key; - - ulint* write_size; /*!< Actual write size initialized - after fist successfull trim - operation for this page and if - initialized we do not trim again if - actual page size does not decrease. */ - - byte* page_buf; /*!< Actual page buffer for - page compressed pages, do not - free this */ - - byte* page_buf2; /*!< Actual page buffer for - page encrypted pages, do not - free this */ - - - ibool page_compress_success; - ibool page_encryption_success; - -#ifdef LINUX_NATIVE_AIO - struct iocb control; /* Linux control block for aio */ - int n_bytes; /* bytes written/read. */ - int ret; /* AIO return code */ -#endif /* WIN_ASYNC_IO */ - byte *lzo_mem; /* Temporal memory used by LZO */ -}; - -/** The asynchronous i/o array structure */ -struct os_aio_array_t{ - os_ib_mutex_t mutex; /*!< the mutex protecting the aio array */ - os_event_t not_full; - /*!< The event which is set to the - signaled state when there is space in - the aio outside the ibuf segment */ - os_event_t is_empty; - /*!< The event which is set to the - signaled state when there are no - pending i/os in this array */ - ulint n_slots;/*!< Total number of slots in the aio - array. This must be divisible by - n_threads. */ - ulint n_segments; - /*!< Number of segments in the aio - array of pending aio requests. A - thread can wait separately for any one - of the segments. */ - ulint cur_seg;/*!< We reserve IO requests in round - robin fashion to different segments. - This points to the segment that is to - be used to service next IO request. */ - ulint n_reserved; - /*!< Number of reserved slots in the - aio array outside the ibuf segment */ - os_aio_slot_t* slots; /*!< Pointer to the slots in the array */ - -#if defined(LINUX_NATIVE_AIO) - io_context_t* aio_ctx; - /* completion queue for IO. There is - one such queue per segment. Each thread - will work on one ctx exclusively. */ - struct io_event* aio_events; - /* The array to collect completed IOs. - There is one such event for each - possible pending IO. The size of the - array is equal to n_slots. */ -#endif /* LINUX_NATIV_AIO */ -}; - -#if defined(LINUX_NATIVE_AIO) -/** timeout for each io_getevents() call = 500ms. */ -#define OS_AIO_REAP_TIMEOUT (500000000UL) - -/** time to sleep, in microseconds if io_setup() returns EAGAIN. */ -#define OS_AIO_IO_SETUP_RETRY_SLEEP (500000UL) - -/** number of attempts before giving up on io_setup(). */ -#define OS_AIO_IO_SETUP_RETRY_ATTEMPTS 5 -#endif - -/** Array of events used in simulated aio */ -static os_event_t* os_aio_segment_wait_events = NULL; - -/** The aio arrays for non-ibuf i/o and ibuf i/o, as well as sync aio. These -are NULL when the module has not yet been initialized. @{ */ -static os_aio_array_t* os_aio_read_array = NULL; /*!< Reads */ -static os_aio_array_t* os_aio_write_array = NULL; /*!< Writes */ -static os_aio_array_t* os_aio_ibuf_array = NULL; /*!< Insert buffer */ -static os_aio_array_t* os_aio_log_array = NULL; /*!< Redo log */ -static os_aio_array_t* os_aio_sync_array = NULL; /*!< Synchronous I/O */ -/* @} */ - -/** Number of asynchronous I/O segments. Set by os_aio_init(). */ -static ulint os_aio_n_segments = ULINT_UNDEFINED; - -/** If the following is TRUE, read i/o handler threads try to -wait until a batch of new read requests have been posted */ -static ibool os_aio_recommend_sleep_for_read_threads = FALSE; -#endif /* !UNIV_HOTBACKUP */ - -UNIV_INTERN ulint os_n_file_reads = 0; -UNIV_INTERN ulint os_bytes_read_since_printout = 0; -UNIV_INTERN ulint os_n_file_writes = 0; -UNIV_INTERN ulint os_n_fsyncs = 0; -UNIV_INTERN ulint os_n_file_reads_old = 0; -UNIV_INTERN ulint os_n_file_writes_old = 0; -UNIV_INTERN ulint os_n_fsyncs_old = 0; -UNIV_INTERN time_t os_last_printout; - -UNIV_INTERN ibool os_has_said_disk_full = FALSE; - -#if !defined(UNIV_HOTBACKUP) \ - && (!defined(HAVE_ATOMIC_BUILTINS) || UNIV_WORD_SIZE < 8) -/** The mutex protecting the following counts of pending I/O operations */ -static os_ib_mutex_t os_file_count_mutex; -#endif /* !UNIV_HOTBACKUP && (!HAVE_ATOMIC_BUILTINS || UNIV_WORD_SIZE < 8) */ - -/** Number of pending os_file_pread() operations */ -UNIV_INTERN ulint os_file_n_pending_preads = 0; -/** Number of pending os_file_pwrite() operations */ -UNIV_INTERN ulint os_file_n_pending_pwrites = 0; -/** Number of pending write operations */ -UNIV_INTERN ulint os_n_pending_writes = 0; -/** Number of pending read operations */ -UNIV_INTERN ulint os_n_pending_reads = 0; - -/** After first fallocate failure we will disable os_file_trim */ -UNIV_INTERN ibool os_fallocate_failed = FALSE; - -/**********************************************************************//** -Directly manipulate the allocated disk space by deallocating for the file referred to -by fd for the byte range starting at offset and continuing for len bytes. -Within the specified range, partial file system blocks are zeroed, and whole -file system blocks are removed from the file. After a successful call, -subsequent reads from this range will return zeroes. -@return true if success, false if error */ -UNIV_INTERN -ibool -os_file_trim( -/*=========*/ - os_file_t file, /*!< in: file to be trimmed */ - os_aio_slot_t* slot, /*!< in: slot structure */ - ulint len); /*!< in: length of area */ - -/**********************************************************************//** -Allocate memory for temporal buffer used for page compression. This -buffer is freed later. */ -UNIV_INTERN -void -os_slot_alloc_page_buf( -/*===================*/ - os_aio_slot_t* slot); /*!< in: slot structure */ - -#ifdef HAVE_LZO -/**********************************************************************//** -Allocate memory for temporal memory used for page compression when -LZO compression method is used */ -UNIV_INTERN -void -os_slot_alloc_lzo_mem( -/*===================*/ - os_aio_slot_t* slot); /*!< in: slot structure */ -#endif - -/**********************************************************************//** -Allocate memory for temporal buffer used for page encryption. This -buffer is freed later. */ -UNIV_INTERN -void -os_slot_alloc_page_buf2( -os_aio_slot_t* slot); /*!< in: slot structure */ -/****************************************************************//** -Does error handling when a file operation fails. -@return TRUE if we should retry the operation */ -ibool -os_file_handle_error_no_exit( -/*=========================*/ - const char* name, /*!< in: name of a file or NULL */ - const char* operation, /*!< in: operation */ - ibool on_error_silent,/*!< in: if TRUE then don't print - any message to the log. */ - const char* file, /*!< in: file name */ - const ulint line); /*!< in: line */ - -/****************************************************************//** -Tries to enable the atomic write feature, if available, for the specified file -handle. -@return TRUE if success */ -static __attribute__((warn_unused_result)) -ibool -os_file_set_atomic_writes( -/*======================*/ - const char* name, /*!< in: name of the file */ - os_file_t file); /*!< in: handle to the file */ - -#ifdef UNIV_DEBUG -# ifndef UNIV_HOTBACKUP -/**********************************************************************//** -Validates the consistency the aio system some of the time. -@return TRUE if ok or the check was skipped */ -UNIV_INTERN -ibool -os_aio_validate_skip(void) -/*======================*/ -{ -/** Try os_aio_validate() every this many times */ -# define OS_AIO_VALIDATE_SKIP 13 - - /** The os_aio_validate() call skip counter. - Use a signed type because of the race condition below. */ - static int os_aio_validate_count = OS_AIO_VALIDATE_SKIP; - - /* There is a race condition below, but it does not matter, - because this call is only for heuristic purposes. We want to - reduce the call frequency of the costly os_aio_validate() - check in debug builds. */ - if (--os_aio_validate_count > 0) { - return(TRUE); - } - - os_aio_validate_count = OS_AIO_VALIDATE_SKIP; - return(os_aio_validate()); -} -# endif /* !UNIV_HOTBACKUP */ -#endif /* UNIV_DEBUG */ - -#ifdef _WIN32 -/** IO completion port used by background io threads */ -static HANDLE completion_port; -/** IO completion port used by background io READ threads */ -static HANDLE read_completion_port; -/** Thread local storage index for the per-thread event used for synchronous IO */ -static DWORD tls_sync_io = TLS_OUT_OF_INDEXES; -#endif - -#ifdef __WIN__ -/***********************************************************************//** -Gets the operating system version. Currently works only on Windows. -@return OS_WIN95, OS_WIN31, OS_WINNT, OS_WIN2000, OS_WINXP, OS_WINVISTA, -OS_WIN7. */ -UNIV_INTERN -ulint -os_get_os_version(void) -/*===================*/ -{ - OSVERSIONINFO os_info; - - os_info.dwOSVersionInfoSize = sizeof(OSVERSIONINFO); - - ut_a(GetVersionEx(&os_info)); - - if (os_info.dwPlatformId == VER_PLATFORM_WIN32s) { - return(OS_WIN31); - } else if (os_info.dwPlatformId == VER_PLATFORM_WIN32_WINDOWS) { - return(OS_WIN95); - } else if (os_info.dwPlatformId == VER_PLATFORM_WIN32_NT) { - switch (os_info.dwMajorVersion) { - case 3: - case 4: - return(OS_WINNT); - case 5: - return (os_info.dwMinorVersion == 0) - ? OS_WIN2000 : OS_WINXP; - case 6: - return (os_info.dwMinorVersion == 0) - ? OS_WINVISTA : OS_WIN7; - default: - return(OS_WIN7); - } - } else { - ut_error; - return(0); - } -} -#endif /* __WIN__ */ - - -#ifdef _WIN32 -/* -Windows : Handling synchronous IO on files opened asynchronously. - -If file is opened for asynchronous IO (FILE_FLAG_OVERLAPPED) and also bound to -a completion port, then every IO on this file would normally be enqueued to the -completion port. Sometimes however we would like to do a synchronous IO. This is -possible if we initialitze have overlapped.hEvent with a valid event and set its -lowest order bit to 1 (see MSDN ReadFile and WriteFile description for more info) - -We'll create this special event once for each thread and store in thread local -storage. -*/ - - -/***********************************************************************//** -Initialize tls index.for event handle used for synchronized IO on files that -might be opened with FILE_FLAG_OVERLAPPED. -*/ -static void win_init_syncio_event() -{ - tls_sync_io = TlsAlloc(); - ut_a(tls_sync_io != TLS_OUT_OF_INDEXES); -} - -/***********************************************************************//** -Retrieve per-thread event for doing synchronous io on asyncronously opened files -*/ -static HANDLE win_get_syncio_event() -{ - HANDLE h; - if(tls_sync_io == TLS_OUT_OF_INDEXES){ - win_init_syncio_event(); - } - - h = (HANDLE)TlsGetValue(tls_sync_io); - if (h) - return h; - h = CreateEventA(NULL, FALSE, FALSE, NULL); - ut_a(h); - h = (HANDLE)((uintptr_t)h | 1); - TlsSetValue(tls_sync_io, h); - return h; -} - -/* - TLS destructor, inspired by Chromium code - http://src.chromium.org/svn/trunk/src/base/threading/thread_local_storage_win.cc -*/ - -static void win_free_syncio_event() -{ - HANDLE h = win_get_syncio_event(); - if (h) { - CloseHandle(h); - } -} - -static void NTAPI win_tls_thread_exit(PVOID module, DWORD reason, PVOID reserved) { - if (DLL_THREAD_DETACH == reason || DLL_PROCESS_DETACH == reason) - win_free_syncio_event(); -} - -extern "C" { -#ifdef _WIN64 -#pragma comment(linker, "/INCLUDE:_tls_used") -#pragma comment(linker, "/INCLUDE:p_thread_callback_base") -#pragma const_seg(".CRT$XLB") -extern const PIMAGE_TLS_CALLBACK p_thread_callback_base; -const PIMAGE_TLS_CALLBACK p_thread_callback_base = win_tls_thread_exit; -#pragma data_seg() -#else -#pragma comment(linker, "/INCLUDE:__tls_used") -#pragma comment(linker, "/INCLUDE:_p_thread_callback_base") -#pragma data_seg(".CRT$XLB") -PIMAGE_TLS_CALLBACK p_thread_callback_base = win_tls_thread_exit; -#pragma data_seg() -#endif -} -#endif /*_WIN32 */ - -/***********************************************************************//** -Retrieves the last error number if an error occurs in a file io function. -The number should be retrieved before any other OS calls (because they may -overwrite the error number). If the number is not known to this program, -the OS error number + 100 is returned. -@return error number, or OS error number + 100 */ -static -ulint -os_file_get_last_error_low( -/*=======================*/ - bool report_all_errors, /*!< in: TRUE if we want an error - message printed of all errors */ - bool on_error_silent) /*!< in: TRUE then don't print any - diagnostic to the log */ -{ -#ifdef __WIN__ - - ulint err = (ulint) GetLastError(); - if (err == ERROR_SUCCESS) { - return(0); - } - - if (report_all_errors - || (!on_error_silent - && err != ERROR_DISK_FULL - && err != ERROR_FILE_EXISTS)) { - - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Operating system error number %lu" - " in a file operation.\n", (ulong) err); - - if (err == ERROR_PATH_NOT_FOUND) { - fprintf(stderr, - "InnoDB: The error means the system" - " cannot find the path specified.\n"); - - if (srv_is_being_started) { - fprintf(stderr, - "InnoDB: If you are installing InnoDB," - " remember that you must create\n" - "InnoDB: directories yourself, InnoDB" - " does not create them.\n"); - } - } else if (err == ERROR_ACCESS_DENIED) { - fprintf(stderr, - "InnoDB: The error means mysqld does not have" - " the access rights to\n" - "InnoDB: the directory. It may also be" - " you have created a subdirectory\n" - "InnoDB: of the same name as a data file.\n"); - } else if (err == ERROR_SHARING_VIOLATION - || err == ERROR_LOCK_VIOLATION) { - fprintf(stderr, - "InnoDB: The error means that another program" - " is using InnoDB's files.\n" - "InnoDB: This might be a backup or antivirus" - " software or another instance\n" - "InnoDB: of MySQL." - " Please close it to get rid of this error.\n"); - } else if (err == ERROR_WORKING_SET_QUOTA - || err == ERROR_NO_SYSTEM_RESOURCES) { - fprintf(stderr, - "InnoDB: The error means that there are no" - " sufficient system resources or quota to" - " complete the operation.\n"); - } else if (err == ERROR_OPERATION_ABORTED) { - fprintf(stderr, - "InnoDB: The error means that the I/O" - " operation has been aborted\n" - "InnoDB: because of either a thread exit" - " or an application request.\n" - "InnoDB: Retry attempt is made.\n"); - } else if (err == ECANCELED || err == ENOTTY) { - if (strerror(err) != NULL) { - fprintf(stderr, - "InnoDB: Error number %d" - " means '%s'.\n", - err, strerror(err)); - } - - if(srv_use_atomic_writes) { - fprintf(stderr, - "InnoDB: Error trying to enable atomic writes on " - "non-supported destination!\n"); - } - } else { - fprintf(stderr, - "InnoDB: Some operating system error numbers" - " are described at\n" - "InnoDB: " - REFMAN - "operating-system-error-codes.html\n"); - } - } - - fflush(stderr); - - if (err == ERROR_FILE_NOT_FOUND) { - return(OS_FILE_NOT_FOUND); - } else if (err == ERROR_DISK_FULL) { - return(OS_FILE_DISK_FULL); - } else if (err == ERROR_FILE_EXISTS) { - return(OS_FILE_ALREADY_EXISTS); - } else if (err == ERROR_SHARING_VIOLATION - || err == ERROR_LOCK_VIOLATION) { - return(OS_FILE_SHARING_VIOLATION); - } else if (err == ERROR_WORKING_SET_QUOTA - || err == ERROR_NO_SYSTEM_RESOURCES) { - return(OS_FILE_INSUFFICIENT_RESOURCE); - } else if (err == ERROR_OPERATION_ABORTED) { - return(OS_FILE_OPERATION_ABORTED); - } else if (err == ERROR_ACCESS_DENIED) { - return(OS_FILE_ACCESS_VIOLATION); - } else { - return(OS_FILE_ERROR_MAX + err); - } -#else - int err = errno; - if (err == 0) { - return(0); - } - - if (report_all_errors - || (err != ENOSPC && err != EEXIST && !on_error_silent)) { - - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Operating system error number %d" - " in a file operation.\n", err); - - if (err == ENOENT) { - fprintf(stderr, - "InnoDB: The error means the system" - " cannot find the path specified.\n"); - - if (srv_is_being_started) { - fprintf(stderr, - "InnoDB: If you are installing InnoDB," - " remember that you must create\n" - "InnoDB: directories yourself, InnoDB" - " does not create them.\n"); - } - } else if (err == EACCES) { - fprintf(stderr, - "InnoDB: The error means mysqld does not have" - " the access rights to\n" - "InnoDB: the directory.\n"); - } else if (err == ECANCELED || err == ENOTTY) { - if (strerror(err) != NULL) { - fprintf(stderr, - "InnoDB: Error number %d" - " means '%s'.\n", - err, strerror(err)); - } - - - if(srv_use_atomic_writes) { - fprintf(stderr, - "InnoDB: Error trying to enable atomic writes on " - "non-supported destination!\n"); - } - } else { - if (strerror(err) != NULL) { - fprintf(stderr, - "InnoDB: Error number %d" - " means '%s'.\n", - err, strerror(err)); - } - - - fprintf(stderr, - "InnoDB: Some operating system" - " error numbers are described at\n" - "InnoDB: " - REFMAN - "operating-system-error-codes.html\n"); - } - } - - fflush(stderr); - - switch (err) { - case ENOSPC: - return(OS_FILE_DISK_FULL); - case ENOENT: - return(OS_FILE_NOT_FOUND); - case EEXIST: - return(OS_FILE_ALREADY_EXISTS); - case EXDEV: - case ENOTDIR: - case EISDIR: - return(OS_FILE_PATH_ERROR); - case EAGAIN: - if (srv_use_native_aio) { - return(OS_FILE_AIO_RESOURCES_RESERVED); - } - break; - case ECANCELED: - case ENOTTY: - return(OS_FILE_OPERATION_NOT_SUPPORTED); - case EINTR: - if (srv_use_native_aio) { - return(OS_FILE_AIO_INTERRUPTED); - } - break; - case EACCES: - return(OS_FILE_ACCESS_VIOLATION); - } - return(OS_FILE_ERROR_MAX + err); -#endif -} - -/***********************************************************************//** -Retrieves the last error number if an error occurs in a file io function. -The number should be retrieved before any other OS calls (because they may -overwrite the error number). If the number is not known to this program, -the OS error number + 100 is returned. -@return error number, or OS error number + 100 */ -UNIV_INTERN -ulint -os_file_get_last_error( -/*===================*/ - bool report_all_errors) /*!< in: TRUE if we want an error - message printed of all errors */ -{ - return(os_file_get_last_error_low(report_all_errors, false)); -} - -/****************************************************************//** -Does error handling when a file operation fails. -Conditionally exits (calling exit(3)) based on should_exit value and the -error type, if should_exit is TRUE then on_error_silent is ignored. -@return TRUE if we should retry the operation */ -ibool -os_file_handle_error_cond_exit( -/*===========================*/ - const char* name, /*!< in: name of a file or NULL */ - const char* operation, /*!< in: operation */ - ibool should_exit, /*!< in: call exit(3) if unknown error - and this parameter is TRUE */ - ibool on_error_silent,/*!< in: if TRUE then don't print - any message to the log iff it is - an unknown non-fatal error */ - const char* file, /*!< in: file name */ - const ulint line) /*!< in: line */ -{ - ulint err; - - err = os_file_get_last_error_low(false, on_error_silent); - - switch (err) { - case OS_FILE_DISK_FULL: - /* We only print a warning about disk full once */ - - if (os_has_said_disk_full) { - - return(FALSE); - } - - /* Disk full error is reported irrespective of the - on_error_silent setting. */ - - if (name) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Encountered a problem with" - " file %s\n", name); - } - - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Disk is full. Try to clean the disk" - " to free space.\n"); - - os_has_said_disk_full = TRUE; - - fprintf(stderr, - " InnoDB: at file %s and at line %ld\n", file, line); - - fflush(stderr); - - return(FALSE); - - case OS_FILE_AIO_RESOURCES_RESERVED: - case OS_FILE_AIO_INTERRUPTED: - - return(TRUE); - - case OS_FILE_PATH_ERROR: - case OS_FILE_ALREADY_EXISTS: - case OS_FILE_ACCESS_VIOLATION: - - return(FALSE); - - case OS_FILE_SHARING_VIOLATION: - - os_thread_sleep(10000000); /* 10 sec */ - return(TRUE); - - case OS_FILE_OPERATION_ABORTED: - case OS_FILE_INSUFFICIENT_RESOURCE: - - os_thread_sleep(100000); /* 100 ms */ - return(TRUE); - - default: - - /* If it is an operation that can crash on error then it - is better to ignore on_error_silent and print an error message - to the log. */ - - if (should_exit || !on_error_silent) { - fprintf(stderr, - " InnoDB: Operation %s to file %s and at line %ld\n", - operation, file, line); - } - - if (should_exit || !on_error_silent) { - ib_logf(IB_LOG_LEVEL_ERROR, "File %s: '%s' returned OS " - "error " ULINTPF ".%s", name ? name : "(unknown)", - operation, err, should_exit - ? " Cannot continue operation" : ""); - } - - if (should_exit) { - exit(1); - } - } - - return(FALSE); -} - -/****************************************************************//** -Does error handling when a file operation fails. -@return TRUE if we should retry the operation */ -static -ibool -os_file_handle_error( -/*=================*/ - const char* name, /*!< in: name of a file or NULL */ - const char* operation, /*!< in: operation */ - const char* file, /*!< in: file name */ - const ulint line) /*!< in: line */ -{ - /* exit in case of unknown error */ - return(os_file_handle_error_cond_exit(name, operation, TRUE, FALSE, file, line)); -} - -/****************************************************************//** -Does error handling when a file operation fails. -@return TRUE if we should retry the operation */ -ibool -os_file_handle_error_no_exit( -/*=========================*/ - const char* name, /*!< in: name of a file or NULL */ - const char* operation, /*!< in: operation */ - ibool on_error_silent,/*!< in: if TRUE then don't print - any message to the log. */ - const char* file, /*!< in: file name */ - const ulint line) /*!< in: line */ -{ - /* don't exit in case of unknown error */ - return(os_file_handle_error_cond_exit( - name, operation, FALSE, on_error_silent, file, line)); -} - -#undef USE_FILE_LOCK -#define USE_FILE_LOCK -#if defined(UNIV_HOTBACKUP) || defined(__WIN__) -/* InnoDB Hot Backup does not lock the data files. - * On Windows, mandatory locking is used. - */ -# undef USE_FILE_LOCK -#endif -#ifdef USE_FILE_LOCK -/****************************************************************//** -Obtain an exclusive lock on a file. -@return 0 on success */ -static -int -os_file_lock( -/*=========*/ - int fd, /*!< in: file descriptor */ - const char* name) /*!< in: file name */ -{ - struct flock lk; - - ut_ad(!srv_read_only_mode); - - lk.l_type = F_WRLCK; - lk.l_whence = SEEK_SET; - lk.l_start = lk.l_len = 0; - - if (fcntl(fd, F_SETLK, &lk) == -1) { - - ib_logf(IB_LOG_LEVEL_ERROR, - "Unable to lock %s, error: %d", name, errno); - - if (errno == EAGAIN || errno == EACCES) { - ib_logf(IB_LOG_LEVEL_INFO, - "Check that you do not already have " - "another mysqld process using the " - "same InnoDB data or log files."); - } - - return(-1); - } - - return(0); -} -#endif /* USE_FILE_LOCK */ - -#ifndef UNIV_HOTBACKUP -/****************************************************************//** -Creates the seek mutexes used in positioned reads and writes. */ -UNIV_INTERN -void -os_io_init_simple(void) -/*===================*/ -{ -#if !defined(HAVE_ATOMIC_BUILTINS) || UNIV_WORD_SIZE < 8 - os_file_count_mutex = os_mutex_create(); -#endif /* !HAVE_ATOMIC_BUILTINS || UNIV_WORD_SIZE < 8 */ - - for (ulint i = 0; i < OS_FILE_N_SEEK_MUTEXES; i++) { - os_file_seek_mutexes[i] = os_mutex_create(); - } -#ifdef _WIN32 - win_init_syncio_event(); -#endif -} - -/***********************************************************************//** -Creates a temporary file. This function is like tmpfile(3), but -the temporary file is created in the MySQL temporary directory. -@return temporary file handle, or NULL on error */ -UNIV_INTERN -FILE* -os_file_create_tmpfile(void) -/*========================*/ -{ - FILE* file = NULL; - int fd; - WAIT_ALLOW_WRITES(); - fd = innobase_mysql_tmpfile(); - - ut_ad(!srv_read_only_mode); - - if (fd >= 0) { - file = fdopen(fd, "w+b"); - } - - if (!file) { - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Error: unable to create temporary file;" - " errno: %d\n", errno); - if (fd >= 0) { - close(fd); - } - } - - return(file); -} -#endif /* !UNIV_HOTBACKUP */ - -/***********************************************************************//** -The os_file_opendir() function opens a directory stream corresponding to the -directory named by the dirname argument. The directory stream is positioned -at the first entry. In both Unix and Windows we automatically skip the '.' -and '..' items at the start of the directory listing. -@return directory stream, NULL if error */ -UNIV_INTERN -os_file_dir_t -os_file_opendir( -/*============*/ - const char* dirname, /*!< in: directory name; it must not - contain a trailing '\' or '/' */ - ibool error_is_fatal) /*!< in: TRUE if we should treat an - error as a fatal error; if we try to - open symlinks then we do not wish a - fatal error if it happens not to be - a directory */ -{ - os_file_dir_t dir; -#ifdef __WIN__ - LPWIN32_FIND_DATA lpFindFileData; - char path[OS_FILE_MAX_PATH + 3]; - - ut_a(strlen(dirname) < OS_FILE_MAX_PATH); - - strcpy(path, dirname); - strcpy(path + strlen(path), "\\*"); - - /* Note that in Windows opening the 'directory stream' also retrieves - the first entry in the directory. Since it is '.', that is no problem, - as we will skip over the '.' and '..' entries anyway. */ - - lpFindFileData = static_cast( - ut_malloc(sizeof(WIN32_FIND_DATA))); - - dir = FindFirstFile((LPCTSTR) path, lpFindFileData); - - ut_free(lpFindFileData); - - if (dir == INVALID_HANDLE_VALUE) { - - if (error_is_fatal) { - os_file_handle_error(dirname, "opendir", __FILE__, __LINE__); - } - - return(NULL); - } - - return(dir); -#else - dir = opendir(dirname); - - if (dir == NULL && error_is_fatal) { - os_file_handle_error(dirname, "opendir", __FILE__, __LINE__); - } - - return(dir); -#endif /* __WIN__ */ -} - -/***********************************************************************//** -Closes a directory stream. -@return 0 if success, -1 if failure */ -UNIV_INTERN -int -os_file_closedir( -/*=============*/ - os_file_dir_t dir) /*!< in: directory stream */ -{ -#ifdef __WIN__ - BOOL ret; - - ret = FindClose(dir); - - if (!ret) { - os_file_handle_error_no_exit(NULL, "closedir", FALSE, __FILE__, __LINE__); - - return(-1); - } - - return(0); -#else - int ret; - - ret = closedir(dir); - - if (ret) { - os_file_handle_error_no_exit(NULL, "closedir", FALSE, __FILE__, __LINE__); - } - - return(ret); -#endif /* __WIN__ */ -} - -/***********************************************************************//** -This function returns information of the next file in the directory. We jump -over the '.' and '..' entries in the directory. -@return 0 if ok, -1 if error, 1 if at the end of the directory */ -UNIV_INTERN -int -os_file_readdir_next_file( -/*======================*/ - const char* dirname,/*!< in: directory name or path */ - os_file_dir_t dir, /*!< in: directory stream */ - os_file_stat_t* info) /*!< in/out: buffer where the info is returned */ -{ -#ifdef __WIN__ - LPWIN32_FIND_DATA lpFindFileData; - BOOL ret; - - lpFindFileData = static_cast( - ut_malloc(sizeof(WIN32_FIND_DATA))); -next_file: - ret = FindNextFile(dir, lpFindFileData); - - if (ret) { - ut_a(strlen((char*) lpFindFileData->cFileName) - < OS_FILE_MAX_PATH); - - if (strcmp((char*) lpFindFileData->cFileName, ".") == 0 - || strcmp((char*) lpFindFileData->cFileName, "..") == 0) { - - goto next_file; - } - - strcpy(info->name, (char*) lpFindFileData->cFileName); - - info->size = (ib_int64_t)(lpFindFileData->nFileSizeLow) - + (((ib_int64_t)(lpFindFileData->nFileSizeHigh)) - << 32); - - if (lpFindFileData->dwFileAttributes - & FILE_ATTRIBUTE_REPARSE_POINT) { - /* TODO: test Windows symlinks */ - /* TODO: MySQL has apparently its own symlink - implementation in Windows, dbname.sym can - redirect a database directory: - REFMAN "windows-symbolic-links.html" */ - info->type = OS_FILE_TYPE_LINK; - } else if (lpFindFileData->dwFileAttributes - & FILE_ATTRIBUTE_DIRECTORY) { - info->type = OS_FILE_TYPE_DIR; - } else { - /* It is probably safest to assume that all other - file types are normal. Better to check them rather - than blindly skip them. */ - - info->type = OS_FILE_TYPE_FILE; - } - } - - ut_free(lpFindFileData); - - if (ret) { - return(0); - } else if (GetLastError() == ERROR_NO_MORE_FILES) { - - return(1); - } else { - os_file_handle_error_no_exit(NULL, "readdir_next_file", FALSE, __FILE__, __LINE__); - return(-1); - } -#else - struct dirent* ent; - char* full_path; - int ret; - struct stat statinfo; -#ifdef HAVE_READDIR_R - char dirent_buf[sizeof(struct dirent) - + _POSIX_PATH_MAX + 100]; - /* In /mysys/my_lib.c, _POSIX_PATH_MAX + 1 is used as - the max file name len; but in most standards, the - length is NAME_MAX; we add 100 to be even safer */ -#endif - -next_file: - -#ifdef HAVE_READDIR_R - ret = readdir_r(dir, (struct dirent*) dirent_buf, &ent); - - if (ret != 0 -#ifdef UNIV_AIX - /* On AIX, only if we got non-NULL 'ent' (result) value and - a non-zero 'ret' (return) value, it indicates a failed - readdir_r() call. An NULL 'ent' with an non-zero 'ret' - would indicate the "end of the directory" is reached. */ - && ent != NULL -#endif - ) { - fprintf(stderr, - "InnoDB: cannot read directory %s, error %lu\n", - dirname, (ulong) ret); - - return(-1); - } - - if (ent == NULL) { - /* End of directory */ - - return(1); - } - - ut_a(strlen(ent->d_name) < _POSIX_PATH_MAX + 100 - 1); -#else - ent = readdir(dir); - - if (ent == NULL) { - - return(1); - } -#endif - ut_a(strlen(ent->d_name) < OS_FILE_MAX_PATH); - - if (strcmp(ent->d_name, ".") == 0 || strcmp(ent->d_name, "..") == 0) { - - goto next_file; - } - - strcpy(info->name, ent->d_name); - - full_path = static_cast( - ut_malloc(strlen(dirname) + strlen(ent->d_name) + 10)); - - sprintf(full_path, "%s/%s", dirname, ent->d_name); - - ret = stat(full_path, &statinfo); - - if (ret) { - - if (errno == ENOENT) { - /* readdir() returned a file that does not exist, - it must have been deleted in the meantime. Do what - would have happened if the file was deleted before - readdir() - ignore and go to the next entry. - If this is the last entry then info->name will still - contain the name of the deleted file when this - function returns, but this is not an issue since the - caller shouldn't be looking at info when end of - directory is returned. */ - - ut_free(full_path); - - goto next_file; - } - - os_file_handle_error_no_exit(full_path, "stat", FALSE, __FILE__, __LINE__); - - ut_free(full_path); - - return(-1); - } - - info->size = (ib_int64_t) statinfo.st_size; - - if (S_ISDIR(statinfo.st_mode)) { - info->type = OS_FILE_TYPE_DIR; - } else if (S_ISLNK(statinfo.st_mode)) { - info->type = OS_FILE_TYPE_LINK; - } else if (S_ISREG(statinfo.st_mode)) { - info->type = OS_FILE_TYPE_FILE; - } else { - info->type = OS_FILE_TYPE_UNKNOWN; - } - - ut_free(full_path); - - return(0); -#endif -} - -/*****************************************************************//** -This function attempts to create a directory named pathname. The new -directory gets default permissions. On Unix the permissions are -(0770 & ~umask). If the directory exists already, nothing is done and -the call succeeds, unless the fail_if_exists arguments is true. -If another error occurs, such as a permission error, this does not crash, -but reports the error and returns FALSE. -@return TRUE if call succeeds, FALSE on error */ -UNIV_INTERN -ibool -os_file_create_directory( -/*=====================*/ - const char* pathname, /*!< in: directory name as - null-terminated string */ - ibool fail_if_exists) /*!< in: if TRUE, pre-existing directory - is treated as an error. */ -{ -#ifdef __WIN__ - BOOL rcode; - - rcode = CreateDirectory((LPCTSTR) pathname, NULL); - if (!(rcode != 0 - || (GetLastError() == ERROR_ALREADY_EXISTS - && !fail_if_exists))) { - - os_file_handle_error_no_exit( - pathname, "CreateDirectory", FALSE, __FILE__, __LINE__); - - return(FALSE); - } - - return(TRUE); -#else - int rcode; - WAIT_ALLOW_WRITES(); - - rcode = mkdir(pathname, 0770); - - if (!(rcode == 0 || (errno == EEXIST && !fail_if_exists))) { - /* failure */ - os_file_handle_error_no_exit(pathname, "mkdir", FALSE, __FILE__, __LINE__); - - return(FALSE); - } - - return (TRUE); -#endif /* __WIN__ */ -} - -/****************************************************************//** -NOTE! Use the corresponding macro os_file_create_simple(), not directly -this function! -A simple function to open or create a file. -@return own: handle to the file, not defined if error, error number -can be retrieved with os_file_get_last_error */ -UNIV_INTERN -os_file_t -os_file_create_simple_func( -/*=======================*/ - const char* name, /*!< in: name of the file or path as a - null-terminated string */ - ulint create_mode,/*!< in: create mode */ - ulint access_type,/*!< in: OS_FILE_READ_ONLY or - OS_FILE_READ_WRITE */ - ibool* success)/*!< out: TRUE if succeed, FALSE if error */ -{ - os_file_t file; - ibool retry; - - *success = FALSE; -#ifdef __WIN__ - DWORD access; - DWORD create_flag; - DWORD attributes = 0; - - ut_a(!(create_mode & OS_FILE_ON_ERROR_SILENT)); - ut_a(!(create_mode & OS_FILE_ON_ERROR_NO_EXIT)); - - if (create_mode == OS_FILE_OPEN) { - - create_flag = OPEN_EXISTING; - - } else if (srv_read_only_mode) { - - create_flag = OPEN_EXISTING; - - } else if (create_mode == OS_FILE_CREATE) { - - create_flag = CREATE_NEW; - - } else if (create_mode == OS_FILE_CREATE_PATH) { - - ut_a(!srv_read_only_mode); - - /* Create subdirs along the path if needed */ - *success = os_file_create_subdirs_if_needed(name); - - if (!*success) { - - ib_logf(IB_LOG_LEVEL_ERROR, - "Unable to create subdirectories '%s'", - name); - - return((os_file_t) -1); - } - - create_flag = CREATE_NEW; - create_mode = OS_FILE_CREATE; - - } else { - ib_logf(IB_LOG_LEVEL_ERROR, - "Unknown file create mode (%lu) for file '%s'", - create_mode, name); - - return((os_file_t) -1); - } - - if (access_type == OS_FILE_READ_ONLY) { - access = GENERIC_READ; - } else if (srv_read_only_mode) { - - ib_logf(IB_LOG_LEVEL_INFO, - "read only mode set. Unable to " - "open file '%s' in RW mode, trying RO mode", name); - - access = GENERIC_READ; - - } else if (access_type == OS_FILE_READ_WRITE) { - access = GENERIC_READ | GENERIC_WRITE; - } else { - ib_logf(IB_LOG_LEVEL_ERROR, - "Unknown file access type (%lu) for file '%s'", - access_type, name); - - return((os_file_t) -1); - } - - do { - /* Use default security attributes and no template file. */ - - file = CreateFile( - (LPCTSTR) name, access, FILE_SHARE_READ, NULL, - create_flag, attributes, NULL); - - if (file == INVALID_HANDLE_VALUE) { - - *success = FALSE; - - retry = os_file_handle_error( - name, create_mode == OS_FILE_OPEN ? - "open" : "create", __FILE__, __LINE__); - - } else { - *success = TRUE; - retry = false; - } - - } while (retry); - -#else /* __WIN__ */ - int create_flag; - if (create_mode != OS_FILE_OPEN && create_mode != OS_FILE_OPEN_RAW) - WAIT_ALLOW_WRITES(); - - ut_a(!(create_mode & OS_FILE_ON_ERROR_SILENT)); - ut_a(!(create_mode & OS_FILE_ON_ERROR_NO_EXIT)); - - if (create_mode == OS_FILE_OPEN) { - - if (access_type == OS_FILE_READ_ONLY) { - create_flag = O_RDONLY; - } else if (srv_read_only_mode) { - create_flag = O_RDONLY; - } else { - create_flag = O_RDWR; - } - - } else if (srv_read_only_mode) { - - create_flag = O_RDONLY; - - } else if (create_mode == OS_FILE_CREATE) { - - create_flag = O_RDWR | O_CREAT | O_EXCL; - - } else if (create_mode == OS_FILE_CREATE_PATH) { - - /* Create subdirs along the path if needed */ - - *success = os_file_create_subdirs_if_needed(name); - - if (!*success) { - - ib_logf(IB_LOG_LEVEL_ERROR, - "Unable to create subdirectories '%s'", - name); - - return((os_file_t) -1); - } - - create_flag = O_RDWR | O_CREAT | O_EXCL; - create_mode = OS_FILE_CREATE; - } else { - - ib_logf(IB_LOG_LEVEL_ERROR, - "Unknown file create mode (%lu) for file '%s'", - create_mode, name); - - return((os_file_t) -1); - } - - do { - file = ::open(name, create_flag, os_innodb_umask); - - if (file == -1) { - *success = FALSE; - - retry = os_file_handle_error( - name, - create_mode == OS_FILE_OPEN - ? "open" : "create", __FILE__, __LINE__); - } else { - *success = TRUE; - retry = false; - } - - } while (retry); - -#ifdef USE_FILE_LOCK - if (!srv_read_only_mode - && *success - && access_type == OS_FILE_READ_WRITE - && os_file_lock(file, name)) { - - *success = FALSE; - close(file); - file = -1; - } -#endif /* USE_FILE_LOCK */ - -#endif /* __WIN__ */ - - return(file); -} - -/****************************************************************//** -NOTE! Use the corresponding macro -os_file_create_simple_no_error_handling(), not directly this function! -A simple function to open or create a file. -@return own: handle to the file, not defined if error, error number -can be retrieved with os_file_get_last_error */ -UNIV_INTERN -os_file_t -os_file_create_simple_no_error_handling_func( -/*=========================================*/ - const char* name, /*!< in: name of the file or path as a - null-terminated string */ - ulint create_mode,/*!< in: create mode */ - ulint access_type,/*!< in: OS_FILE_READ_ONLY, - OS_FILE_READ_WRITE, or - OS_FILE_READ_ALLOW_DELETE; the last option is - used by a backup program reading the file */ - ibool* success,/*!< out: TRUE if succeed, FALSE if error */ - ulint atomic_writes) /*! in: atomic writes table option - value */ -{ - os_file_t file; - atomic_writes_t awrites = (atomic_writes_t) atomic_writes; - - *success = FALSE; -#ifdef __WIN__ - DWORD access; - DWORD create_flag; - DWORD attributes = 0; - DWORD share_mode = FILE_SHARE_READ; - - ut_a(name); - - ut_a(!(create_mode & OS_FILE_ON_ERROR_SILENT)); - ut_a(!(create_mode & OS_FILE_ON_ERROR_NO_EXIT)); - - if (create_mode == OS_FILE_OPEN) { - create_flag = OPEN_EXISTING; - } else if (srv_read_only_mode) { - create_flag = OPEN_EXISTING; - } else if (create_mode == OS_FILE_CREATE) { - create_flag = CREATE_NEW; - } else { - - ib_logf(IB_LOG_LEVEL_ERROR, - "Unknown file create mode (%lu) for file '%s'", - create_mode, name); - - return((os_file_t) -1); - } - - if (access_type == OS_FILE_READ_ONLY) { - access = GENERIC_READ; - } else if (srv_read_only_mode) { - access = GENERIC_READ; - } else if (access_type == OS_FILE_READ_WRITE) { - access = GENERIC_READ | GENERIC_WRITE; - } else if (access_type == OS_FILE_READ_ALLOW_DELETE) { - - ut_a(!srv_read_only_mode); - - access = GENERIC_READ; - - /*!< A backup program has to give mysqld the maximum - freedom to do what it likes with the file */ - - share_mode |= FILE_SHARE_DELETE | FILE_SHARE_WRITE; - } else { - ib_logf(IB_LOG_LEVEL_ERROR, - "Unknown file access type (%lu) for file '%s'", - access_type, name); - - return((os_file_t) -1); - } - - file = CreateFile((LPCTSTR) name, - access, - share_mode, - NULL, // Security attributes - create_flag, - attributes, - NULL); // No template file - - /* If we have proper file handle and atomic writes should be used, - try to set atomic writes and if that fails when creating a new - table, produce a error. If atomic writes are used on existing - file, ignore error and use traditional writes for that file */ - if (file != INVALID_HANDLE_VALUE - && (awrites == ATOMIC_WRITES_ON || - (srv_use_atomic_writes && awrites == ATOMIC_WRITES_DEFAULT)) - && !os_file_set_atomic_writes(name, file)) { - if (create_mode == OS_FILE_CREATE) { - fprintf(stderr, "InnoDB: Error: Can't create file using atomic writes\n"); - CloseHandle(file); - os_file_delete_if_exists_func(name); - *success = FALSE; - file = INVALID_HANDLE_VALUE; - } - } - - *success = (file != INVALID_HANDLE_VALUE); -#else /* __WIN__ */ - int create_flag; - - ut_a(name); - if (create_mode != OS_FILE_OPEN && create_mode != OS_FILE_OPEN_RAW) - WAIT_ALLOW_WRITES(); - - ut_a(!(create_mode & OS_FILE_ON_ERROR_SILENT)); - ut_a(!(create_mode & OS_FILE_ON_ERROR_NO_EXIT)); - - if (create_mode == OS_FILE_OPEN) { - - if (access_type == OS_FILE_READ_ONLY) { - - create_flag = O_RDONLY; - - } else if (srv_read_only_mode) { - - create_flag = O_RDONLY; - - } else { - - ut_a(access_type == OS_FILE_READ_WRITE - || access_type == OS_FILE_READ_ALLOW_DELETE); - - create_flag = O_RDWR; - } - - } else if (srv_read_only_mode) { - - create_flag = O_RDONLY; - - } else if (create_mode == OS_FILE_CREATE) { - - create_flag = O_RDWR | O_CREAT | O_EXCL; - - } else { - ib_logf(IB_LOG_LEVEL_ERROR, - "Unknown file create mode (%lu) for file '%s'", - create_mode, name); - - return((os_file_t) -1); - } - - file = ::open(name, create_flag, os_innodb_umask); - - *success = file == -1 ? FALSE : TRUE; - -#ifdef USE_FILE_LOCK - if (!srv_read_only_mode - && *success - && access_type == OS_FILE_READ_WRITE - && os_file_lock(file, name)) { - - *success = FALSE; - close(file); - file = -1; - - } -#endif /* USE_FILE_LOCK */ - - /* If we have proper file handle and atomic writes should be used, - try to set atomic writes and if that fails when creating a new - table, produce a error. If atomic writes are used on existing - file, ignore error and use traditional writes for that file */ - if (file != -1 - && (awrites == ATOMIC_WRITES_ON || - (srv_use_atomic_writes && awrites == ATOMIC_WRITES_DEFAULT)) - && !os_file_set_atomic_writes(name, file)) { - if (create_mode == OS_FILE_CREATE) { - fprintf(stderr, "InnoDB: Error: Can't create file using atomic writes\n"); - close(file); - os_file_delete_if_exists_func(name); - *success = FALSE; - file = -1; - } - } - -#endif /* __WIN__ */ - - return(file); -} - -/****************************************************************//** -Tries to disable OS caching on an opened file descriptor. */ -UNIV_INTERN -void -os_file_set_nocache( -/*================*/ - int fd /*!< in: file descriptor to alter */ - __attribute__((unused)), - const char* file_name /*!< in: used in the diagnostic - message */ - __attribute__((unused)), - const char* operation_name __attribute__((unused))) - /*!< in: "open" or "create"; used - in the diagnostic message */ -{ - /* some versions of Solaris may not have DIRECTIO_ON */ -#if defined(UNIV_SOLARIS) && defined(DIRECTIO_ON) - if (directio(fd, DIRECTIO_ON) == -1) { - int errno_save = errno; - - ib_logf(IB_LOG_LEVEL_ERROR, - "Failed to set DIRECTIO_ON on file %s: %s: %s, " - "continuing anyway.", - file_name, operation_name, strerror(errno_save)); - } -#elif defined(O_DIRECT) - if (fcntl(fd, F_SETFL, O_DIRECT) == -1) { - int errno_save = errno; - static bool warning_message_printed = false; - if (errno_save == EINVAL) { - if (!warning_message_printed) { - warning_message_printed = true; -# ifdef UNIV_LINUX - ib_logf(IB_LOG_LEVEL_WARN, - "Failed to set O_DIRECT on file " - "%s: %s: %s, continuing anyway. " - "O_DIRECT is known to result " - "in 'Invalid argument' on Linux on " - "tmpfs, see MySQL Bug#26662.", - file_name, operation_name, - strerror(errno_save)); -# else /* UNIV_LINUX */ - goto short_warning; -# endif /* UNIV_LINUX */ - } - } else { -# ifndef UNIV_LINUX -short_warning: -# endif - ib_logf(IB_LOG_LEVEL_WARN, - "Failed to set O_DIRECT on file %s: %s: %s, " - "continuing anyway.", - file_name, operation_name, strerror(errno_save)); - } - } -#endif /* defined(UNIV_SOLARIS) && defined(DIRECTIO_ON) */ -} - - -/****************************************************************//** -Tries to enable the atomic write feature, if available, for the specified file -handle. -@return TRUE if success */ -static __attribute__((warn_unused_result)) -ibool -os_file_set_atomic_writes( -/*======================*/ - const char* name /*!< in: name of the file */ - __attribute__((unused)), - os_file_t file /*!< in: handle to the file */ - __attribute__((unused))) - -{ -#ifdef DFS_IOCTL_ATOMIC_WRITE_SET - int atomic_option = 1; - - if (ioctl(file, DFS_IOCTL_ATOMIC_WRITE_SET, &atomic_option)) { - - fprintf(stderr, "InnoDB: Warning:Trying to enable atomic writes on " - "file %s on non-supported platform!\n", name); - os_file_handle_error_no_exit(name, "ioctl(DFS_IOCTL_ATOMIC_WRITE_SET)", FALSE, __FILE__, __LINE__); - return(FALSE); - } - - return(TRUE); -#else - fprintf(stderr, "InnoDB: Error: trying to enable atomic writes on " - "file %s on non-supported platform!\n", name); - return(FALSE); -#endif -} - -/****************************************************************//** -NOTE! Use the corresponding macro os_file_create(), not directly -this function! -Opens an existing file or creates a new. -@return own: handle to the file, not defined if error, error number -can be retrieved with os_file_get_last_error */ -UNIV_INTERN -os_file_t -os_file_create_func( -/*================*/ - const char* name, /*!< in: name of the file or path as a - null-terminated string */ - ulint create_mode,/*!< in: create mode */ - ulint purpose,/*!< in: OS_FILE_AIO, if asynchronous, - non-buffered i/o is desired, - OS_FILE_NORMAL, if any normal file; - NOTE that it also depends on type, os_aio_.. - and srv_.. variables whether we really use - async i/o or unbuffered i/o: look in the - function source code for the exact rules */ - ulint type, /*!< in: OS_DATA_FILE or OS_LOG_FILE */ - ibool* success,/*!< out: TRUE if succeed, FALSE if error */ - ulint atomic_writes) /*! in: atomic writes table option - value */ -{ - os_file_t file; - ibool retry; - ibool on_error_no_exit; - ibool on_error_silent; - atomic_writes_t awrites = (atomic_writes_t) atomic_writes; - -#ifdef __WIN__ - DBUG_EXECUTE_IF( - "ib_create_table_fail_disk_full", - *success = FALSE; - SetLastError(ERROR_DISK_FULL); - return((os_file_t) -1); - ); -#else /* __WIN__ */ - DBUG_EXECUTE_IF( - "ib_create_table_fail_disk_full", - *success = FALSE; - errno = ENOSPC; - return((os_file_t) -1); - ); -#endif /* __WIN__ */ - -#ifdef __WIN__ - DWORD create_flag; - DWORD share_mode = FILE_SHARE_READ; - - on_error_no_exit = create_mode & OS_FILE_ON_ERROR_NO_EXIT - ? TRUE : FALSE; - - on_error_silent = create_mode & OS_FILE_ON_ERROR_SILENT - ? TRUE : FALSE; - - create_mode &= ~OS_FILE_ON_ERROR_NO_EXIT; - create_mode &= ~OS_FILE_ON_ERROR_SILENT; - - if (create_mode == OS_FILE_OPEN_RAW) { - - ut_a(!srv_read_only_mode); - - create_flag = OPEN_EXISTING; - - /* On Windows Physical devices require admin privileges and - have to have the write-share mode set. See the remarks - section for the CreateFile() function documentation in MSDN. */ - - share_mode |= FILE_SHARE_WRITE; - - } else if (create_mode == OS_FILE_OPEN - || create_mode == OS_FILE_OPEN_RETRY) { - - create_flag = OPEN_EXISTING; - - } else if (srv_read_only_mode) { - - create_flag = OPEN_EXISTING; - - } else if (create_mode == OS_FILE_CREATE) { - - create_flag = CREATE_NEW; - - } else if (create_mode == OS_FILE_OVERWRITE) { - - create_flag = CREATE_ALWAYS; - - } else { - ib_logf(IB_LOG_LEVEL_ERROR, - "Unknown file create mode (%lu) for file '%s'", - create_mode, name); - - return((os_file_t) -1); - } - - DWORD attributes = 0; - -#ifdef UNIV_HOTBACKUP - attributes |= FILE_FLAG_NO_BUFFERING; -#else - if (purpose == OS_FILE_AIO) { - -#ifdef WIN_ASYNC_IO - /* If specified, use asynchronous (overlapped) io and no - buffering of writes in the OS */ - - if (srv_use_native_aio) { - attributes |= FILE_FLAG_OVERLAPPED; - } -#endif /* WIN_ASYNC_IO */ - - } else if (purpose == OS_FILE_NORMAL) { - /* Use default setting. */ - } else { - ib_logf(IB_LOG_LEVEL_ERROR, - "Unknown purpose flag (%lu) while opening file '%s'", - purpose, name); - - return((os_file_t)(-1)); - } - -#ifdef UNIV_NON_BUFFERED_IO - // TODO: Create a bug, this looks wrong. The flush log - // parameter is dynamic. - if (type == OS_LOG_FILE && thd_flush_log_at_trx_commit(NULL) == 2) { - - /* Do not use unbuffered i/o for the log files because - value 2 denotes that we do not flush the log at every - commit, but only once per second */ - - } else if (srv_win_file_flush_method == SRV_WIN_IO_UNBUFFERED) { - - attributes |= FILE_FLAG_NO_BUFFERING; - } -#endif /* UNIV_NON_BUFFERED_IO */ - -#endif /* UNIV_HOTBACKUP */ - DWORD access = GENERIC_READ; - - if (!srv_read_only_mode) { - access |= GENERIC_WRITE; - } - - if (type == OS_LOG_FILE) { - if (srv_unix_file_flush_method == SRV_UNIX_O_DSYNC) { - /* Map O_DSYNC to WRITE_THROUGH */ - attributes |= FILE_FLAG_WRITE_THROUGH; - } else if (srv_unix_file_flush_method == SRV_UNIX_ALL_O_DIRECT) { - /* Open log file without buffering */ - attributes |= FILE_FLAG_NO_BUFFERING; - } - } - - do { - /* Use default security attributes and no template file. */ - file = CreateFile( - (LPCTSTR) name, access, share_mode, NULL, - create_flag, attributes, NULL); - - if (file == INVALID_HANDLE_VALUE) { - const char* operation; - - operation = (create_mode == OS_FILE_CREATE - && !srv_read_only_mode) - ? "create" : "open"; - - *success = FALSE; - - if (on_error_no_exit) { - retry = os_file_handle_error_no_exit( - name, operation, on_error_silent, __FILE__, __LINE__); - } else { - retry = os_file_handle_error(name, operation, __FILE__, __LINE__); - } - } else { - *success = TRUE; - retry = FALSE; - if (srv_use_native_aio && ((attributes & FILE_FLAG_OVERLAPPED) != 0)) { - ut_a(CreateIoCompletionPort(file, completion_port, 0, 0)); - } - } - - } while (retry); - - /* If we have proper file handle and atomic writes should be used, - try to set atomic writes and if that fails when creating a new - table, produce a error. If atomic writes are used on existing - file, ignore error and use traditional writes for that file */ - if (file != INVALID_HANDLE_VALUE - && (awrites == ATOMIC_WRITES_ON || - (srv_use_atomic_writes && awrites == ATOMIC_WRITES_DEFAULT)) - && !os_file_set_atomic_writes(name, file)) { - if (create_mode == OS_FILE_CREATE) { - fprintf(stderr, "InnoDB: Error: Can't create file using atomic writes\n"); - CloseHandle(file); - os_file_delete_if_exists_func(name); - *success = FALSE; - file = INVALID_HANDLE_VALUE; - } - } - -#else /* __WIN__ */ - int create_flag; - const char* mode_str = NULL; - if (create_mode != OS_FILE_OPEN && create_mode != OS_FILE_OPEN_RAW) - WAIT_ALLOW_WRITES(); - - on_error_no_exit = create_mode & OS_FILE_ON_ERROR_NO_EXIT - ? TRUE : FALSE; - on_error_silent = create_mode & OS_FILE_ON_ERROR_SILENT - ? TRUE : FALSE; - - create_mode &= ~OS_FILE_ON_ERROR_NO_EXIT; - create_mode &= ~OS_FILE_ON_ERROR_SILENT; - - if (create_mode == OS_FILE_OPEN - || create_mode == OS_FILE_OPEN_RAW - || create_mode == OS_FILE_OPEN_RETRY) { - - mode_str = "OPEN"; - - create_flag = srv_read_only_mode ? O_RDONLY : O_RDWR; - - } else if (srv_read_only_mode) { - - mode_str = "OPEN"; - - create_flag = O_RDONLY; - - } else if (create_mode == OS_FILE_CREATE) { - - mode_str = "CREATE"; - create_flag = O_RDWR | O_CREAT | O_EXCL; - - } else if (create_mode == OS_FILE_OVERWRITE) { - - mode_str = "OVERWRITE"; - create_flag = O_RDWR | O_CREAT | O_TRUNC; - - } else { - ib_logf(IB_LOG_LEVEL_ERROR, - "Unknown file create mode (%lu) for file '%s'", - create_mode, name); - - return((os_file_t) -1); - } - - ut_a(type == OS_LOG_FILE || type == OS_DATA_FILE); - ut_a(purpose == OS_FILE_AIO || purpose == OS_FILE_NORMAL); - -#ifdef O_SYNC - /* We let O_SYNC only affect log files; note that we map O_DSYNC to - O_SYNC because the datasync options seemed to corrupt files in 2001 - in both Linux and Solaris */ - - if (!srv_read_only_mode - && type == OS_LOG_FILE - && srv_unix_file_flush_method == SRV_UNIX_O_DSYNC) { - - create_flag |= O_SYNC; - } -#endif /* O_SYNC */ - - do { - file = ::open(name, create_flag, os_innodb_umask); - - if (file == -1) { - const char* operation; - - operation = (create_mode == OS_FILE_CREATE - && !srv_read_only_mode) - ? "create" : "open"; - - *success = FALSE; - - if (on_error_no_exit) { - retry = os_file_handle_error_no_exit( - name, operation, on_error_silent, __FILE__, __LINE__); - } else { - retry = os_file_handle_error(name, operation, __FILE__, __LINE__); - } - } else { - *success = TRUE; - retry = false; - } - - } while (retry); - - if (!srv_read_only_mode - && *success - && type != OS_LOG_FILE - && (srv_unix_file_flush_method == SRV_UNIX_O_DIRECT - || srv_unix_file_flush_method == SRV_UNIX_O_DIRECT_NO_FSYNC)) { - - os_file_set_nocache(file, name, mode_str); - } else if (!srv_read_only_mode - && *success - && srv_unix_file_flush_method == SRV_UNIX_ALL_O_DIRECT) { - os_file_set_nocache(file, name, mode_str); - } - -#ifdef USE_FILE_LOCK - if (!srv_read_only_mode - && *success - && create_mode != OS_FILE_OPEN_RAW - && os_file_lock(file, name)) { - - if (create_mode == OS_FILE_OPEN_RETRY) { - - ut_a(!srv_read_only_mode); - - ib_logf(IB_LOG_LEVEL_INFO, - "Retrying to lock the first data file"); - - for (int i = 0; i < 100; i++) { - os_thread_sleep(1000000); - - if (!os_file_lock(file, name)) { - *success = TRUE; - return(file); - } - } - - ib_logf(IB_LOG_LEVEL_INFO, - "Unable to open the first data file"); - } - - *success = FALSE; - close(file); - file = -1; - } -#endif /* USE_FILE_LOCK */ - - /* If we have proper file handle and atomic writes should be used, - try to set atomic writes and if that fails when creating a new - table, produce a error. If atomic writes are used on existing - file, ignore error and use traditional writes for that file */ - if (file != -1 - && (awrites == ATOMIC_WRITES_ON || - (srv_use_atomic_writes && awrites == ATOMIC_WRITES_DEFAULT)) - && !os_file_set_atomic_writes(name, file)) { - if (create_mode == OS_FILE_CREATE) { - fprintf(stderr, "InnoDB: Error: Can't create file using atomic writes\n"); - close(file); - os_file_delete_if_exists_func(name); - *success = FALSE; - file = -1; - } - } - - -#endif /* __WIN__ */ - - return(file); -} - -/***********************************************************************//** -Deletes a file if it exists. The file has to be closed before calling this. -@return TRUE if success */ -UNIV_INTERN -bool -os_file_delete_if_exists_func( -/*==========================*/ - const char* name) /*!< in: file path as a null-terminated - string */ -{ -#ifdef __WIN__ - bool ret; - ulint count = 0; -loop: - /* In Windows, deleting an .ibd file may fail if ibbackup is copying - it */ - - ret = DeleteFile((LPCTSTR) name); - - if (ret) { - return(true); - } - - DWORD lasterr = GetLastError(); - if (lasterr == ERROR_FILE_NOT_FOUND - || lasterr == ERROR_PATH_NOT_FOUND) { - /* the file does not exist, this not an error */ - - return(true); - } - - count++; - - if (count > 100 && 0 == (count % 10)) { - os_file_get_last_error(true); /* print error information */ - - ib_logf(IB_LOG_LEVEL_WARN, "Delete of file %s failed.", name); - } - - os_thread_sleep(1000000); /* sleep for a second */ - - if (count > 2000) { - - return(false); - } - - goto loop; -#else - int ret; - WAIT_ALLOW_WRITES(); - - ret = unlink(name); - - if (ret != 0 && errno != ENOENT) { - os_file_handle_error_no_exit(name, "delete", FALSE, __FILE__, __LINE__); - - return(false); - } - - return(true); -#endif /* __WIN__ */ -} - -/***********************************************************************//** -Deletes a file. The file has to be closed before calling this. -@return TRUE if success */ -UNIV_INTERN -bool -os_file_delete_func( -/*================*/ - const char* name) /*!< in: file path as a null-terminated - string */ -{ -#ifdef __WIN__ - BOOL ret; - ulint count = 0; -loop: - /* In Windows, deleting an .ibd file may fail if ibbackup is copying - it */ - - ret = DeleteFile((LPCTSTR) name); - - if (ret) { - return(true); - } - - if (GetLastError() == ERROR_FILE_NOT_FOUND) { - /* If the file does not exist, we classify this as a 'mild' - error and return */ - - return(false); - } - - count++; - - if (count > 100 && 0 == (count % 10)) { - os_file_get_last_error(true); /* print error information */ - - fprintf(stderr, - "InnoDB: Warning: cannot delete file %s\n" - "InnoDB: Are you running ibbackup" - " to back up the file?\n", name); - } - - os_thread_sleep(1000000); /* sleep for a second */ - - if (count > 2000) { - - return(false); - } - - goto loop; -#else - int ret; - WAIT_ALLOW_WRITES(); - - ret = unlink(name); - - if (ret != 0) { - os_file_handle_error_no_exit(name, "delete", FALSE, __FILE__, __LINE__); - - return(false); - } - - return(true); -#endif -} - -/***********************************************************************//** -NOTE! Use the corresponding macro os_file_rename(), not directly this function! -Renames a file (can also move it to another directory). It is safest that the -file is closed before calling this function. -@return TRUE if success */ -UNIV_INTERN -ibool -os_file_rename_func( -/*================*/ - const char* oldpath,/*!< in: old file path as a null-terminated - string */ - const char* newpath)/*!< in: new file path */ -{ -#ifdef UNIV_DEBUG - os_file_type_t type; - ibool exists; - - /* New path must not exist. */ - ut_ad(os_file_status(newpath, &exists, &type)); - ut_ad(!exists); - - /* Old path must exist. */ - ut_ad(os_file_status(oldpath, &exists, &type)); - ut_ad(exists); -#endif /* UNIV_DEBUG */ - -#ifdef __WIN__ - BOOL ret; - - ret = MoveFileEx((LPCTSTR)oldpath, (LPCTSTR)newpath, MOVEFILE_REPLACE_EXISTING); - - if (ret) { - return(TRUE); - } - - os_file_handle_error_no_exit(oldpath, "rename", FALSE, __FILE__, __LINE__); - - return(FALSE); -#else - int ret; - WAIT_ALLOW_WRITES(); - - ret = rename(oldpath, newpath); - - if (ret != 0) { - os_file_handle_error_no_exit(oldpath, "rename", FALSE, __FILE__, __LINE__); - - return(FALSE); - } - - return(TRUE); -#endif /* __WIN__ */ -} - -/***********************************************************************//** -NOTE! Use the corresponding macro os_file_close(), not directly this function! -Closes a file handle. In case of error, error number can be retrieved with -os_file_get_last_error. -@return TRUE if success */ -UNIV_INTERN -ibool -os_file_close_func( -/*===============*/ - os_file_t file) /*!< in, own: handle to a file */ -{ -#ifdef __WIN__ - BOOL ret; - - ut_a(file); - - ret = CloseHandle(file); - - if (ret) { - return(TRUE); - } - - os_file_handle_error(NULL, "close", __FILE__, __LINE__); - - return(FALSE); -#else - int ret; - - ret = close(file); - - if (ret == -1) { - os_file_handle_error(NULL, "close", __FILE__, __LINE__); - - return(FALSE); - } - - return(TRUE); -#endif /* __WIN__ */ -} - -/***********************************************************************//** -Closes a file handle. -@return TRUE if success */ -UNIV_INTERN -ibool -os_file_close_no_error_handling( -/*============================*/ - os_file_t file) /*!< in, own: handle to a file */ -{ -#ifdef __WIN__ - BOOL ret; - - ut_a(file); - - ret = CloseHandle(file); - - if (ret) { - return(TRUE); - } - - return(FALSE); -#else - int ret; - - ret = close(file); - - if (ret == -1) { - - return(FALSE); - } - - return(TRUE); -#endif /* __WIN__ */ -} - -/***********************************************************************//** -Gets a file size. -@return file size, or (os_offset_t) -1 on failure */ -UNIV_INTERN -os_offset_t -os_file_get_size( -/*=============*/ - os_file_t file) /*!< in: handle to a file */ -{ -#ifdef __WIN__ - os_offset_t offset; - DWORD high; - DWORD low; - - low = GetFileSize(file, &high); - - if ((low == 0xFFFFFFFF) && (GetLastError() != NO_ERROR)) { - return((os_offset_t) -1); - } - - offset = (os_offset_t) low | ((os_offset_t) high << 32); - - return(offset); -#else - return((os_offset_t) lseek(file, 0, SEEK_END)); -#endif /* __WIN__ */ -} - -/***********************************************************************//** -Write the specified number of zeros to a newly created file. -@return TRUE if success */ -UNIV_INTERN -ibool -os_file_set_size( -/*=============*/ - const char* name, /*!< in: name of the file or path as a - null-terminated string */ - os_file_t file, /*!< in: handle to a file */ - os_offset_t size) /*!< in: file size */ -{ - os_offset_t current_size; - ibool ret; - byte* buf; - byte* buf2; - ulint buf_size; - - current_size = 0; - -#ifdef HAVE_POSIX_FALLOCATE - if (srv_use_posix_fallocate) { - - if (posix_fallocate(file, current_size, size) == -1) { - - ib_logf(IB_LOG_LEVEL_ERROR, "preallocating file " - "space for file \'%s\' failed. Current size " - INT64PF ", desired size " INT64PF "\n", - name, current_size, size); - os_file_handle_error_no_exit (name, "posix_fallocate", - FALSE, __FILE__, __LINE__); - return(FALSE); - } - return(TRUE); - } -#endif - - /* Write up to 1 megabyte at a time. */ - buf_size = ut_min(64, (ulint) (size / UNIV_PAGE_SIZE)) - * UNIV_PAGE_SIZE; - buf2 = static_cast(ut_malloc(buf_size + UNIV_PAGE_SIZE)); - - /* Align the buffer for possible raw i/o */ - buf = static_cast(ut_align(buf2, UNIV_PAGE_SIZE)); - - /* Write buffer full of zeros */ - memset(buf, 0, buf_size); - - if (size >= (os_offset_t) 100 << 20) { - - fprintf(stderr, "InnoDB: Progress in MB:"); - } - - while (current_size < size) { - ulint n_bytes; - - if (size - current_size < (os_offset_t) buf_size) { - n_bytes = (ulint) (size - current_size); - } else { - n_bytes = buf_size; - } - - ret = os_file_write(name, file, buf, current_size, n_bytes); - - if (!ret) { - ut_free(buf2); - goto error_handling; - } - - /* Print about progress for each 100 MB written */ - if ((current_size + n_bytes) / (100 << 20) - != current_size / (100 << 20)) { - - fprintf(stderr, " %lu00", - (ulong) ((current_size + n_bytes) - / (100 << 20))); - } - - current_size += n_bytes; - } - - if (size >= (os_offset_t) 100 << 20) { - - fprintf(stderr, "\n"); - } - - ut_free(buf2); - - ret = os_file_flush(file); - - if (ret) { - return(TRUE); - } - -error_handling: - return(FALSE); -} - -/***********************************************************************//** -Truncates a file at its current position. -@return TRUE if success */ -UNIV_INTERN -ibool -os_file_set_eof( -/*============*/ - FILE* file) /*!< in: file to be truncated */ -{ -#ifdef __WIN__ - HANDLE h = (HANDLE) _get_osfhandle(fileno(file)); - return(SetEndOfFile(h)); -#else /* __WIN__ */ - WAIT_ALLOW_WRITES(); - return(!ftruncate(fileno(file), ftell(file))); -#endif /* __WIN__ */ -} - -/***********************************************************************//** -Truncates a file at the specified position. -@return TRUE if success */ -UNIV_INTERN -ibool -os_file_set_eof_at( - os_file_t file, /*!< in: handle to a file */ - ib_uint64_t new_len)/*!< in: new file length */ -{ -#ifdef __WIN__ - LARGE_INTEGER li, li2; - li.QuadPart = new_len; - return(SetFilePointerEx(file, li, &li2,FILE_BEGIN) - && SetEndOfFile(file)); -#else - WAIT_ALLOW_WRITES(); - /* TODO: works only with -D_FILE_OFFSET_BITS=64 ? */ - return(!ftruncate(file, new_len)); -#endif -} - - -#ifndef __WIN__ -/***********************************************************************//** -Wrapper to fsync(2) that retries the call on some errors. -Returns the value 0 if successful; otherwise the value -1 is returned and -the global variable errno is set to indicate the error. -@return 0 if success, -1 otherwise */ - -static -int -os_file_fsync( -/*==========*/ - os_file_t file) /*!< in: handle to a file */ -{ - int ret; - int failures; - ibool retry; - - failures = 0; - - do { - ret = fsync(file); - - os_n_fsyncs++; - - if (ret == -1 && errno == ENOLCK) { - - if (failures % 100 == 0) { - - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: fsync(): " - "No locks available; retrying\n"); - } - - os_thread_sleep(200000 /* 0.2 sec */); - - failures++; - - retry = TRUE; - } else if (ret == -1 && errno == EINTR) { - /* Handle signal interruptions correctly */ - retry = TRUE; - } else { - - retry = FALSE; - } - } while (retry); - - return(ret); -} -#endif /* !__WIN__ */ - -/***********************************************************************//** -NOTE! Use the corresponding macro os_file_flush(), not directly this function! -Flushes the write buffers of a given file to the disk. -@return TRUE if success */ -UNIV_INTERN -ibool -os_file_flush_func( -/*===============*/ - os_file_t file) /*!< in, own: handle to a file */ -{ -#ifdef __WIN__ - BOOL ret; - - ut_a(file); - - os_n_fsyncs++; - - ret = FlushFileBuffers(file); - - if (ret) { - return(TRUE); - } - - /* Since Windows returns ERROR_INVALID_FUNCTION if the 'file' is - actually a raw device, we choose to ignore that error if we are using - raw disks */ - - if (srv_start_raw_disk_in_use && GetLastError() - == ERROR_INVALID_FUNCTION) { - return(TRUE); - } - - os_file_handle_error(NULL, "flush", __FILE__, __LINE__); - - /* It is a fatal error if a file flush does not succeed, because then - the database can get corrupt on disk */ - ut_error; - - return(FALSE); -#else - int ret; - WAIT_ALLOW_WRITES(); - -#if defined(HAVE_DARWIN_THREADS) -# ifndef F_FULLFSYNC - /* The following definition is from the Mac OS X 10.3 */ -# define F_FULLFSYNC 51 /* fsync + ask the drive to flush to the media */ -# elif F_FULLFSYNC != 51 -# error "F_FULLFSYNC != 51: ABI incompatibility with Mac OS X 10.3" -# endif - /* Apple has disabled fsync() for internal disk drives in OS X. That - caused corruption for a user when he tested a power outage. Let us in - OS X use a nonstandard flush method recommended by an Apple - engineer. */ - - if (!srv_have_fullfsync) { - /* If we are not on an operating system that supports this, - then fall back to a plain fsync. */ - - ret = os_file_fsync(file); - } else { - ret = fcntl(file, F_FULLFSYNC, NULL); - - if (ret) { - /* If we are not on a file system that supports this, - then fall back to a plain fsync. */ - ret = os_file_fsync(file); - } - } -#else - ret = os_file_fsync(file); -#endif - - if (ret == 0) { - return(TRUE); - } - - /* Since Linux returns EINVAL if the 'file' is actually a raw device, - we choose to ignore that error if we are using raw disks */ - - if (srv_start_raw_disk_in_use && errno == EINVAL) { - - return(TRUE); - } - - ib_logf(IB_LOG_LEVEL_ERROR, "The OS said file flush did not succeed"); - - os_file_handle_error(NULL, "flush", __FILE__, __LINE__); - - /* It is a fatal error if a file flush does not succeed, because then - the database can get corrupt on disk */ - ut_error; - - return(FALSE); -#endif -} - -#ifndef __WIN__ -/*******************************************************************//** -Does a synchronous read operation in Posix. -@return number of bytes read, -1 if error */ -static __attribute__((nonnull(2), warn_unused_result)) -ssize_t -os_file_pread( -/*==========*/ - os_file_t file, /*!< in: handle to a file */ - void* buf, /*!< in: buffer where to read */ - ulint n, /*!< in: number of bytes to read */ - os_offset_t offset, /*!< in: file offset from where to read */ - trx_t* trx) -{ - off_t offs; -#if defined(HAVE_PREAD) && !defined(HAVE_BROKEN_PREAD) - ssize_t n_bytes; - ssize_t n_read; -#endif /* HAVE_PREAD && !HAVE_BROKEN_PREAD */ - ulint sec; - ulint ms; - ib_uint64_t start_time; - ib_uint64_t finish_time; - - ut_ad(n); - - /* If off_t is > 4 bytes in size, then we assume we can pass a - 64-bit address */ - offs = (off_t) offset; - - if (sizeof(off_t) <= 4) { - if (offset != (os_offset_t) offs) { - ib_logf(IB_LOG_LEVEL_ERROR, - "File read at offset > 4 GB"); - } - } - - os_n_file_reads++; - - if (UNIV_UNLIKELY(trx && trx->take_stats)) - { - trx->io_reads++; - trx->io_read += n; - ut_usectime(&sec, &ms); - start_time = (ib_uint64_t)sec * 1000000 + ms; - } else { - start_time = 0; - } -#if defined(HAVE_PREAD) && !defined(HAVE_BROKEN_PREAD) -#if defined(HAVE_ATOMIC_BUILTINS) && UNIV_WORD_SIZE == 8 - (void) os_atomic_increment_ulint(&os_n_pending_reads, 1); - (void) os_atomic_increment_ulint(&os_file_n_pending_preads, 1); - MONITOR_ATOMIC_INC(MONITOR_OS_PENDING_READS); -#else - os_mutex_enter(os_file_count_mutex); - os_file_n_pending_preads++; - os_n_pending_reads++; - MONITOR_INC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); -#endif /* HAVE_ATOMIC_BUILTINS && UNIV_WORD == 8 */ - - /* Handle partial reads and signal interruptions correctly */ - for (n_bytes = 0; n_bytes < (ssize_t) n; ) { - n_read = pread(file, buf, (ssize_t)n - n_bytes, offs); - if (n_read > 0) { - n_bytes += n_read; - offs += n_read; - buf = (char *)buf + n_read; - } else if (n_read == -1 && errno == EINTR) { - continue; - } else { - break; - } - } - -#if defined(HAVE_ATOMIC_BUILTINS) && UNIV_WORD_SIZE == 8 - (void) os_atomic_decrement_ulint(&os_n_pending_reads, 1); - (void) os_atomic_decrement_ulint(&os_file_n_pending_preads, 1); - MONITOR_ATOMIC_DEC(MONITOR_OS_PENDING_READS); -#else - os_mutex_enter(os_file_count_mutex); - os_file_n_pending_preads--; - os_n_pending_reads--; - MONITOR_DEC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); -#endif /* !HAVE_ATOMIC_BUILTINS || UNIV_WORD == 8 */ - - if (UNIV_UNLIKELY(start_time != 0)) - { - ut_usectime(&sec, &ms); - finish_time = (ib_uint64_t)sec * 1000000 + ms; - trx->io_reads_wait_timer += (ulint)(finish_time - start_time); - } - - return(n_bytes); -#else - { - off_t ret_offset; - ssize_t ret; - ssize_t n_read; -#ifndef UNIV_HOTBACKUP - ulint i; -#endif /* !UNIV_HOTBACKUP */ - -#if defined(HAVE_ATOMIC_BUILTINS) && UNIV_WORD_SIZE == 8 - (void) os_atomic_increment_ulint(&os_n_pending_reads, 1); - MONITOR_ATOMIC_INC(MONITOR_OS_PENDING_READS); -#else - os_mutex_enter(os_file_count_mutex); - os_n_pending_reads++; - MONITOR_INC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); -#endif /* HAVE_ATOMIC_BUILTINS && UNIV_WORD == 8 */ -#ifndef UNIV_HOTBACKUP - /* Protect the seek / read operation with a mutex */ - i = ((ulint) file) % OS_FILE_N_SEEK_MUTEXES; - - os_mutex_enter(os_file_seek_mutexes[i]); -#endif /* !UNIV_HOTBACKUP */ - - ret_offset = lseek(file, offs, SEEK_SET); - - if (ret_offset < 0) { - ret = -1; - } else { - /* Handle signal interruptions correctly */ - for (ret = 0; ret < (ssize_t) n; ) { - n_read = read(file, buf, (ssize_t)n); - if (n_read > 0) { - ret += n_read; - } else if (n_read == -1 && errno == EINTR) { - continue; - } else { - break; - } - } - } - -#ifndef UNIV_HOTBACKUP - os_mutex_exit(os_file_seek_mutexes[i]); -#endif /* !UNIV_HOTBACKUP */ - -#if defined(HAVE_ATOMIC_BUILTINS) && UNIV_WORD_SIZE == 8 - (void) os_atomic_decrement_ulint(&os_n_pending_reads, 1); - MONITOR_ATOIC_DEC(MONITOR_OS_PENDING_READS); -#else - os_mutex_enter(os_file_count_mutex); - os_n_pending_reads--; - MONITOR_DEC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); -#endif /* HAVE_ATOMIC_BUILTINS && UNIV_WORD_SIZE == 8 */ - - if (UNIV_UNLIKELY(start_time != 0) - { - ut_usectime(&sec, &ms); - finish_time = (ib_uint64_t)sec * 1000000 + ms; - trx->io_reads_wait_timer += (ulint)(finish_time - start_time); - } - - return(ret); - } -#endif -} - -/*******************************************************************//** -Does a synchronous write operation in Posix. -@return number of bytes written, -1 if error */ -static __attribute__((nonnull, warn_unused_result)) -ssize_t -os_file_pwrite( -/*===========*/ - os_file_t file, /*!< in: handle to a file */ - const void* buf, /*!< in: buffer from where to write */ - ulint n, /*!< in: number of bytes to write */ - os_offset_t offset) /*!< in: file offset where to write */ -{ - ssize_t ret; - ssize_t n_written; - off_t offs; - - ut_ad(n); - ut_ad(!srv_read_only_mode); - - /* If off_t is > 4 bytes in size, then we assume we can pass a - 64-bit address */ - offs = (off_t) offset; - - if (sizeof(off_t) <= 4) { - if (offset != (os_offset_t) offs) { - ib_logf(IB_LOG_LEVEL_ERROR, - "File write at offset > 4 GB."); - } - } - - os_n_file_writes++; - -#if defined(HAVE_PWRITE) && !defined(HAVE_BROKEN_PREAD) -#if !defined(HAVE_ATOMIC_BUILTINS) || UNIV_WORD_SIZE < 8 - os_mutex_enter(os_file_count_mutex); - os_file_n_pending_pwrites++; - os_n_pending_writes++; - MONITOR_INC(MONITOR_OS_PENDING_WRITES); - os_mutex_exit(os_file_count_mutex); -#else - (void) os_atomic_increment_ulint(&os_n_pending_writes, 1); - (void) os_atomic_increment_ulint(&os_file_n_pending_pwrites, 1); - MONITOR_ATOMIC_INC(MONITOR_OS_PENDING_WRITES); -#endif /* !HAVE_ATOMIC_BUILTINS || UNIV_WORD < 8 */ - - /* Handle partial writes and signal interruptions correctly */ - for (ret = 0; ret < (ssize_t) n; ) { - n_written = pwrite(file, buf, (ssize_t)n - ret, offs); - if (n_written >= 0) { - ret += n_written; - offs += n_written; - buf = (char *)buf + n_written; - } else if (n_written == -1 && errno == EINTR) { - continue; - } else { - break; - } - } - -#if !defined(HAVE_ATOMIC_BUILTINS) || UNIV_WORD_SIZE < 8 - os_mutex_enter(os_file_count_mutex); - os_file_n_pending_pwrites--; - os_n_pending_writes--; - MONITOR_DEC(MONITOR_OS_PENDING_WRITES); - os_mutex_exit(os_file_count_mutex); -#else - (void) os_atomic_decrement_ulint(&os_n_pending_writes, 1); - (void) os_atomic_decrement_ulint(&os_file_n_pending_pwrites, 1); - MONITOR_ATOMIC_DEC(MONITOR_OS_PENDING_WRITES); -#endif /* !HAVE_ATOMIC_BUILTINS || UNIV_WORD < 8 */ - - return(ret); -#else - { - off_t ret_offset; -# ifndef UNIV_HOTBACKUP - ulint i; -# endif /* !UNIV_HOTBACKUP */ - - os_mutex_enter(os_file_count_mutex); - os_n_pending_writes++; - MONITOR_INC(MONITOR_OS_PENDING_WRITES); - os_mutex_exit(os_file_count_mutex); - -# ifndef UNIV_HOTBACKUP - /* Protect the seek / write operation with a mutex */ - i = ((ulint) file) % OS_FILE_N_SEEK_MUTEXES; - - os_mutex_enter(os_file_seek_mutexes[i]); -# endif /* UNIV_HOTBACKUP */ - - ret_offset = lseek(file, offs, SEEK_SET); - - if (ret_offset < 0) { - ret = -1; - - goto func_exit; - } - - /* Handle signal interruptions correctly */ - for (ret = 0; ret < (ssize_t) n; ) { - n_written = write(file, buf, (ssize_t)n); - if (n_written > 0) { - ret += n_written; - } else if (n_written == -1 && errno == EINTR) { - continue; - } else { - break; - } - } - -func_exit: -# ifndef UNIV_HOTBACKUP - os_mutex_exit(os_file_seek_mutexes[i]); -# endif /* !UNIV_HOTBACKUP */ - - os_mutex_enter(os_file_count_mutex); - os_n_pending_writes--; - MONITOR_DEC(MONITOR_OS_PENDING_WRITES); - os_mutex_exit(os_file_count_mutex); - - return(ret); - } -#endif /* !UNIV_HOTBACKUP */ -} -#endif - -/*******************************************************************//** -NOTE! Use the corresponding macro os_file_read(), not directly this -function! -Requests a synchronous positioned read operation. -@return TRUE if request was successful, FALSE if fail */ -UNIV_INTERN -ibool -os_file_read_func( -/*==============*/ - os_file_t file, /*!< in: handle to a file */ - void* buf, /*!< in: buffer where to read */ - os_offset_t offset, /*!< in: file offset where to read */ - ulint n, /*!< in: number of bytes to read */ - trx_t* trx, - ibool compressed) /*!< in: is this file space - compressed ? */ -{ -#ifdef __WIN__ - BOOL ret; - DWORD len; - ibool retry; - OVERLAPPED overlapped; - - - /* On 64-bit Windows, ulint is 64 bits. But offset and n should be - no more than 32 bits. */ - ut_a((n & 0xFFFFFFFFUL) == n); - - os_n_file_reads++; - os_bytes_read_since_printout += n; - -try_again: - ut_ad(file); - ut_ad(buf); - ut_ad(n > 0); - - os_mutex_enter(os_file_count_mutex); - os_n_pending_reads++; - MONITOR_INC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); - - memset (&overlapped, 0, sizeof (overlapped)); - overlapped.Offset = (DWORD)(offset & 0xFFFFFFFF); - overlapped.OffsetHigh = (DWORD)(offset >> 32); - overlapped.hEvent = win_get_syncio_event(); - ret = ReadFile(file, buf, n, NULL, &overlapped); - if (ret) { - ret = GetOverlappedResult(file, &overlapped, (DWORD *)&len, FALSE); - } - else if(GetLastError() == ERROR_IO_PENDING) { - ret = GetOverlappedResult(file, &overlapped, (DWORD *)&len, TRUE); - } - os_mutex_enter(os_file_count_mutex); - os_n_pending_reads--; - MONITOR_DEC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); - - if (ret && len == n) { - /* Note that InnoDB writes files that are not formated - as file spaces and they do not have FIL_PAGE_TYPE - field, thus we must use here information is the actual - file space compressed. */ - if (compressed && fil_page_is_compressed((byte *)buf)) { - fil_decompress_page(NULL, (byte *)buf, len, NULL); - } - - return(TRUE); - } -#else /* __WIN__ */ - ibool retry; - ssize_t ret; - - os_bytes_read_since_printout += n; - -try_again: - ret = os_file_pread(file, buf, n, offset, trx); - - if ((ulint) ret == n) { - if (fil_page_is_encrypted((byte *)buf)) { - fil_decrypt_page(NULL, (byte *)buf, n, NULL); - } - /* Note that InnoDB writes files that are not formated - as file spaces and they do not have FIL_PAGE_TYPE - field, thus we must use here information is the actual - file space compressed. */ - if (compressed && fil_page_is_compressed((byte *)buf)) { - fil_decompress_page(NULL, (byte *)buf, n, NULL); - } - - - return(TRUE); - } - - ib_logf(IB_LOG_LEVEL_ERROR, - "Tried to read "ULINTPF" bytes at offset " UINT64PF". " - "Was only able to read %ld.", n, offset, (lint) ret); -#endif /* __WIN__ */ - retry = os_file_handle_error(NULL, "read", __FILE__, __LINE__); - - if (retry) { - goto try_again; - } - - fprintf(stderr, - "InnoDB: Fatal error: cannot read from file." - " OS error number %lu.\n", -#ifdef __WIN__ - (ulong) GetLastError() -#else - (ulong) errno -#endif /* __WIN__ */ - ); - fflush(stderr); - - ut_error; - - return(FALSE); -} - -/*******************************************************************//** -NOTE! Use the corresponding macro os_file_read_no_error_handling(), -not directly this function! -Requests a synchronous positioned read operation. This function does not do -any error handling. In case of error it returns FALSE. -@return TRUE if request was successful, FALSE if fail */ -UNIV_INTERN -ibool -os_file_read_no_error_handling_func( -/*================================*/ - os_file_t file, /*!< in: handle to a file */ - void* buf, /*!< in: buffer where to read */ - os_offset_t offset, /*!< in: file offset where to read */ - ulint n, /*!< in: number of bytes to read */ - ibool compressed) /*!< in: is this file space - compressed ? */ -{ -#ifdef __WIN__ - BOOL ret; - DWORD len; - ibool retry; - OVERLAPPED overlapped; - overlapped.Offset = (DWORD)(offset & 0xFFFFFFFF); - overlapped.OffsetHigh = (DWORD)(offset >> 32); - - - /* On 64-bit Windows, ulint is 64 bits. But offset and n should be - no more than 32 bits. */ - ut_a((n & 0xFFFFFFFFUL) == n); - - os_n_file_reads++; - os_bytes_read_since_printout += n; - -try_again: - ut_ad(file); - ut_ad(buf); - ut_ad(n > 0); - - os_mutex_enter(os_file_count_mutex); - os_n_pending_reads++; - MONITOR_INC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); - - memset (&overlapped, 0, sizeof (overlapped)); - overlapped.Offset = (DWORD)(offset & 0xFFFFFFFF); - overlapped.OffsetHigh = (DWORD)(offset >> 32); - overlapped.hEvent = win_get_syncio_event(); - ret = ReadFile(file, buf, n, NULL, &overlapped); - if (ret) { - ret = GetOverlappedResult(file, &overlapped, (DWORD *)&len, FALSE); - } - else if(GetLastError() == ERROR_IO_PENDING) { - ret = GetOverlappedResult(file, &overlapped, (DWORD *)&len, TRUE); - } - os_mutex_enter(os_file_count_mutex); - os_n_pending_reads--; - MONITOR_DEC(MONITOR_OS_PENDING_READS); - os_mutex_exit(os_file_count_mutex); - - if (ret && len == n) { - - /* Note that InnoDB writes files that are not formated - as file spaces and they do not have FIL_PAGE_TYPE - field, thus we must use here information is the actual - file space compressed. */ - if (compressed && fil_page_is_compressed((byte *)buf)) { - fil_decompress_page(NULL, (byte *)buf, n, NULL); - } - - return(TRUE); - } -#else /* __WIN__ */ - ibool retry; - ssize_t ret; - - os_bytes_read_since_printout += n; - -try_again: - ret = os_file_pread(file, buf, n, offset, NULL); - - if ((ulint) ret == n) { - - /* Note that InnoDB writes files that are not formated - as file spaces and they do not have FIL_PAGE_TYPE - field, thus we must use here information is the actual - file space compressed. */ - if (compressed && fil_page_is_compressed((byte *)buf)) { - fil_decompress_page(NULL, (byte *)buf, n, NULL); - } - - if (fil_page_is_encrypted((byte *)buf)) { - fil_decrypt_page(NULL, (byte *)buf, n, NULL); - } - - - - return(TRUE); - } -#endif /* __WIN__ */ - retry = os_file_handle_error_no_exit(NULL, "read", FALSE, __FILE__, __LINE__); - - if (retry) { - goto try_again; - } - - return(FALSE); -} - -/*******************************************************************//** -Rewind file to its start, read at most size - 1 bytes from it to str, and -NUL-terminate str. All errors are silently ignored. This function is -mostly meant to be used with temporary files. */ -UNIV_INTERN -void -os_file_read_string( -/*================*/ - FILE* file, /*!< in: file to read from */ - char* str, /*!< in: buffer where to read */ - ulint size) /*!< in: size of buffer */ -{ - size_t flen; - - if (size == 0) { - return; - } - - rewind(file); - flen = fread(str, 1, size - 1, file); - str[flen] = '\0'; -} - -/*******************************************************************//** -NOTE! Use the corresponding macro os_file_write(), not directly -this function! -Requests a synchronous write operation. -@return TRUE if request was successful, FALSE if fail */ -UNIV_INTERN -ibool -os_file_write_func( -/*===============*/ - const char* name, /*!< in: name of the file or path as a - null-terminated string */ - os_file_t file, /*!< in: handle to a file */ - const void* buf, /*!< in: buffer from which to write */ - os_offset_t offset, /*!< in: file offset where to write */ - ulint n) /*!< in: number of bytes to write */ -{ - ut_ad(!srv_read_only_mode); - -#ifdef __WIN__ - BOOL ret; - DWORD len; - ulint n_retries = 0; - ulint err; - OVERLAPPED overlapped; - - /* On 64-bit Windows, ulint is 64 bits. But offset and n should be - no more than 32 bits. */ - ut_a((n & 0xFFFFFFFFUL) == n); - - os_n_file_writes++; - - ut_ad(file); - ut_ad(buf); - ut_ad(n > 0); - -retry: - - os_mutex_enter(os_file_count_mutex); - os_n_pending_writes++; - MONITOR_INC(MONITOR_OS_PENDING_WRITES); - os_mutex_exit(os_file_count_mutex); - - memset (&overlapped, 0, sizeof (overlapped)); - overlapped.Offset = (DWORD)(offset & 0xFFFFFFFF); - overlapped.OffsetHigh = (DWORD)(offset >> 32); - - overlapped.hEvent = win_get_syncio_event(); - ret = WriteFile(file, buf, n, NULL, &overlapped); - if (ret) { - ret = GetOverlappedResult(file, &overlapped, (DWORD *)&len, FALSE); - } - else if(GetLastError() == ERROR_IO_PENDING) { - ret = GetOverlappedResult(file, &overlapped, (DWORD *)&len, TRUE); - } - - os_mutex_enter(os_file_count_mutex); - os_n_pending_writes--; - MONITOR_DEC(MONITOR_OS_PENDING_WRITES); - os_mutex_exit(os_file_count_mutex); - - if (ret && len == n) { - - return(TRUE); - } - - /* If some background file system backup tool is running, then, at - least in Windows 2000, we may get here a specific error. Let us - retry the operation 100 times, with 1 second waits. */ - - if (GetLastError() == ERROR_LOCK_VIOLATION && n_retries < 100) { - - os_thread_sleep(1000000); - - n_retries++; - - goto retry; - } - - if (!os_has_said_disk_full) { - - err = (ulint) GetLastError(); - - ut_print_timestamp(stderr); - - fprintf(stderr, - " InnoDB: Error: Write to file %s failed" - " at offset %llu.\n" - "InnoDB: %lu bytes should have been written," - " only %lu were written.\n" - "InnoDB: Operating system error number %lu.\n" - "InnoDB: Check that your OS and file system" - " support files of this size.\n" - "InnoDB: Check also that the disk is not full" - " or a disk quota exceeded.\n", - name, offset, - (ulong) n, (ulong) len, (ulong) err); - - if (strerror((int) err) != NULL) { - fprintf(stderr, - "InnoDB: Error number %lu means '%s'.\n", - (ulong) err, strerror((int) err)); - } - - fprintf(stderr, - "InnoDB: Some operating system error numbers" - " are described at\n" - "InnoDB: " - REFMAN "operating-system-error-codes.html\n"); - - os_has_said_disk_full = TRUE; - } - - return(FALSE); -#else - ssize_t ret; - WAIT_ALLOW_WRITES(); - - ret = os_file_pwrite(file, buf, n, offset); - - if ((ulint) ret == n) { - - return(TRUE); - } - - if (!os_has_said_disk_full) { - - ut_print_timestamp(stderr); - - fprintf(stderr, - " InnoDB: Error: Write to file %s failed" - " at offset "UINT64PF".\n" - "InnoDB: %lu bytes should have been written," - " only %ld were written.\n" - "InnoDB: Operating system error number %lu.\n" - "InnoDB: Check that your OS and file system" - " support files of this size.\n" - "InnoDB: Check also that the disk is not full" - " or a disk quota exceeded.\n", - name, offset, n, (lint) ret, - (ulint) errno); - if (strerror(errno) != NULL) { - fprintf(stderr, - "InnoDB: Error number %d means '%s'.\n", - errno, strerror(errno)); - } - - fprintf(stderr, - "InnoDB: Some operating system error numbers" - " are described at\n" - "InnoDB: " - REFMAN "operating-system-error-codes.html\n"); - - os_has_said_disk_full = TRUE; - } - - return(FALSE); -#endif -} - -/*******************************************************************//** -Check the existence and type of the given file. -@return TRUE if call succeeded */ -UNIV_INTERN -ibool -os_file_status( -/*===========*/ - const char* path, /*!< in: pathname of the file */ - ibool* exists, /*!< out: TRUE if file exists */ - os_file_type_t* type) /*!< out: type of the file (if it exists) */ -{ -#ifdef __WIN__ - int ret; - struct _stat64 statinfo; - - ret = _stat64(path, &statinfo); - if (ret && (errno == ENOENT || errno == ENOTDIR)) { - /* file does not exist */ - *exists = FALSE; - return(TRUE); - } else if (ret) { - /* file exists, but stat call failed */ - - os_file_handle_error_no_exit(path, "stat", FALSE, __FILE__, __LINE__); - - return(FALSE); - } - - if (_S_IFDIR & statinfo.st_mode) { - *type = OS_FILE_TYPE_DIR; - } else if (_S_IFREG & statinfo.st_mode) { - *type = OS_FILE_TYPE_FILE; - } else { - *type = OS_FILE_TYPE_UNKNOWN; - } - - *exists = TRUE; - - return(TRUE); -#else - int ret; - struct stat statinfo; - - ret = stat(path, &statinfo); - if (ret && (errno == ENOENT || errno == ENOTDIR)) { - /* file does not exist */ - *exists = FALSE; - return(TRUE); - } else if (ret) { - /* file exists, but stat call failed */ - - os_file_handle_error_no_exit(path, "stat", FALSE, __FILE__, __LINE__); - - return(FALSE); - } - - if (S_ISDIR(statinfo.st_mode)) { - *type = OS_FILE_TYPE_DIR; - } else if (S_ISLNK(statinfo.st_mode)) { - *type = OS_FILE_TYPE_LINK; - } else if (S_ISREG(statinfo.st_mode)) { - *type = OS_FILE_TYPE_FILE; - } else { - *type = OS_FILE_TYPE_UNKNOWN; - } - - *exists = TRUE; - - return(TRUE); -#endif -} - -/*******************************************************************//** -This function returns information about the specified file -@return DB_SUCCESS if all OK */ -UNIV_INTERN -dberr_t -os_file_get_status( -/*===============*/ - const char* path, /*!< in: pathname of the file */ - os_file_stat_t* stat_info, /*!< information of a file in a - directory */ - bool check_rw_perm) /*!< in: for testing whether the - file can be opened in RW mode */ -{ - int ret; - -#ifdef __WIN__ - struct _stat64 statinfo; - - ret = _stat64(path, &statinfo); - - if (ret && (errno == ENOENT || errno == ENOTDIR)) { - /* file does not exist */ - - return(DB_NOT_FOUND); - - } else if (ret) { - /* file exists, but stat call failed */ - - os_file_handle_error_no_exit(path, "stat", FALSE, __FILE__, __LINE__); - - return(DB_FAIL); - - } else if (_S_IFDIR & statinfo.st_mode) { - stat_info->type = OS_FILE_TYPE_DIR; - } else if (_S_IFREG & statinfo.st_mode) { - - DWORD access = GENERIC_READ; - - if (!srv_read_only_mode) { - access |= GENERIC_WRITE; - } - - stat_info->type = OS_FILE_TYPE_FILE; - - /* Check if we can open it in read-only mode. */ - - if (check_rw_perm) { - HANDLE fh; - - fh = CreateFile( - (LPCTSTR) path, // File to open - access, - 0, // No sharing - NULL, // Default security - OPEN_EXISTING, // Existing file only - FILE_ATTRIBUTE_NORMAL, // Normal file - NULL); // No attr. template - - if (fh == INVALID_HANDLE_VALUE) { - stat_info->rw_perm = false; - } else { - stat_info->rw_perm = true; - CloseHandle(fh); - } - } - } else { - stat_info->type = OS_FILE_TYPE_UNKNOWN; - } -#else - struct stat statinfo; - - ret = stat(path, &statinfo); - - if (ret && (errno == ENOENT || errno == ENOTDIR)) { - /* file does not exist */ - - return(DB_NOT_FOUND); - - } else if (ret) { - /* file exists, but stat call failed */ - - os_file_handle_error_no_exit(path, "stat", FALSE, __FILE__, __LINE__); - - return(DB_FAIL); - - } - - switch (statinfo.st_mode & S_IFMT) { - case S_IFDIR: - stat_info->type = OS_FILE_TYPE_DIR; - break; - case S_IFLNK: - stat_info->type = OS_FILE_TYPE_LINK; - break; - case S_IFBLK: - stat_info->type = OS_FILE_TYPE_BLOCK; - break; - case S_IFREG: - stat_info->type = OS_FILE_TYPE_FILE; - break; - default: - stat_info->type = OS_FILE_TYPE_UNKNOWN; - } - - - if (check_rw_perm && (stat_info->type == OS_FILE_TYPE_FILE - || stat_info->type == OS_FILE_TYPE_BLOCK)) { - int fh; - int access; - - access = !srv_read_only_mode ? O_RDWR : O_RDONLY; - - fh = ::open(path, access, os_innodb_umask); - - if (fh == -1) { - stat_info->rw_perm = false; - } else { - stat_info->rw_perm = true; - close(fh); - } - } - -#endif /* _WIN_ */ - - stat_info->ctime = statinfo.st_ctime; - stat_info->atime = statinfo.st_atime; - stat_info->mtime = statinfo.st_mtime; - stat_info->size = statinfo.st_size; - - return(DB_SUCCESS); -} - -/* path name separator character */ -#ifdef __WIN__ -# define OS_FILE_PATH_SEPARATOR '\\' -#else -# define OS_FILE_PATH_SEPARATOR '/' -#endif - -/****************************************************************//** -This function returns a new path name after replacing the basename -in an old path with a new basename. The old_path is a full path -name including the extension. The tablename is in the normal -form "databasename/tablename". The new base name is found after -the forward slash. Both input strings are null terminated. - -This function allocates memory to be returned. It is the callers -responsibility to free the return value after it is no longer needed. - -@return own: new full pathname */ -UNIV_INTERN -char* -os_file_make_new_pathname( -/*======================*/ - const char* old_path, /*!< in: pathname */ - const char* tablename) /*!< in: contains new base name */ -{ - ulint dir_len; - char* last_slash; - char* base_name; - char* new_path; - ulint new_path_len; - - /* Split the tablename into its database and table name components. - They are separated by a '/'. */ - last_slash = strrchr((char*) tablename, '/'); - base_name = last_slash ? last_slash + 1 : (char*) tablename; - - /* Find the offset of the last slash. We will strip off the - old basename.ibd which starts after that slash. */ - last_slash = strrchr((char*) old_path, OS_FILE_PATH_SEPARATOR); - dir_len = last_slash ? last_slash - old_path : strlen(old_path); - - /* allocate a new path and move the old directory path to it. */ - new_path_len = dir_len + strlen(base_name) + sizeof "/.ibd"; - new_path = static_cast(mem_alloc(new_path_len)); - memcpy(new_path, old_path, dir_len); - - ut_snprintf(new_path + dir_len, - new_path_len - dir_len, - "%c%s.ibd", - OS_FILE_PATH_SEPARATOR, - base_name); - - return(new_path); -} - -/****************************************************************//** -This function returns a remote path name by combining a data directory -path provided in a DATA DIRECTORY clause with the tablename which is -in the form 'database/tablename'. It strips the file basename (which -is the tablename) found after the last directory in the path provided. -The full filepath created will include the database name as a directory -under the path provided. The filename is the tablename with the '.ibd' -extension. All input and output strings are null-terminated. - -This function allocates memory to be returned. It is the callers -responsibility to free the return value after it is no longer needed. - -@return own: A full pathname; data_dir_path/databasename/tablename.ibd */ -UNIV_INTERN -char* -os_file_make_remote_pathname( -/*=========================*/ - const char* data_dir_path, /*!< in: pathname */ - const char* tablename, /*!< in: tablename */ - const char* extention) /*!< in: file extention; ibd,cfg */ -{ - ulint data_dir_len; - char* last_slash; - char* new_path; - ulint new_path_len; - - ut_ad(extention && strlen(extention) == 3); - - /* Find the offset of the last slash. We will strip off the - old basename or tablename which starts after that slash. */ - last_slash = strrchr((char*) data_dir_path, OS_FILE_PATH_SEPARATOR); - data_dir_len = last_slash ? last_slash - data_dir_path : strlen(data_dir_path); - - /* allocate a new path and move the old directory path to it. */ - new_path_len = data_dir_len + strlen(tablename) - + sizeof "/." + strlen(extention); - new_path = static_cast(mem_alloc(new_path_len)); - memcpy(new_path, data_dir_path, data_dir_len); - ut_snprintf(new_path + data_dir_len, - new_path_len - data_dir_len, - "%c%s.%s", - OS_FILE_PATH_SEPARATOR, - tablename, - extention); - - srv_normalize_path_for_win(new_path); - - return(new_path); -} - -/****************************************************************//** -This function reduces a null-terminated full remote path name into -the path that is sent by MySQL for DATA DIRECTORY clause. It replaces -the 'databasename/tablename.ibd' found at the end of the path with just -'tablename'. - -Since the result is always smaller than the path sent in, no new memory -is allocated. The caller should allocate memory for the path sent in. -This function manipulates that path in place. - -If the path format is not as expected, just return. The result is used -to inform a SHOW CREATE TABLE command. */ -UNIV_INTERN -void -os_file_make_data_dir_path( -/*========================*/ - char* data_dir_path) /*!< in/out: full path/data_dir_path */ -{ - char* ptr; - char* tablename; - ulint tablename_len; - - /* Replace the period before the extension with a null byte. */ - ptr = strrchr((char*) data_dir_path, '.'); - if (!ptr) { - return; - } - ptr[0] = '\0'; - - /* The tablename starts after the last slash. */ - ptr = strrchr((char*) data_dir_path, OS_FILE_PATH_SEPARATOR); - if (!ptr) { - return; - } - ptr[0] = '\0'; - tablename = ptr + 1; - - /* The databasename starts after the next to last slash. */ - ptr = strrchr((char*) data_dir_path, OS_FILE_PATH_SEPARATOR); - if (!ptr) { - return; - } - tablename_len = ut_strlen(tablename); - - ut_memmove(++ptr, tablename, tablename_len); - - ptr[tablename_len] = '\0'; -} - -/****************************************************************//** -The function os_file_dirname returns a directory component of a -null-terminated pathname string. In the usual case, dirname returns -the string up to, but not including, the final '/', and basename -is the component following the final '/'. Trailing '/' characters -are not counted as part of the pathname. - -If path does not contain a slash, dirname returns the string ".". - -Concatenating the string returned by dirname, a "/", and the basename -yields a complete pathname. - -The return value is a copy of the directory component of the pathname. -The copy is allocated from heap. It is the caller responsibility -to free it after it is no longer needed. - -The following list of examples (taken from SUSv2) shows the strings -returned by dirname and basename for different paths: - - path dirname basename - "/usr/lib" "/usr" "lib" - "/usr/" "/" "usr" - "usr" "." "usr" - "/" "/" "/" - "." "." "." - ".." "." ".." - -@return own: directory component of the pathname */ -UNIV_INTERN -char* -os_file_dirname( -/*============*/ - const char* path) /*!< in: pathname */ -{ - /* Find the offset of the last slash */ - const char* last_slash = strrchr(path, OS_FILE_PATH_SEPARATOR); - if (!last_slash) { - /* No slash in the path, return "." */ - - return(mem_strdup(".")); - } - - /* Ok, there is a slash */ - - if (last_slash == path) { - /* last slash is the first char of the path */ - - return(mem_strdup("/")); - } - - /* Non-trivial directory component */ - - return(mem_strdupl(path, last_slash - path)); -} - -/****************************************************************//** -Creates all missing subdirectories along the given path. -@return TRUE if call succeeded FALSE otherwise */ -UNIV_INTERN -ibool -os_file_create_subdirs_if_needed( -/*=============================*/ - const char* path) /*!< in: path name */ -{ - if (srv_read_only_mode) { - - ib_logf(IB_LOG_LEVEL_ERROR, - "read only mode set. Can't create subdirectories '%s'", - path); - - return(FALSE); - - } - - char* subdir = os_file_dirname(path); - - if (strlen(subdir) == 1 - && (*subdir == OS_FILE_PATH_SEPARATOR || *subdir == '.')) { - /* subdir is root or cwd, nothing to do */ - mem_free(subdir); - - return(TRUE); - } - - /* Test if subdir exists */ - os_file_type_t type; - ibool subdir_exists; - ibool success = os_file_status(subdir, &subdir_exists, &type); - - if (success && !subdir_exists) { - - /* subdir does not exist, create it */ - success = os_file_create_subdirs_if_needed(subdir); - - if (!success) { - mem_free(subdir); - - return(FALSE); - } - - success = os_file_create_directory(subdir, FALSE); - } - - mem_free(subdir); - - return(success); -} - -#ifndef UNIV_HOTBACKUP -/****************************************************************//** -Returns a pointer to the nth slot in the aio array. -@return pointer to slot */ -static -os_aio_slot_t* -os_aio_array_get_nth_slot( -/*======================*/ - os_aio_array_t* array, /*!< in: aio array */ - ulint index) /*!< in: index of the slot */ -{ - ut_a(index < array->n_slots); - - return(&array->slots[index]); -} - -#if defined(LINUX_NATIVE_AIO) -/******************************************************************//** -Creates an io_context for native linux AIO. -@return TRUE on success. */ -static -ibool -os_aio_linux_create_io_ctx( -/*=======================*/ - ulint max_events, /*!< in: number of events. */ - io_context_t* io_ctx) /*!< out: io_ctx to initialize. */ -{ - int ret; - ulint retries = 0; - -retry: - memset(io_ctx, 0x0, sizeof(*io_ctx)); - - /* Initialize the io_ctx. Tell it how many pending - IO requests this context will handle. */ - - ret = io_setup(max_events, io_ctx); - if (ret == 0) { -#if defined(UNIV_AIO_DEBUG) - fprintf(stderr, - "InnoDB: Linux native AIO:" - " initialized io_ctx for segment\n"); -#endif - /* Success. Return now. */ - return(TRUE); - } - - /* If we hit EAGAIN we'll make a few attempts before failing. */ - - switch (ret) { - case -EAGAIN: - if (retries == 0) { - /* First time around. */ - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Warning: io_setup() failed" - " with EAGAIN. Will make %d attempts" - " before giving up.\n", - OS_AIO_IO_SETUP_RETRY_ATTEMPTS); - } - - if (retries < OS_AIO_IO_SETUP_RETRY_ATTEMPTS) { - ++retries; - fprintf(stderr, - "InnoDB: Warning: io_setup() attempt" - " %lu failed.\n", - retries); - os_thread_sleep(OS_AIO_IO_SETUP_RETRY_SLEEP); - goto retry; - } - - /* Have tried enough. Better call it a day. */ - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Error: io_setup() failed" - " with EAGAIN after %d attempts.\n", - OS_AIO_IO_SETUP_RETRY_ATTEMPTS); - break; - - case -ENOSYS: - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Error: Linux Native AIO interface" - " is not supported on this platform. Please" - " check your OS documentation and install" - " appropriate binary of InnoDB.\n"); - - break; - - default: - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: Error: Linux Native AIO setup" - " returned following error[%d]\n", -ret); - break; - } - - fprintf(stderr, - "InnoDB: You can disable Linux Native AIO by" - " setting innodb_use_native_aio = 0 in my.cnf\n"); - return(FALSE); -} - -/******************************************************************//** -Checks if the system supports native linux aio. On some kernel -versions where native aio is supported it won't work on tmpfs. In such -cases we can't use native aio as it is not possible to mix simulated -and native aio. -@return: TRUE if supported, FALSE otherwise. */ -static -ibool -os_aio_native_aio_supported(void) -/*=============================*/ -{ - int fd; - io_context_t io_ctx; - char name[1000]; - - if (!os_aio_linux_create_io_ctx(1, &io_ctx)) { - /* The platform does not support native aio. */ - return(FALSE); - } else if (!srv_read_only_mode) { - /* Now check if tmpdir supports native aio ops. */ - fd = innobase_mysql_tmpfile(); - - if (fd < 0) { - ib_logf(IB_LOG_LEVEL_WARN, - "Unable to create temp file to check " - "native AIO support."); - - return(FALSE); - } - } else { - - srv_normalize_path_for_win(srv_log_group_home_dir); - - ulint dirnamelen = strlen(srv_log_group_home_dir); - ut_a(dirnamelen < (sizeof name) - 10 - sizeof "ib_logfile"); - memcpy(name, srv_log_group_home_dir, dirnamelen); - - /* Add a path separator if needed. */ - if (dirnamelen && name[dirnamelen - 1] != SRV_PATH_SEPARATOR) { - name[dirnamelen++] = SRV_PATH_SEPARATOR; - } - - strcpy(name + dirnamelen, "ib_logfile0"); - - fd = ::open(name, O_RDONLY); - - if (fd == -1) { - - ib_logf(IB_LOG_LEVEL_WARN, - "Unable to open \"%s\" to check " - "native AIO read support.", name); - - return(FALSE); - } - } - - struct io_event io_event; - - memset(&io_event, 0x0, sizeof(io_event)); - - byte* buf = static_cast(ut_malloc(UNIV_PAGE_SIZE * 2)); - byte* ptr = static_cast(ut_align(buf, UNIV_PAGE_SIZE)); - - struct iocb iocb; - - /* Suppress valgrind warning. */ - memset(buf, 0x00, UNIV_PAGE_SIZE * 2); - memset(&iocb, 0x0, sizeof(iocb)); - - struct iocb* p_iocb = &iocb; - - if (!srv_read_only_mode) { - io_prep_pwrite(p_iocb, fd, ptr, UNIV_PAGE_SIZE, 0); - } else { - ut_a(UNIV_PAGE_SIZE >= 512); - io_prep_pread(p_iocb, fd, ptr, 512, 0); - } - - int err = io_submit(io_ctx, 1, &p_iocb); - - if (err >= 1) { - /* Now collect the submitted IO request. */ - err = io_getevents(io_ctx, 1, 1, &io_event, NULL); - } - - ut_free(buf); - close(fd); - - switch (err) { - case 1: - return(TRUE); - - case -EINVAL: - case -ENOSYS: - ib_logf(IB_LOG_LEVEL_ERROR, - "Linux Native AIO not supported. You can either " - "move %s to a file system that supports native " - "AIO or you can set innodb_use_native_aio to " - "FALSE to avoid this message.", - srv_read_only_mode ? name : "tmpdir"); - - /* fall through. */ - default: - ib_logf(IB_LOG_LEVEL_ERROR, - "Linux Native AIO check on %s returned error[%d]", - srv_read_only_mode ? name : "tmpdir", -err); - } - - return(FALSE); -} -#endif /* LINUX_NATIVE_AIO */ - -/******************************************************************//** -Creates an aio wait array. Note that we return NULL in case of failure. -We don't care about freeing memory here because we assume that a -failure will result in server refusing to start up. -@return own: aio array, NULL on failure */ -static -os_aio_array_t* -os_aio_array_create( -/*================*/ - ulint n, /*!< in: maximum number of pending aio - operations allowed; n must be - divisible by n_segments */ - ulint n_segments) /*!< in: number of segments in the aio array */ -{ - os_aio_array_t* array; -#ifdef LINUX_NATIVE_AIO - struct io_event* io_event = NULL; -#endif - ut_a(n > 0); - ut_a(n_segments > 0); - - array = static_cast(ut_malloc(sizeof(*array))); - memset(array, 0x0, sizeof(*array)); - - array->mutex = os_mutex_create(); - array->not_full = os_event_create(); - array->is_empty = os_event_create(); - - os_event_set(array->is_empty); - - array->n_slots = n; - array->n_segments = n_segments; - - array->slots = static_cast( - ut_malloc(n * sizeof(*array->slots))); - - memset(array->slots, 0x0, n * sizeof(*array->slots)); - -#if defined(LINUX_NATIVE_AIO) - array->aio_ctx = NULL; - array->aio_events = NULL; - - /* If we are not using native aio interface then skip this - part of initialization. */ - if (!srv_use_native_aio) { - goto skip_native_aio; - } - - /* Initialize the io_context array. One io_context - per segment in the array. */ - - array->aio_ctx = static_cast( - ut_malloc(n_segments * sizeof(*array->aio_ctx))); - - for (ulint i = 0; i < n_segments; ++i) { - if (!os_aio_linux_create_io_ctx(n/n_segments, - &array->aio_ctx[i])) { - /* If something bad happened during aio setup - we disable linux native aio. - The disadvantage will be a small memory leak - at shutdown but that's ok compared to a crash - or a not working server. - This frequently happens when running the test suite - with many threads on a system with low fs.aio-max-nr! - */ - - fprintf(stderr, - " InnoDB: Warning: Linux Native AIO disabled " - "because os_aio_linux_create_io_ctx() " - "failed. To get rid of this warning you can " - "try increasing system " - "fs.aio-max-nr to 1048576 or larger or " - "setting innodb_use_native_aio = 0 in my.cnf\n"); - srv_use_native_aio = FALSE; - goto skip_native_aio; - } - } - - /* Initialize the event array. One event per slot. */ - io_event = static_cast( - ut_malloc(n * sizeof(*io_event))); - - memset(io_event, 0x0, sizeof(*io_event) * n); - array->aio_events = io_event; - -skip_native_aio: -#endif /* LINUX_NATIVE_AIO */ - for (ulint i = 0; i < n; i++) { - os_aio_slot_t* slot; - - slot = os_aio_array_get_nth_slot(array, i); - slot->pos = i; - slot->reserved = FALSE; -#ifdef LINUX_NATIVE_AIO - memset(&slot->control, 0x0, sizeof(slot->control)); - slot->n_bytes = 0; - slot->ret = 0; -#endif /* WIN_ASYNC_IO */ - } - - return(array); -} - -/************************************************************************//** -Frees an aio wait array. */ -static -void -os_aio_array_free( -/*==============*/ - os_aio_array_t*& array) /*!< in, own: array to free */ -{ - ulint i; - - os_mutex_free(array->mutex); - os_event_free(array->not_full); - os_event_free(array->is_empty); - -#if defined(LINUX_NATIVE_AIO) - if (srv_use_native_aio) { - ut_free(array->aio_events); - ut_free(array->aio_ctx); - } -#endif /* LINUX_NATIVE_AIO */ - - for (i = 0; i < array->n_slots; i++) { - os_aio_slot_t* slot = os_aio_array_get_nth_slot(array, i); - if (slot->page_compression_page) { - ut_free(slot->page_compression_page); - slot->page_compression_page = NULL; - } - - if (slot->lzo_mem) { - ut_free(slot->lzo_mem); - slot->lzo_mem = NULL; - } - } - - for (i = 0; i < array->n_slots; i++) { - os_aio_slot_t* slot = os_aio_array_get_nth_slot(array, i); - if (slot->page_encryption_page) { - ut_free(slot->page_encryption_page); - slot->page_encryption_page = NULL; - } - } - - - ut_free(array->slots); - ut_free(array); - - array = 0; -} - -/*********************************************************************** -Initializes the asynchronous io system. Creates one array each for ibuf -and log i/o. Also creates one array each for read and write where each -array is divided logically into n_read_segs and n_write_segs -respectively. The caller must create an i/o handler thread for each -segment in these arrays. This function also creates the sync array. -No i/o handler thread needs to be created for that */ -UNIV_INTERN -ibool -os_aio_init( -/*========*/ - ulint n_per_seg, /*= 4); - } else { - ut_ad(n_segments > 0); - } - - os_aio_sync_array = os_aio_array_create(n_slots_sync, 1); - - if (os_aio_sync_array == NULL) { - return(FALSE); - } - - os_aio_n_segments = n_segments; - - os_aio_validate(); - - os_aio_segment_wait_events = static_cast( - ut_malloc(n_segments * sizeof *os_aio_segment_wait_events)); - - for (ulint i = 0; i < n_segments; ++i) { - os_aio_segment_wait_events[i] = os_event_create(); - } - - os_last_printout = ut_time(); - -#ifdef _WIN32 - ut_a(completion_port == 0 && read_completion_port == 0); - completion_port = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0); - read_completion_port = CreateIoCompletionPort(INVALID_HANDLE_VALUE, NULL, 0, 0); - ut_a(completion_port && read_completion_port); -#endif - - return(TRUE); - -} - -/*********************************************************************** -Frees the asynchronous io system. */ -UNIV_INTERN -void -os_aio_free(void) -/*=============*/ -{ - if (os_aio_ibuf_array != 0) { - os_aio_array_free(os_aio_ibuf_array); - } - - if (os_aio_log_array != 0) { - os_aio_array_free(os_aio_log_array); - } - - if (os_aio_write_array != 0) { - os_aio_array_free(os_aio_write_array); - } - - if (os_aio_sync_array != 0) { - os_aio_array_free(os_aio_sync_array); - } - - os_aio_array_free(os_aio_read_array); - - for (ulint i = 0; i < os_aio_n_segments; i++) { - os_event_free(os_aio_segment_wait_events[i]); - } - - ut_free(os_aio_segment_wait_events); - os_aio_segment_wait_events = 0; - os_aio_n_segments = 0; -#ifdef _WIN32 - completion_port = 0; - read_completion_port = 0; -#endif -} - -#ifdef WIN_ASYNC_IO -/************************************************************************//** -Wakes up all async i/o threads in the array in Windows async i/o at -shutdown. */ -static -void -os_aio_array_wake_win_aio_at_shutdown( -/*==================================*/ - os_aio_array_t* array) /*!< in: aio array */ -{ - if(completion_port) - { - PostQueuedCompletionStatus(completion_port, 0, IOCP_SHUTDOWN_KEY, NULL); - PostQueuedCompletionStatus(read_completion_port, 0, IOCP_SHUTDOWN_KEY, NULL); - } -} -#endif - -/************************************************************************//** -Wakes up all async i/o threads so that they know to exit themselves in -shutdown. */ -UNIV_INTERN -void -os_aio_wake_all_threads_at_shutdown(void) -/*=====================================*/ -{ -#ifdef WIN_ASYNC_IO - /* This code wakes up all ai/o threads in Windows native aio */ - os_aio_array_wake_win_aio_at_shutdown(os_aio_read_array); - if (os_aio_write_array != 0) { - os_aio_array_wake_win_aio_at_shutdown(os_aio_write_array); - } - - if (os_aio_ibuf_array != 0) { - os_aio_array_wake_win_aio_at_shutdown(os_aio_ibuf_array); - } - - if (os_aio_log_array != 0) { - os_aio_array_wake_win_aio_at_shutdown(os_aio_log_array); - } - -#elif defined(LINUX_NATIVE_AIO) - - /* When using native AIO interface the io helper threads - wait on io_getevents with a timeout value of 500ms. At - each wake up these threads check the server status. - No need to do anything to wake them up. */ - - if (srv_use_native_aio) { - return; - } - - /* Fall through to simulated AIO handler wakeup if we are - not using native AIO. */ -#endif /* !WIN_ASYNC_AIO */ - - /* This loop wakes up all simulated ai/o threads */ - - for (ulint i = 0; i < os_aio_n_segments; i++) { - - os_event_set(os_aio_segment_wait_events[i]); - } -} - -/************************************************************************//** -Waits until there are no pending writes in os_aio_write_array. There can -be other, synchronous, pending writes. */ -UNIV_INTERN -void -os_aio_wait_until_no_pending_writes(void) -/*=====================================*/ -{ - ut_ad(!srv_read_only_mode); - os_event_wait(os_aio_write_array->is_empty); -} - -/**********************************************************************//** -Calculates segment number for a slot. -@return segment number (which is the number used by, for example, -i/o-handler threads) */ -static -ulint -os_aio_get_segment_no_from_slot( -/*============================*/ - os_aio_array_t* array, /*!< in: aio wait array */ - os_aio_slot_t* slot) /*!< in: slot in this array */ -{ - ulint segment; - ulint seg_len; - - if (array == os_aio_ibuf_array) { - ut_ad(!srv_read_only_mode); - - segment = IO_IBUF_SEGMENT; - - } else if (array == os_aio_log_array) { - ut_ad(!srv_read_only_mode); - - segment = IO_LOG_SEGMENT; - - } else if (array == os_aio_read_array) { - seg_len = os_aio_read_array->n_slots - / os_aio_read_array->n_segments; - - segment = (srv_read_only_mode ? 0 : 2) + slot->pos / seg_len; - } else { - ut_ad(!srv_read_only_mode); - ut_a(array == os_aio_write_array); - - seg_len = os_aio_write_array->n_slots - / os_aio_write_array->n_segments; - - segment = os_aio_read_array->n_segments + 2 - + slot->pos / seg_len; - } - - return(segment); -} - -/**********************************************************************//** -Calculates local segment number and aio array from global segment number. -@return local segment number within the aio array */ -static -ulint -os_aio_get_array_and_local_segment( -/*===============================*/ - os_aio_array_t** array, /*!< out: aio wait array */ - ulint global_segment)/*!< in: global segment number */ -{ - ulint segment; - - ut_a(global_segment < os_aio_n_segments); - - if (srv_read_only_mode) { - *array = os_aio_read_array; - - return(global_segment); - } else if (global_segment == IO_IBUF_SEGMENT) { - *array = os_aio_ibuf_array; - segment = 0; - - } else if (global_segment == IO_LOG_SEGMENT) { - *array = os_aio_log_array; - segment = 0; - - } else if (global_segment < os_aio_read_array->n_segments + 2) { - *array = os_aio_read_array; - - segment = global_segment - 2; - } else { - *array = os_aio_write_array; - - segment = global_segment - (os_aio_read_array->n_segments + 2); - } - - return(segment); -} - -/*******************************************************************//** -Requests for a slot in the aio array. If no slot is available, waits until -not_full-event becomes signaled. -@return pointer to slot */ -static -os_aio_slot_t* -os_aio_array_reserve_slot( -/*======================*/ - ulint type, /*!< in: OS_FILE_READ or OS_FILE_WRITE */ - os_aio_array_t* array, /*!< in: aio array */ - fil_node_t* message1,/*!< in: message to be passed along with - the aio operation */ - void* message2,/*!< in: message to be passed along with - the aio operation */ - os_file_t file, /*!< in: file handle */ - const char* name, /*!< in: name of the file or path as a - null-terminated string */ - void* buf, /*!< in: buffer where to read or from which - to write */ - os_offset_t offset, /*!< in: file offset */ - ulint len, /*!< in: length of the block to read or write */ - ulint space_id, - ibool page_compression, /*!< in: is page compression used - on this file space */ - ulint page_compression_level, /*!< page compression - level to be used */ - ibool page_encryption, /*!< in: is page encryption used - on this file space */ - ulint page_encryption_key, /*!< page encryption key - to be used */ - ulint* write_size)/*!< in/out: Actual write size initialized - after fist successfull trim - operation for this page and if - initialized we do not trim again if - actual page size does not decrease. */ -{ - os_aio_slot_t* slot = NULL; -#ifdef WIN_ASYNC_IO - OVERLAPPED* control; - -#elif defined(LINUX_NATIVE_AIO) - - struct iocb* iocb; - off_t aio_offset; - -#endif /* WIN_ASYNC_IO */ - ulint i; - ulint counter; - ulint slots_per_seg; - ulint local_seg; - -#ifdef WIN_ASYNC_IO - ut_a((len & 0xFFFFFFFFUL) == len); -#endif /* WIN_ASYNC_IO */ - - /* No need of a mutex. Only reading constant fields */ - slots_per_seg = array->n_slots / array->n_segments; - - /* We attempt to keep adjacent blocks in the same local - segment. This can help in merging IO requests when we are - doing simulated AIO */ - local_seg = (offset >> (UNIV_PAGE_SIZE_SHIFT + 6)) - % array->n_segments; - -loop: - os_mutex_enter(array->mutex); - - if (array->n_reserved == array->n_slots) { - os_mutex_exit(array->mutex); - - if (!srv_use_native_aio) { - /* If the handler threads are suspended, wake them - so that we get more slots */ - - os_aio_simulated_wake_handler_threads(); - } - - os_event_wait(array->not_full); - - goto loop; - } - - /* We start our search for an available slot from our preferred - local segment and do a full scan of the array. We are - guaranteed to find a slot in full scan. */ - for (i = local_seg * slots_per_seg, counter = 0; - counter < array->n_slots; - i++, counter++) { - - i %= array->n_slots; - - slot = os_aio_array_get_nth_slot(array, i); - - if (slot->reserved == FALSE) { - goto found; - } - } - - /* We MUST always be able to get hold of a reserved slot. */ - ut_error; - -found: - ut_a(slot->reserved == FALSE); - array->n_reserved++; - - if (array->n_reserved == 1) { - os_event_reset(array->is_empty); - } - - if (array->n_reserved == array->n_slots) { - os_event_reset(array->not_full); - } - - slot->reserved = TRUE; - slot->reservation_time = ut_time(); - slot->message1 = message1; - slot->message2 = message2; - slot->file = file; - slot->name = name; - slot->len = len; - slot->type = type; - slot->buf = static_cast(buf); - slot->offset = offset; - slot->io_already_done = FALSE; - slot->space_id = space_id; - - slot->page_compress_success = FALSE; - slot->page_encryption_success = FALSE; - - slot->write_size = write_size; - slot->page_compression_level = page_compression_level; - slot->page_compression = page_compression; - slot->page_encryption_key = page_encryption_key; - slot->page_encryption = page_encryption; - - /* If the space is page compressed and this is write operation - then we compress the page */ - if (message1 && type == OS_FILE_WRITE && page_compression ) { - ulint real_len = len; - byte* tmp = NULL; - - /* Release the array mutex while compressing */ - os_mutex_exit(array->mutex); - - // We allocate memory for page compressed buffer if and only - // if it is not yet allocated. - if (slot->page_buf == NULL) { - os_slot_alloc_page_buf(slot); - } - -#ifdef HAVE_LZO - if (innodb_compression_algorithm == 3 && slot->lzo_mem == NULL) { - os_slot_alloc_lzo_mem(slot); - } -#endif - - /* Call page compression */ - tmp = fil_compress_page(fil_node_get_space_id(slot->message1), - (byte *)buf, - slot->page_buf, - len, - page_compression_level, - &real_len, - slot->lzo_mem - ); - - /* If compression succeeded, set up the length and buffer */ - if (tmp != buf) { - len = real_len; - buf = slot->page_buf; - slot->len = real_len; - slot->page_compress_success = TRUE; - } else { - slot->page_compress_success = FALSE; - } - - /* Take array mutex back */ - os_mutex_enter(array->mutex); - - } //CMD - /* If the space is page encryption and this is write operation - then we encrypt the page */ - if (message1 && type == OS_FILE_WRITE && page_encryption ) { - ulint real_len = len; - byte* tmp = NULL; - - /* Release the array mutex while encrypting */ - os_mutex_exit(array->mutex); - - // We allocate memory for page encrypted buffer if and only - // if it is not yet allocated. - if (slot->page_buf2 == NULL) { - os_slot_alloc_page_buf2(slot); - } - - ut_ad(slot->page_buf2); - tmp = fil_encrypt_page(fil_node_get_space_id(slot->message1), (byte *)buf, slot->page_buf2, len, page_encryption_key, &real_len); - - /* If encryption succeeded, set up the length and buffer */ - if (tmp != buf) { - len = real_len; - buf = slot->page_buf2; - slot->len = real_len; - slot->page_encryption_success = TRUE; - } else { - slot->page_encryption_success = FALSE; - } - - /* Take array mutex back */ - os_mutex_enter(array->mutex); - } - -#ifdef WIN_ASYNC_IO - control = &slot->control; - control->Offset = (DWORD) offset & 0xFFFFFFFF; - control->OffsetHigh = (DWORD) (offset >> 32); - control->hEvent = 0; - slot->arr = array; - -#elif defined(LINUX_NATIVE_AIO) - - /* If we are not using native AIO skip this part. */ - if (!srv_use_native_aio) { - goto skip_native_aio; - } - - /* Check if we are dealing with 64 bit arch. - If not then make sure that offset fits in 32 bits. */ - aio_offset = (off_t) offset; - - ut_a(sizeof(aio_offset) >= sizeof(offset) - || ((os_offset_t) aio_offset) == offset); - - iocb = &slot->control; - - if (type == OS_FILE_READ) { - io_prep_pread(iocb, file, buf, len, aio_offset); - } else { - ut_a(type == OS_FILE_WRITE); - io_prep_pwrite(iocb, file, buf, len, aio_offset); - } - - iocb->data = (void*) slot; - slot->n_bytes = 0; - slot->ret = 0; - -skip_native_aio: -#endif /* LINUX_NATIVE_AIO */ - os_mutex_exit(array->mutex); - - return(slot); -} - -/*******************************************************************//** -Frees a slot in the aio array. */ -static -void -os_aio_array_free_slot( -/*===================*/ - os_aio_array_t* array, /*!< in: aio array */ - os_aio_slot_t* slot) /*!< in: pointer to slot */ -{ - os_mutex_enter(array->mutex); - - ut_ad(slot->reserved); - - slot->reserved = FALSE; - - array->n_reserved--; - - if (array->n_reserved == array->n_slots - 1) { - os_event_set(array->not_full); - } - - if (array->n_reserved == 0) { - os_event_set(array->is_empty); - } - -#ifdef LINUX_NATIVE_AIO - - if (srv_use_native_aio) { - memset(&slot->control, 0x0, sizeof(slot->control)); - slot->n_bytes = 0; - slot->ret = 0; - /*fprintf(stderr, "Freed up Linux native slot.\n");*/ - } else { - /* These fields should not be used if we are not - using native AIO. */ - ut_ad(slot->n_bytes == 0); - ut_ad(slot->ret == 0); - } - -#endif - os_mutex_exit(array->mutex); -} - -/**********************************************************************//** -Wakes up a simulated aio i/o-handler thread if it has something to do. */ -static -void -os_aio_simulated_wake_handler_thread( -/*=================================*/ - ulint global_segment) /*!< in: the number of the segment in the aio - arrays */ -{ - os_aio_array_t* array; - ulint segment; - - ut_ad(!srv_use_native_aio); - - segment = os_aio_get_array_and_local_segment(&array, global_segment); - - ulint n = array->n_slots / array->n_segments; - - segment *= n; - - /* Look through n slots after the segment * n'th slot */ - - os_mutex_enter(array->mutex); - - for (ulint i = 0; i < n; ++i) { - const os_aio_slot_t* slot; - - slot = os_aio_array_get_nth_slot(array, segment + i); - - if (slot->reserved) { - - /* Found an i/o request */ - - os_mutex_exit(array->mutex); - - os_event_t event; - - event = os_aio_segment_wait_events[global_segment]; - - os_event_set(event); - - return; - } - } - - os_mutex_exit(array->mutex); -} - -/**********************************************************************//** -Wakes up simulated aio i/o-handler threads if they have something to do. */ -UNIV_INTERN -void -os_aio_simulated_wake_handler_threads(void) -/*=======================================*/ -{ - if (srv_use_native_aio) { - /* We do not use simulated aio: do nothing */ - - return; - } - - os_aio_recommend_sleep_for_read_threads = FALSE; - - for (ulint i = 0; i < os_aio_n_segments; i++) { - os_aio_simulated_wake_handler_thread(i); - } -} - -/**********************************************************************//** -This function can be called if one wants to post a batch of reads and -prefers an i/o-handler thread to handle them all at once later. You must -call os_aio_simulated_wake_handler_threads later to ensure the threads -are not left sleeping! */ -UNIV_INTERN -void -os_aio_simulated_put_read_threads_to_sleep(void) -/*============================================*/ -{ - -/* The idea of putting background IO threads to sleep is only for -Windows when using simulated AIO. Windows XP seems to schedule -background threads too eagerly to allow for coalescing during -readahead requests. */ -#ifdef __WIN__ - os_aio_array_t* array; - - if (srv_use_native_aio) { - /* We do not use simulated aio: do nothing */ - - return; - } - - os_aio_recommend_sleep_for_read_threads = TRUE; - - for (ulint i = 0; i < os_aio_n_segments; i++) { - os_aio_get_array_and_local_segment(&array, i); - - if (array == os_aio_read_array) { - - os_event_reset(os_aio_segment_wait_events[i]); - } - } -#endif /* __WIN__ */ -} - -#if defined(LINUX_NATIVE_AIO) -/*******************************************************************//** -Dispatch an AIO request to the kernel. -@return TRUE on success. */ -static -ibool -os_aio_linux_dispatch( -/*==================*/ - os_aio_array_t* array, /*!< in: io request array. */ - os_aio_slot_t* slot) /*!< in: an already reserved slot. */ -{ - int ret; - ulint io_ctx_index; - struct iocb* iocb; - - ut_ad(slot != NULL); - ut_ad(array); - - ut_a(slot->reserved); - - /* Find out what we are going to work with. - The iocb struct is directly in the slot. - The io_context is one per segment. */ - - iocb = &slot->control; - io_ctx_index = (slot->pos * array->n_segments) / array->n_slots; - - ret = io_submit(array->aio_ctx[io_ctx_index], 1, &iocb); - -#if defined(UNIV_AIO_DEBUG) - fprintf(stderr, - "io_submit[%c] ret[%d]: slot[%p] ctx[%p] seg[%lu]\n", - (slot->type == OS_FILE_WRITE) ? 'w' : 'r', ret, slot, - array->aio_ctx[io_ctx_index], (ulong) io_ctx_index); -#endif - - /* io_submit returns number of successfully - queued requests or -errno. */ - if (UNIV_UNLIKELY(ret != 1)) { - errno = -ret; - return(FALSE); - } - - return(TRUE); -} -#endif /* LINUX_NATIVE_AIO */ - - -/*******************************************************************//** -NOTE! Use the corresponding macro os_aio(), not directly this function! -Requests an asynchronous i/o operation. -@return TRUE if request was queued successfully, FALSE if fail */ -UNIV_INTERN -ibool -os_aio_func( -/*========*/ - ulint type, /*!< in: OS_FILE_READ or OS_FILE_WRITE */ - ulint mode, /*!< in: OS_AIO_NORMAL, ..., possibly ORed - to OS_AIO_SIMULATED_WAKE_LATER: the - last flag advises this function not to wake - i/o-handler threads, but the caller will - do the waking explicitly later, in this - way the caller can post several requests in - a batch; NOTE that the batch must not be - so big that it exhausts the slots in aio - arrays! NOTE that a simulated batch - may introduce hidden chances of deadlocks, - because i/os are not actually handled until - all have been posted: use with great - caution! */ - const char* name, /*!< in: name of the file or path as a - null-terminated string */ - os_file_t file, /*!< in: handle to a file */ - void* buf, /*!< in: buffer where to read or from which - to write */ - os_offset_t offset, /*!< in: file offset where to read or write */ - ulint n, /*!< in: number of bytes to read or write */ - fil_node_t* message1,/*!< in: message for the aio handler - (can be used to identify a completed - aio operation); ignored if mode is - OS_AIO_SYNC */ - void* message2,/*!< in: message for the aio handler - (can be used to identify a completed - aio operation); ignored if mode is - OS_AIO_SYNC */ - ulint space_id, - trx_t* trx, - ibool page_compression, /*!< in: is page compression used - on this file space */ - ulint page_compression_level, /*!< page compression - level to be used */ - ibool page_encryption, /*!< in: is page encryption used - on this file space */ - ulint page_encryption_key, /*!< page encryption key - to be used */ - ulint* write_size)/*!< in/out: Actual write size initialized - after fist successfull trim - operation for this page and if - initialized we do not trim again if - actual page size does not decrease. */ -{ - os_aio_array_t* array; - os_aio_slot_t* slot; -#ifdef WIN_ASYNC_IO - DWORD len = (DWORD) n; - BOOL ret; -#endif - ulint wake_later; - - ut_ad(file); - ut_ad(buf); - ut_ad(n > 0); - ut_ad(n % OS_MIN_LOG_BLOCK_SIZE == 0); - ut_ad(offset % OS_MIN_LOG_BLOCK_SIZE == 0); - ut_ad(os_aio_validate_skip()); -#ifdef WIN_ASYNC_IO - ut_ad((n & 0xFFFFFFFFUL) == n); -#endif - - wake_later = mode & OS_AIO_SIMULATED_WAKE_LATER; - mode = mode & (~OS_AIO_SIMULATED_WAKE_LATER); - - if (mode == OS_AIO_SYNC) - { - ibool ret; - /* This is actually an ordinary synchronous read or write: - no need to use an i/o-handler thread */ - - if (type == OS_FILE_READ) { - ret = os_file_read_func(file, buf, offset, n, trx, - page_compression); - } - else { - ut_ad(!srv_read_only_mode); - ut_a(type == OS_FILE_WRITE); - - ret = os_file_write(name, file, buf, offset, n); - } - ut_a(ret); - return ret; - } - -try_again: - switch (mode) { - case OS_AIO_NORMAL: - if (type == OS_FILE_READ) { - array = os_aio_read_array; - } else { - ut_ad(!srv_read_only_mode); - array = os_aio_write_array; - } - break; - case OS_AIO_IBUF: - ut_ad(type == OS_FILE_READ); - /* Reduce probability of deadlock bugs in connection with ibuf: - do not let the ibuf i/o handler sleep */ - - wake_later = FALSE; - - if (srv_read_only_mode) { - array = os_aio_read_array; - } else { - array = os_aio_ibuf_array; - } - break; - case OS_AIO_LOG: - if (srv_read_only_mode) { - array = os_aio_read_array; - } else { - array = os_aio_log_array; - } - break; - case OS_AIO_SYNC: - array = os_aio_sync_array; -#if defined(LINUX_NATIVE_AIO) - /* In Linux native AIO we don't use sync IO array. */ - ut_a(!srv_use_native_aio); -#endif /* LINUX_NATIVE_AIO */ - break; - default: - ut_error; - array = NULL; /* Eliminate compiler warning */ - } - - if (trx && type == OS_FILE_READ) - { - trx->io_reads++; - trx->io_read += n; - } - slot = os_aio_array_reserve_slot(type, array, message1, message2, file, - name, buf, offset, n, space_id, - page_compression, page_compression_level, - page_encryption, page_encryption_key, write_size); - if (type == OS_FILE_READ) { - if (srv_use_native_aio) { - os_n_file_reads++; - os_bytes_read_since_printout += n; -#ifdef WIN_ASYNC_IO - ret = ReadFile(file, buf, (DWORD) n, &len, - &(slot->control)); - if(!ret && GetLastError() != ERROR_IO_PENDING) - goto err_exit; - -#elif defined(LINUX_NATIVE_AIO) - if (!os_aio_linux_dispatch(array, slot)) { - goto err_exit; - } -#endif /* WIN_ASYNC_IO */ - } else { - if (!wake_later) { - os_aio_simulated_wake_handler_thread( - os_aio_get_segment_no_from_slot( - array, slot)); - } - } - } else if (type == OS_FILE_WRITE) { - ut_ad(!srv_read_only_mode); - if (srv_use_native_aio) { - os_n_file_writes++; -#ifdef WIN_ASYNC_IO - ret = WriteFile(file, buf, (DWORD) n, &len, - &(slot->control)); - - if(!ret && GetLastError() != ERROR_IO_PENDING) - goto err_exit; -#elif defined(LINUX_NATIVE_AIO) - if (!os_aio_linux_dispatch(array, slot)) { - goto err_exit; - } -#endif /* WIN_ASYNC_IO */ - } else { - if (!wake_later) { - os_aio_simulated_wake_handler_thread( - os_aio_get_segment_no_from_slot( - array, slot)); - } - } - } else { - ut_error; - } - - /* aio was queued successfully! */ - return(TRUE); - -#if defined LINUX_NATIVE_AIO || defined WIN_ASYNC_IO -err_exit: -#endif /* LINUX_NATIVE_AIO || WIN_ASYNC_IO */ - os_aio_array_free_slot(array, slot); - - if (os_file_handle_error( - name,type == OS_FILE_READ ? "aio read" : "aio write", __FILE__, __LINE__)) { - - goto try_again; - } - - return(FALSE); -} - -#ifdef WIN_ASYNC_IO -#define READ_SEGMENT(x) (x < srv_n_read_io_threads) -#define WRITE_SEGMENT(x) !READ_SEGMENT(x) - -/**********************************************************************//** -This function is only used in Windows asynchronous i/o. -Waits for an aio operation to complete. This function is used to wait the -for completed requests. The aio array of pending requests is divided -into segments. The thread specifies which segment or slot it wants to wait -for. NOTE: this function will also take care of freeing the aio slot, -therefore no other thread is allowed to do the freeing! -@return TRUE if the aio operation succeeded */ -UNIV_INTERN -ibool -os_aio_windows_handle( -/*==================*/ - ulint segment, /*!< in: the number of the segment in the aio - arrays to wait for; segment 0 is the ibuf - i/o thread, segment 1 the log i/o thread, - then follow the non-ibuf read threads, and as - the last are the non-ibuf write threads; if - this is ULINT_UNDEFINED, then it means that - sync aio is used, and this parameter is - ignored */ - ulint pos, /*!< this parameter is used only in sync aio: - wait for the aio slot at this position */ - fil_node_t**message1, /*!< out: the messages passed with the aio - request; note that also in the case where - the aio operation failed, these output - parameters are valid and can be used to - restart the operation, for example */ - void** message2, - ulint* type, /*!< out: OS_FILE_WRITE or ..._READ */ - ulint* space_id) -{ - ulint orig_seg = segment; - os_aio_slot_t* slot; - ibool ret_val; - BOOL ret; - DWORD len; - BOOL retry = FALSE; - ULONG_PTR key; - HANDLE port = READ_SEGMENT(segment)? read_completion_port : completion_port; - - for(;;) { - ret = GetQueuedCompletionStatus(port, &len, &key, - (OVERLAPPED **)&slot, INFINITE); - - /* If shutdown key was received, repost the shutdown message and exit */ - if (ret && (key == IOCP_SHUTDOWN_KEY)) { - PostQueuedCompletionStatus(port, 0, key, NULL); - os_thread_exit(NULL); - } - - if (srv_shutdown_state == SRV_SHUTDOWN_EXIT_THREADS) { - os_thread_exit(NULL); - } - - if(WRITE_SEGMENT(segment)&& slot->type == OS_FILE_READ) { - /* - Redirect read completions to the dedicated completion port - and thread. We need to split read and write threads. If we do not - do that, and just allow all io threads process all IO, it is possible - to get stuck in a deadlock in buffer pool code, - - Currently, the problem is solved this way - "write io" threads - always get all completion notifications, from both async reads and - writes. Write completion is handled in the same thread that gets it. - Read completion is forwarded via PostQueueCompletionStatus()) - to the second completion port dedicated solely to reads. One of the - "read io" threads waiting on this port will finally handle the IO. - - Forwarding IO completion this way costs a context switch , and this - seems tolerable since asynchronous reads are by far less frequent. - */ - ut_a(PostQueuedCompletionStatus(read_completion_port, len, key, - &slot->control)); - } - else { - break; - } - } - - *message1 = slot->message1; - *message2 = slot->message2; - - *type = slot->type; - *space_id = slot->space_id; - - if (ret && len == slot->len) { - - ret_val = TRUE; - } else if (os_file_handle_error(slot->name, "Windows aio", __FILE__, __LINE__)) { - - retry = TRUE; - } else { - - ret_val = FALSE; - } - - if (retry) { - /* retry failed read/write operation synchronously. - No need to hold array->mutex. */ - -#ifdef UNIV_PFS_IO - /* This read/write does not go through os_file_read - and os_file_write APIs, need to register with - performance schema explicitly here. */ - struct PSI_file_locker* locker = NULL; - register_pfs_file_io_begin(locker, slot->file, slot->len, - (slot->type == OS_FILE_WRITE) - ? PSI_FILE_WRITE - : PSI_FILE_READ, - __FILE__, __LINE__); -#endif - - ut_a((slot->len & 0xFFFFFFFFUL) == slot->len); - - switch (slot->type) { - case OS_FILE_WRITE: - if (slot->message1 && slot->page_compression && slot->page_buf) { - ret_val = os_file_write(slot->name, slot->file, slot->page_buf, - slot->offset, slot->len); - } else { - - ret_val = os_file_write(slot->name, slot->file, slot->buf, - slot->offset, slot->len); - } - break; - case OS_FILE_READ: - ret_val = os_file_read(slot->file, slot->buf, - slot->offset, slot->len, slot->page_compression); - break; - default: - ut_error; - } - -#ifdef UNIV_PFS_IO - register_pfs_file_io_end(locker, len); -#endif - - if (!ret && GetLastError() == ERROR_IO_PENDING) { - /* aio was queued successfully! - We want a synchronous i/o operation on a - file where we also use async i/o: in Windows - we must use the same wait mechanism as for - async i/o */ - - ret = GetOverlappedResult(slot->file, - &(slot->control), - &len, TRUE); - } - - ret_val = ret && len == slot->len; - } - - if (slot->message1 && slot->page_compression) { - // We allocate memory for page compressed buffer if and only - // if it is not yet allocated. - if (slot->page_buf == NULL) { - os_slot_alloc_page_buf(slot); - } - -#ifdef HAVE_LZO - if (innodb_compression_algorithm == 3 && slot->lzo_mem == NULL) { - os_slot_alloc_lzo_mem(slot); - } -#endif - if (slot->type == OS_FILE_READ) { - fil_decompress_page(slot->page_buf, slot->buf, slot->len, slot->write_size); - } else { - if (slot->page_compress_success && fil_page_is_compressed(slot->page_buf)) { - if (srv_use_trim && os_fallocate_failed == FALSE) { - // Deallocate unused blocks from file system - os_file_trim(slot->file, slot, slot->len); - } - } - } - } - - os_aio_array_free_slot((os_aio_array_t *)slot->arr, slot); - - return(ret_val); -} -#endif - -#if defined(LINUX_NATIVE_AIO) -/******************************************************************//** -This function is only used in Linux native asynchronous i/o. This is -called from within the io-thread. If there are no completed IO requests -in the slot array, the thread calls this function to collect more -requests from the kernel. -The io-thread waits on io_getevents(), which is a blocking call, with -a timeout value. Unless the system is very heavy loaded, keeping the -io-thread very busy, the io-thread will spend most of its time waiting -in this function. -The io-thread also exits in this function. It checks server status at -each wakeup and that is why we use timed wait in io_getevents(). */ -static -void -os_aio_linux_collect( -/*=================*/ - os_aio_array_t* array, /*!< in/out: slot array. */ - ulint segment, /*!< in: local segment no. */ - ulint seg_size) /*!< in: segment size. */ -{ - int i; - int ret; - ulint start_pos; - ulint end_pos; - struct timespec timeout; - struct io_event* events; - struct io_context* io_ctx; - - /* sanity checks. */ - ut_ad(array != NULL); - ut_ad(seg_size > 0); - ut_ad(segment < array->n_segments); - - /* Which part of event array we are going to work on. */ - events = &array->aio_events[segment * seg_size]; - - /* Which io_context we are going to use. */ - io_ctx = array->aio_ctx[segment]; - - /* Starting point of the segment we will be working on. */ - start_pos = segment * seg_size; - - /* End point. */ - end_pos = start_pos + seg_size; - -retry: - - /* Initialize the events. The timeout value is arbitrary. - We probably need to experiment with it a little. */ - memset(events, 0, sizeof(*events) * seg_size); - timeout.tv_sec = 0; - timeout.tv_nsec = OS_AIO_REAP_TIMEOUT; - - ret = io_getevents(io_ctx, 1, seg_size, events, &timeout); - - if (ret > 0) { - for (i = 0; i < ret; i++) { - os_aio_slot_t* slot; - struct iocb* control; - - control = (struct iocb*) events[i].obj; - ut_a(control != NULL); - - slot = (os_aio_slot_t*) control->data; - - /* Some sanity checks. */ - ut_a(slot != NULL); - ut_a(slot->reserved); - -#if defined(UNIV_AIO_DEBUG) - fprintf(stderr, - "io_getevents[%c]: slot[%p] ctx[%p]" - " seg[%lu]\n", - (slot->type == OS_FILE_WRITE) ? 'w' : 'r', - slot, io_ctx, segment); -#endif - - /* We are not scribbling previous segment. */ - ut_a(slot->pos >= start_pos); - - /* We have not overstepped to next segment. */ - ut_a(slot->pos < end_pos); - - /* If the table is page compressed and this is read, - we decompress before we annouce the read is - complete. For writes, we free the compressed page. */ - if (slot->message1 && slot->page_compression) { - // We allocate memory for page compressed buffer if and only - // if it is not yet allocated. - if (slot->page_buf == NULL) { - os_slot_alloc_page_buf(slot); - } - -#ifdef HAVE_LZO - if (innodb_compression_algorithm == 3 && slot->lzo_mem == NULL) { - os_slot_alloc_lzo_mem(slot); - } -#endif - if (slot->type == OS_FILE_READ) { - fil_decompress_page(slot->page_buf, slot->buf, slot->len, slot->write_size); - } else { - if (slot->page_compress_success && - fil_page_is_compressed(slot->page_buf)) { - ut_ad(slot->page_compression_page); - if (srv_use_trim && os_fallocate_failed == FALSE) { - // Deallocate unused blocks from file system - os_file_trim(slot->file, slot, slot->len); - } - } - } - } - - /* page encryption */ - if (slot->message1 && slot->page_encryption) { - if (slot->page_buf2==NULL) { - os_slot_alloc_page_buf2(slot); - } - - ut_ad(slot->page_buf2); - - if (slot->type == OS_FILE_READ) { - if (fil_page_is_encrypted(slot->buf)) { - fil_decrypt_page(slot->page_buf2, slot->buf, slot->len, slot->write_size); - } - } else { - if (slot->page_encryption_success && - fil_page_is_encrypted(slot->page_buf2)) { - ut_ad(slot->page_encryption_page); - if (srv_use_trim && os_fallocate_failed == FALSE) { - // Deallocate unused blocks from file system - os_file_trim(slot->file, slot, slot->len); - } - } - } - } - - /* Mark this request as completed. The error handling - will be done in the calling function. */ - os_mutex_enter(array->mutex); - slot->n_bytes = events[i].res; - slot->ret = events[i].res2; - slot->io_already_done = TRUE; - os_mutex_exit(array->mutex); - } - return; - } - - if (UNIV_UNLIKELY(srv_shutdown_state == SRV_SHUTDOWN_EXIT_THREADS)) { - return; - } - - /* This error handling is for any error in collecting the - IO requests. The errors, if any, for any particular IO - request are simply passed on to the calling routine. */ - - switch (ret) { - case -EAGAIN: - /* Not enough resources! Try again. */ - case -EINTR: - /* Interrupted! I have tested the behaviour in case of an - interrupt. If we have some completed IOs available then - the return code will be the number of IOs. We get EINTR only - if there are no completed IOs and we have been interrupted. */ - case 0: - /* No pending request! Go back and check again. */ - goto retry; - } - - /* All other errors should cause a trap for now. */ - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: unexpected ret_code[%d] from io_getevents()!\n", - ret); - ut_error; -} - -/**********************************************************************//** -This function is only used in Linux native asynchronous i/o. -Waits for an aio operation to complete. This function is used to wait for -the completed requests. The aio array of pending requests is divided -into segments. The thread specifies which segment or slot it wants to wait -for. NOTE: this function will also take care of freeing the aio slot, -therefore no other thread is allowed to do the freeing! -@return TRUE if the IO was successful */ -UNIV_INTERN -ibool -os_aio_linux_handle( -/*================*/ - ulint global_seg, /*!< in: segment number in the aio array - to wait for; segment 0 is the ibuf - i/o thread, segment 1 is log i/o thread, - then follow the non-ibuf read threads, - and the last are the non-ibuf write - threads. */ - fil_node_t**message1, /*!< out: the messages passed with the */ - void** message2, /*!< aio request; note that in case the - aio operation failed, these output - parameters are valid and can be used to - restart the operation. */ - ulint* type, /*!< out: OS_FILE_WRITE or ..._READ */ - ulint* space_id) -{ - ulint segment; - os_aio_array_t* array; - os_aio_slot_t* slot; - ulint n; - ulint i; - ibool ret = FALSE; - - /* Should never be doing Sync IO here. */ - ut_a(global_seg != ULINT_UNDEFINED); - - /* Find the array and the local segment. */ - segment = os_aio_get_array_and_local_segment(&array, global_seg); - n = array->n_slots / array->n_segments; - - wait_for_event: - /* Loop until we have found a completed request. */ - for (;;) { - ibool any_reserved = FALSE; - os_mutex_enter(array->mutex); - for (i = 0; i < n; ++i) { - slot = os_aio_array_get_nth_slot( - array, i + segment * n); - if (!slot->reserved) { - continue; - } else if (slot->io_already_done) { - /* Something for us to work on. */ - goto found; - } else { - any_reserved = TRUE; - } - } - - os_mutex_exit(array->mutex); - - /* There is no completed request. - If there is no pending request at all, - and the system is being shut down, exit. */ - if (UNIV_UNLIKELY - (!any_reserved - && srv_shutdown_state == SRV_SHUTDOWN_EXIT_THREADS)) { - *message1 = NULL; - *message2 = NULL; - return(TRUE); - } - - /* Wait for some request. Note that we return - from wait iff we have found a request. */ - - srv_set_io_thread_op_info(global_seg, - "waiting for completed aio requests"); - os_aio_linux_collect(array, segment, n); - } - -found: - /* Note that it may be that there are more then one completed - IO requests. We process them one at a time. We may have a case - here to improve the performance slightly by dealing with all - requests in one sweep. */ - srv_set_io_thread_op_info(global_seg, - "processing completed aio requests"); - - /* Ensure that we are scribbling only our segment. */ - ut_a(i < n); - - ut_ad(slot != NULL); - ut_ad(slot->reserved); - ut_ad(slot->io_already_done); - - *message1 = slot->message1; - *message2 = slot->message2; - - *type = slot->type; - *space_id = slot->space_id; - - if (slot->ret == 0 && slot->n_bytes == (long) slot->len) { - - ret = TRUE; - } else if ((slot->ret == 0) && (slot->n_bytes > 0) - && (slot->n_bytes < (long) slot->len)) { - /* Partial read or write scenario */ - int submit_ret; - struct iocb* iocb; - slot->buf = (byte*)slot->buf + slot->n_bytes; - slot->offset = slot->offset + slot->n_bytes; - slot->len = slot->len - slot->n_bytes; - /* Resetting the bytes read/written */ - slot->n_bytes = 0; - slot->io_already_done = FALSE; - iocb = &(slot->control); - - if (slot->type == OS_FILE_READ) { - io_prep_pread(&slot->control, slot->file, slot->buf, - slot->len, (off_t) slot->offset); - } else { - ut_a(slot->type == OS_FILE_WRITE); - io_prep_pwrite(&slot->control, slot->file, slot->buf, - slot->len, (off_t) slot->offset); - } - /* Resubmit an I/O request */ - submit_ret = io_submit(array->aio_ctx[segment], 1, &iocb); - if (submit_ret < 0 ) { - /* Aborting in case of submit failure */ - ib_logf(IB_LOG_LEVEL_FATAL, - "Native Linux AIO interface. io_submit()" - " call failed when resubmitting a partial" - " I/O request on the file %s.", - slot->name); - } else { - ret = FALSE; - os_mutex_exit(array->mutex); - goto wait_for_event; - } - } else { - errno = -slot->ret; - - if (slot->ret == 0) { - fprintf(stderr, - "InnoDB: Number of bytes after aio %d requested %lu\n" - "InnoDB: from file %s\n", - slot->n_bytes, slot->len, slot->name); - } - - /* os_file_handle_error does tell us if we should retry - this IO. As it stands now, we don't do this retry when - reaping requests from a different context than - the dispatcher. This non-retry logic is the same for - windows and linux native AIO. - We should probably look into this to transparently - re-submit the IO. */ - os_file_handle_error(slot->name, "Linux aio", __FILE__, __LINE__); - - ret = FALSE; - } - - os_mutex_exit(array->mutex); - - os_aio_array_free_slot(array, slot); - - return(ret); -} -#endif /* LINUX_NATIVE_AIO */ - -/**********************************************************************//** -Does simulated aio. This function should be called by an i/o-handler -thread. -@return TRUE if the aio operation succeeded */ -UNIV_INTERN -ibool -os_aio_simulated_handle( -/*====================*/ - ulint global_segment, /*!< in: the number of the segment in the aio - arrays to wait for; segment 0 is the ibuf - i/o thread, segment 1 the log i/o thread, - then follow the non-ibuf read threads, and as - the last are the non-ibuf write threads */ - fil_node_t**message1, /*!< out: the messages passed with the aio - request; note that also in the case where - the aio operation failed, these output - parameters are valid and can be used to - restart the operation, for example */ - void** message2, - ulint* type, /*!< out: OS_FILE_WRITE or ..._READ */ - ulint* space_id) -{ - os_aio_array_t* array; - ulint segment; - os_aio_slot_t* consecutive_ios[OS_AIO_MERGE_N_CONSECUTIVE]; - ulint n_consecutive; - ulint total_len; - ulint offs; - os_offset_t lowest_offset; - ulint biggest_age; - ulint age; - byte* combined_buf; - byte* combined_buf2; - ibool ret; - ibool any_reserved; - ulint n; - os_aio_slot_t* aio_slot; - - /* Fix compiler warning */ - *consecutive_ios = NULL; - - segment = os_aio_get_array_and_local_segment(&array, global_segment); - -restart: - /* NOTE! We only access constant fields in os_aio_array. Therefore - we do not have to acquire the protecting mutex yet */ - - srv_set_io_thread_op_info(global_segment, - "looking for i/o requests (a)"); - ut_ad(os_aio_validate_skip()); - ut_ad(segment < array->n_segments); - - n = array->n_slots / array->n_segments; - - /* Look through n slots after the segment * n'th slot */ - - if (array == os_aio_read_array - && os_aio_recommend_sleep_for_read_threads) { - - /* Give other threads chance to add several i/os to the array - at once. */ - - goto recommended_sleep; - } - - srv_set_io_thread_op_info(global_segment, - "looking for i/o requests (b)"); - - /* Check if there is a slot for which the i/o has already been - done */ - any_reserved = FALSE; - - os_mutex_enter(array->mutex); - - for (ulint i = 0; i < n; i++) { - os_aio_slot_t* slot; - - slot = os_aio_array_get_nth_slot(array, i + segment * n); - - if (!slot->reserved) { - continue; - } else if (slot->io_already_done) { - - if (os_aio_print_debug) { - fprintf(stderr, - "InnoDB: i/o for slot %lu" - " already done, returning\n", - (ulong) i); - } - - aio_slot = slot; - ret = TRUE; - goto slot_io_done; - } else { - any_reserved = TRUE; - } - } - - /* There is no completed request. - If there is no pending request at all, - and the system is being shut down, exit. */ - if (!any_reserved && srv_shutdown_state == SRV_SHUTDOWN_EXIT_THREADS) { - os_mutex_exit(array->mutex); - *message1 = NULL; - *message2 = NULL; - return(TRUE); - } - - n_consecutive = 0; - - /* If there are at least 2 seconds old requests, then pick the oldest - one to prevent starvation. If several requests have the same age, - then pick the one at the lowest offset. */ - - biggest_age = 0; - lowest_offset = IB_UINT64_MAX; - - for (ulint i = 0; i < n; i++) { - os_aio_slot_t* slot; - - slot = os_aio_array_get_nth_slot(array, i + segment * n); - - if (slot->reserved) { - - age = (ulint) difftime( - ut_time(), slot->reservation_time); - - if ((age >= 2 && age > biggest_age) - || (age >= 2 && age == biggest_age - && slot->offset < lowest_offset)) { - - /* Found an i/o request */ - consecutive_ios[0] = slot; - - n_consecutive = 1; - - biggest_age = age; - lowest_offset = slot->offset; - } - } - } - - if (n_consecutive == 0) { - /* There were no old requests. Look for an i/o request at the - lowest offset in the array (we ignore the high 32 bits of the - offset in these heuristics) */ - - lowest_offset = IB_UINT64_MAX; - - for (ulint i = 0; i < n; i++) { - os_aio_slot_t* slot; - - slot = os_aio_array_get_nth_slot( - array, i + segment * n); - - if (slot->reserved && slot->offset < lowest_offset) { - - /* Found an i/o request */ - consecutive_ios[0] = slot; - - n_consecutive = 1; - - lowest_offset = slot->offset; - } - } - } - - if (n_consecutive == 0) { - - /* No i/o requested at the moment */ - - goto wait_for_io; - } - - /* if n_consecutive != 0, then we have assigned - something valid to consecutive_ios[0] */ - ut_ad(n_consecutive != 0); - ut_ad(consecutive_ios[0] != NULL); - - aio_slot = consecutive_ios[0]; - - /* Check if there are several consecutive blocks to read or write */ - -consecutive_loop: - for (ulint i = 0; i < n; i++) { - os_aio_slot_t* slot; - - slot = os_aio_array_get_nth_slot(array, i + segment * n); - - if (slot->reserved - && slot != aio_slot - && slot->offset == aio_slot->offset + aio_slot->len - && slot->type == aio_slot->type - && slot->file == aio_slot->file) { - - /* Found a consecutive i/o request */ - - consecutive_ios[n_consecutive] = slot; - n_consecutive++; - - aio_slot = slot; - - if (n_consecutive < OS_AIO_MERGE_N_CONSECUTIVE) { - - goto consecutive_loop; - } else { - break; - } - } - } - - srv_set_io_thread_op_info(global_segment, "consecutive i/o requests"); - - /* We have now collected n_consecutive i/o requests in the array; - allocate a single buffer which can hold all data, and perform the - i/o */ - - total_len = 0; - aio_slot = consecutive_ios[0]; - - for (ulint i = 0; i < n_consecutive; i++) { - total_len += consecutive_ios[i]->len; - } - - if (n_consecutive == 1) { - /* We can use the buffer of the i/o request */ - combined_buf = aio_slot->buf; - combined_buf2 = NULL; - } else { - combined_buf2 = static_cast( - ut_malloc(total_len + UNIV_PAGE_SIZE)); - - ut_a(combined_buf2); - - combined_buf = static_cast( - ut_align(combined_buf2, UNIV_PAGE_SIZE)); - } - - /* We release the array mutex for the time of the i/o: NOTE that - this assumes that there is just one i/o-handler thread serving - a single segment of slots! */ - - os_mutex_exit(array->mutex); - - if (aio_slot->type == OS_FILE_WRITE && n_consecutive > 1) { - /* Copy the buffers to the combined buffer */ - offs = 0; - - for (ulint i = 0; i < n_consecutive; i++) { - - ut_memcpy(combined_buf + offs, consecutive_ios[i]->buf, - consecutive_ios[i]->len); - - offs += consecutive_ios[i]->len; - } - } - - srv_set_io_thread_op_info(global_segment, "doing file i/o"); - - /* Do the i/o with ordinary, synchronous i/o functions: */ - if (aio_slot->type == OS_FILE_WRITE) { - ut_ad(!srv_read_only_mode); - ret = os_file_write( - aio_slot->name, aio_slot->file, combined_buf, - aio_slot->offset, total_len); - } else { - ret = os_file_read( - aio_slot->file, combined_buf, - aio_slot->offset, total_len, - aio_slot->page_compression); - } - - ut_a(ret); - srv_set_io_thread_op_info(global_segment, "file i/o done"); - - if (aio_slot->type == OS_FILE_READ && n_consecutive > 1) { - /* Copy the combined buffer to individual buffers */ - offs = 0; - - for (ulint i = 0; i < n_consecutive; i++) { - - ut_memcpy(consecutive_ios[i]->buf, combined_buf + offs, - consecutive_ios[i]->len); - offs += consecutive_ios[i]->len; - } - } - - if (combined_buf2) { - ut_free(combined_buf2); - } - - os_mutex_enter(array->mutex); - - /* Mark the i/os done in slots */ - - for (ulint i = 0; i < n_consecutive; i++) { - consecutive_ios[i]->io_already_done = TRUE; - } - - /* We return the messages for the first slot now, and if there were - several slots, the messages will be returned with subsequent calls - of this function */ - -slot_io_done: - - ut_a(aio_slot->reserved); - - *message1 = aio_slot->message1; - *message2 = aio_slot->message2; - - *type = aio_slot->type; - *space_id = aio_slot->space_id; - - os_mutex_exit(array->mutex); - - os_aio_array_free_slot(array, aio_slot); - - return(ret); - -wait_for_io: - srv_set_io_thread_op_info(global_segment, "resetting wait event"); - - /* We wait here until there again can be i/os in the segment - of this thread */ - - os_event_reset(os_aio_segment_wait_events[global_segment]); - - os_mutex_exit(array->mutex); - -recommended_sleep: - srv_set_io_thread_op_info(global_segment, "waiting for i/o request"); - - os_event_wait(os_aio_segment_wait_events[global_segment]); - - goto restart; -} - -/**********************************************************************//** -Validates the consistency of an aio array. -@return true if ok */ -static -bool -os_aio_array_validate( -/*==================*/ - os_aio_array_t* array) /*!< in: aio wait array */ -{ - ulint i; - ulint n_reserved = 0; - - os_mutex_enter(array->mutex); - - ut_a(array->n_slots > 0); - ut_a(array->n_segments > 0); - - for (i = 0; i < array->n_slots; i++) { - os_aio_slot_t* slot; - - slot = os_aio_array_get_nth_slot(array, i); - - if (slot->reserved) { - n_reserved++; - ut_a(slot->len > 0); - } - } - - ut_a(array->n_reserved == n_reserved); - - os_mutex_exit(array->mutex); - - return(true); -} - -/**********************************************************************//** -Validates the consistency the aio system. -@return TRUE if ok */ -UNIV_INTERN -ibool -os_aio_validate(void) -/*=================*/ -{ - os_aio_array_validate(os_aio_read_array); - - if (os_aio_write_array != 0) { - os_aio_array_validate(os_aio_write_array); - } - - if (os_aio_ibuf_array != 0) { - os_aio_array_validate(os_aio_ibuf_array); - } - - if (os_aio_log_array != 0) { - os_aio_array_validate(os_aio_log_array); - } - - if (os_aio_sync_array != 0) { - os_aio_array_validate(os_aio_sync_array); - } - - return(TRUE); -} - -/**********************************************************************//** -Prints pending IO requests per segment of an aio array. -We probably don't need per segment statistics but they can help us -during development phase to see if the IO requests are being -distributed as expected. */ -static -void -os_aio_print_segment_info( -/*======================*/ - FILE* file, /*!< in: file where to print */ - ulint* n_seg, /*!< in: pending IO array */ - os_aio_array_t* array) /*!< in: array to process */ -{ - ulint i; - - ut_ad(array); - ut_ad(n_seg); - ut_ad(array->n_segments > 0); - - if (array->n_segments == 1) { - return; - } - - fprintf(file, " ["); - for (i = 0; i < array->n_segments; i++) { - if (i != 0) { - fprintf(file, ", "); - } - - fprintf(file, "%lu", n_seg[i]); - } - fprintf(file, "] "); -} - -/**********************************************************************//** -Prints info about the aio array. */ -UNIV_INTERN -void -os_aio_print_array( -/*==============*/ - FILE* file, /*!< in: file where to print */ - os_aio_array_t* array) /*!< in: aio array to print */ -{ - ulint n_reserved = 0; - ulint n_res_seg[SRV_MAX_N_IO_THREADS]; - - os_mutex_enter(array->mutex); - - ut_a(array->n_slots > 0); - ut_a(array->n_segments > 0); - - memset(n_res_seg, 0x0, sizeof(n_res_seg)); - - for (ulint i = 0; i < array->n_slots; ++i) { - os_aio_slot_t* slot; - ulint seg_no; - - slot = os_aio_array_get_nth_slot(array, i); - - seg_no = (i * array->n_segments) / array->n_slots; - - if (slot->reserved) { - ++n_reserved; - ++n_res_seg[seg_no]; - - ut_a(slot->len > 0); - } - } - - ut_a(array->n_reserved == n_reserved); - - fprintf(file, " %lu", (ulong) n_reserved); - - os_aio_print_segment_info(file, n_res_seg, array); - - os_mutex_exit(array->mutex); -} - -/**********************************************************************//** -Prints info of the aio arrays. */ -UNIV_INTERN -void -os_aio_print( -/*=========*/ - FILE* file) /*!< in: file where to print */ -{ - time_t current_time; - double time_elapsed; - double avg_bytes_read; - - for (ulint i = 0; i < srv_n_file_io_threads; ++i) { - fprintf(file, "I/O thread %lu state: %s (%s)", - (ulong) i, - srv_io_thread_op_info[i], - srv_io_thread_function[i]); - -#ifndef __WIN__ - if (os_aio_segment_wait_events[i]->is_set) { - fprintf(file, " ev set"); - } -#endif /* __WIN__ */ - - fprintf(file, "\n"); - } - - fputs("Pending normal aio reads:", file); - - os_aio_print_array(file, os_aio_read_array); - - if (os_aio_write_array != 0) { - fputs(", aio writes:", file); - os_aio_print_array(file, os_aio_write_array); - } - - if (os_aio_ibuf_array != 0) { - fputs(",\n ibuf aio reads:", file); - os_aio_print_array(file, os_aio_ibuf_array); - } - - if (os_aio_log_array != 0) { - fputs(", log i/o's:", file); - os_aio_print_array(file, os_aio_log_array); - } - - if (os_aio_sync_array != 0) { - fputs(", sync i/o's:", file); - os_aio_print_array(file, os_aio_sync_array); - } - - putc('\n', file); - current_time = ut_time(); - time_elapsed = 0.001 + difftime(current_time, os_last_printout); - - fprintf(file, - "Pending flushes (fsync) log: %lu; buffer pool: %lu\n" - "%lu OS file reads, %lu OS file writes, %lu OS fsyncs\n", - (ulong) fil_n_pending_log_flushes, - (ulong) fil_n_pending_tablespace_flushes, - (ulong) os_n_file_reads, - (ulong) os_n_file_writes, - (ulong) os_n_fsyncs); - - if (os_file_n_pending_preads != 0 || os_file_n_pending_pwrites != 0) { - fprintf(file, - "%lu pending preads, %lu pending pwrites\n", - (ulong) os_file_n_pending_preads, - (ulong) os_file_n_pending_pwrites); - } - - if (os_n_file_reads == os_n_file_reads_old) { - avg_bytes_read = 0.0; - } else { - avg_bytes_read = (double) os_bytes_read_since_printout - / (os_n_file_reads - os_n_file_reads_old); - } - - fprintf(file, - "%.2f reads/s, %lu avg bytes/read," - " %.2f writes/s, %.2f fsyncs/s\n", - (os_n_file_reads - os_n_file_reads_old) - / time_elapsed, - (ulong) avg_bytes_read, - (os_n_file_writes - os_n_file_writes_old) - / time_elapsed, - (os_n_fsyncs - os_n_fsyncs_old) - / time_elapsed); - - os_n_file_reads_old = os_n_file_reads; - os_n_file_writes_old = os_n_file_writes; - os_n_fsyncs_old = os_n_fsyncs; - os_bytes_read_since_printout = 0; - - os_last_printout = current_time; -} - -/**********************************************************************//** -Refreshes the statistics used to print per-second averages. */ -UNIV_INTERN -void -os_aio_refresh_stats(void) -/*======================*/ -{ - os_n_file_reads_old = os_n_file_reads; - os_n_file_writes_old = os_n_file_writes; - os_n_fsyncs_old = os_n_fsyncs; - os_bytes_read_since_printout = 0; - - os_last_printout = time(NULL); -} - -#ifdef UNIV_DEBUG -/**********************************************************************//** -Checks that all slots in the system have been freed, that is, there are -no pending io operations. -@return TRUE if all free */ -UNIV_INTERN -ibool -os_aio_all_slots_free(void) -/*=======================*/ -{ - os_aio_array_t* array; - ulint n_res = 0; - - array = os_aio_read_array; - - os_mutex_enter(array->mutex); - - n_res += array->n_reserved; - - os_mutex_exit(array->mutex); - - if (!srv_read_only_mode) { - ut_a(os_aio_write_array == 0); - - array = os_aio_write_array; - - os_mutex_enter(array->mutex); - - n_res += array->n_reserved; - - os_mutex_exit(array->mutex); - - ut_a(os_aio_ibuf_array == 0); - - array = os_aio_ibuf_array; - - os_mutex_enter(array->mutex); - - n_res += array->n_reserved; - - os_mutex_exit(array->mutex); - } - - ut_a(os_aio_log_array == 0); - - array = os_aio_log_array; - - os_mutex_enter(array->mutex); - - n_res += array->n_reserved; - - os_mutex_exit(array->mutex); - - array = os_aio_sync_array; - - os_mutex_enter(array->mutex); - - n_res += array->n_reserved; - - os_mutex_exit(array->mutex); - - if (n_res == 0) { - - return(TRUE); - } - - return(FALSE); -} -#endif /* UNIV_DEBUG */ - -#endif /* !UNIV_HOTBACKUP */ - -#ifdef _WIN32 -#include -#ifndef FSCTL_FILE_LEVEL_TRIM -#define FSCTL_FILE_LEVEL_TRIM CTL_CODE(FILE_DEVICE_FILE_SYSTEM, 130, METHOD_BUFFERED, FILE_WRITE_DATA) -typedef struct _FILE_LEVEL_TRIM_RANGE { - DWORDLONG Offset; - DWORDLONG Length; -} FILE_LEVEL_TRIM_RANGE, *PFILE_LEVEL_TRIM_RANGE; - -typedef struct _FILE_LEVEL_TRIM { - DWORD Key; - DWORD NumRanges; - FILE_LEVEL_TRIM_RANGE Ranges[1]; -} FILE_LEVEL_TRIM, *PFILE_LEVEL_TRIM; -#endif -#endif - -/**********************************************************************//** -Directly manipulate the allocated disk space by deallocating for the file referred to -by fd for the byte range starting at offset and continuing for len bytes. -Within the specified range, partial file system blocks are zeroed, and whole -file system blocks are removed from the file. After a successful call, -subsequent reads from this range will return zeroes. -@return true if success, false if error */ -UNIV_INTERN -ibool -os_file_trim( -/*=========*/ - os_file_t file, /*!< in: file to be trimmed */ - os_aio_slot_t* slot, /*!< in: slot structure */ - ulint len) /*!< in: length of area */ -{ - -#define SECT_SIZE 512 - size_t trim_len = UNIV_PAGE_SIZE - len; - os_offset_t off = slot->offset + len; - // len here should be alligned to sector size - ut_a((trim_len % SECT_SIZE) == 0); - ut_a((len % SECT_SIZE) == 0); - - // Nothing to do if trim length is zero or if actual write - // size is initialized and it is smaller than current write size. - // In first write if we trim we set write_size to actual bytes - // written and rest of the page is trimmed. In following writes - // there is no need to trim again if write_size only increases - // because rest of the page is already trimmed. If actual write - // size decreases we need to trim again. - if (trim_len == 0 || - (slot->write_size && - *slot->write_size > 0 && - len >= *slot->write_size)) { - -#ifdef UNIV_PAGECOMPRESS_DEBUG - fprintf(stderr, "Note: TRIM: write_size %lu trim_len %lu len %lu\n", - *slot->write_size, trim_len, len); -#endif - - if (*slot->write_size > 0 && len >= *slot->write_size) { - srv_stats.page_compressed_trim_op_saved.inc(); - } - - *slot->write_size = len; - - return (TRUE); - } - -#ifdef __linux__ -#if defined(FALLOC_FL_PUNCH_HOLE) && defined (FALLOC_FL_KEEP_SIZE) - int ret = fallocate(file, FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE, off, trim_len); - - if (ret) { - /* After first failure do not try to trim again */ - os_fallocate_failed = TRUE; - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: [Warning] fallocate call failed with error code %d.\n" - " InnoDB: start: %lx len: %lu payload: %lu\n" - " InnoDB: Disabling fallocate for now.\n", ret, (slot->offset+len), trim_len, len); - - os_file_handle_error_no_exit(slot->name, - " fallocate(FALLOC_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE) ", - FALSE, __FILE__, __LINE__); - - if (slot->write_size) { - *slot->write_size = 0; - } - - return (FALSE); - } else { - if (slot->write_size) { - *slot->write_size = len; - } - } -#else - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: [Warning] fallocate not supported on this installation." - " InnoDB: Disabling fallocate for now."); - os_fallocate_failed = TRUE; - if (slot->write_size) { - *slot->write_size = 0; - } - -#endif /* HAVE_FALLOCATE ... */ - -#elif defined(_WIN32) - FILE_LEVEL_TRIM flt; - flt.Key = 0; - flt.NumRanges = 1; - flt.Ranges[0].Offset = off; - flt.Ranges[0].Length = trim_len; - - BOOL ret = DeviceIoControl(file,FSCTL_FILE_LEVEL_TRIM,&flt, sizeof(flt), NULL, NULL, NULL, NULL); - - if (!ret) { - /* After first failure do not try to trim again */ - os_fallocate_failed = TRUE; - ut_print_timestamp(stderr); - fprintf(stderr, - " InnoDB: [Warning] fallocate call failed with error.\n" - " InnoDB: start: %lx len: %du payload: %lu\n" - " InnoDB: Disabling fallocate for now.\n", (slot->offset+len), trim_len, len); - - os_file_handle_error_no_exit(slot->name, - " DeviceIOControl(FSCTL_FILE_LEVEL_TRIM) ", - FALSE, __FILE__, __LINE__); - - if (slot->write_size) { - *slot->write_size = 0; - } - return (FALSE); - } else { - if (slot->write_size) { - *slot->write_size = len; - } - } -#endif - - srv_stats.page_compression_trim_sect512.add((trim_len / SECT_SIZE)); - srv_stats.page_compression_trim_sect4096.add((trim_len / (SECT_SIZE*8))); - srv_stats.page_compressed_trim_op.inc(); - - return (TRUE); - -} - -/**********************************************************************//** -Allocate memory for temporal buffer used for page encryption. This -buffer is freed later. */ -UNIV_INTERN -void -os_slot_alloc_page_buf2( -/*===================*/ - os_aio_slot_t* slot) /*!< in: slot structure */ -{ - byte* cbuf2; - byte* cbuf; - - cbuf2 = static_cast(ut_malloc(UNIV_PAGE_SIZE*2)); - cbuf = static_cast(ut_align(cbuf2, UNIV_PAGE_SIZE)); - slot->page_encryption_page = static_cast(cbuf2); - slot->page_buf2 = static_cast(cbuf); -} - -/**********************************************************************//** -Allocate memory for temporal buffer used for page compression. This -buffer is freed later. */ -UNIV_INTERN -void -os_slot_alloc_page_buf( -/*===================*/ - os_aio_slot_t* slot) /*!< in: slot structure */ -{ - byte* cbuf2; - byte* cbuf; - - ut_a(slot != NULL); - /* We allocate extra to avoid memory overwrite on compression */ - cbuf2 = static_cast(ut_malloc(UNIV_PAGE_SIZE*2)); - cbuf = static_cast(ut_align(cbuf2, UNIV_PAGE_SIZE)); - slot->page_compression_page = static_cast(cbuf2); - slot->page_buf = static_cast(cbuf); - ut_a(slot->page_buf != NULL); -} - -#ifdef HAVE_LZO -/**********************************************************************//** -Allocate memory for temporal memory used for page compression when -LZO compression method is used */ -UNIV_INTERN -void -os_slot_alloc_lzo_mem( -/*===================*/ - os_aio_slot_t* slot) /*!< in: slot structure */ -{ - ut_a(slot != NULL); - slot->lzo_mem = static_cast(ut_malloc(LZO1X_1_15_MEM_COMPRESS)); - ut_a(slot->lzo_mem != NULL); -} -#endif - From 9fd4585153dc6d336e6aa207777f9c64f42404db Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Clemens=20D=C3=B6rrh=C3=B6fer?= Date: Fri, 10 Oct 2014 15:18:58 +0200 Subject: [PATCH 38/70] printKeyEntry will no longer reveal key when not compiled in debug mode. --- storage/xtradb/enc/EncKeys.cc | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/storage/xtradb/enc/EncKeys.cc b/storage/xtradb/enc/EncKeys.cc index 361c3b77d4823..7a487bcf518df 100644 --- a/storage/xtradb/enc/EncKeys.cc +++ b/storage/xtradb/enc/EncKeys.cc @@ -336,6 +336,14 @@ bool EncKeys::isComment(const char *line) { void EncKeys::printKeyEntry( ulint id) { keyentry *entry = getKeys(id); - if( NULL == entry) fprintf(stderr, "No such keyID = %u\n", id); - else fprintf(stderr, "Key: id:%3u \tiv:%s \tkey:%s\n", entry->id, entry->iv, entry->key); + if( NULL == entry) { + fprintf(stderr, "No such keyID = %u\n", id); + } + else { +#ifdef UNIV_DEBUG + fprintf(stderr, "Key: id:%3u \tiv:%s \tkey:%s\n", entry->id, entry->iv, entry->key); +#else + fprintf(stderr, "Key: id:%3u\n", entry->id); +#endif //UNIV_DEBUG + } } From 8380b82d416a276c56c46ce27a635dee6bdc1af0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludger=20G=C3=B6ckel?= Date: Fri, 10 Oct 2014 16:11:46 +0200 Subject: [PATCH 39/70] improved key file handling --- storage/xtradb/enc/EncKeys.cc | 2 +- storage/xtradb/fil/fil0fil.cc | 4 +--- storage/xtradb/fil/fil0pageencryption.cc | 12 +++++++++++- storage/xtradb/handler/ha_innodb.cc | 19 +++++++++++++++++++ storage/xtradb/include/KeySingleton.h | 4 ++++ storage/xtradb/os/os0file.cc | 8 +++++++- 6 files changed, 43 insertions(+), 6 deletions(-) diff --git a/storage/xtradb/enc/EncKeys.cc b/storage/xtradb/enc/EncKeys.cc index 3d1f3eaea14ee..5266d97f45b94 100644 --- a/storage/xtradb/enc/EncKeys.cc +++ b/storage/xtradb/enc/EncKeys.cc @@ -78,7 +78,7 @@ EncKeys::~EncKeys() { bool EncKeys::initKeys(const char *name, const char *url, const int initType, const char *filekey) { if (KEYINITTYPE_FILE == initType) { // url == path && name == filename int result = initKeysThroughFile(name, url, filekey); - return ERROR_FALSE_FILE_KEY != result; + return ERROR_FALSE_FILE_KEY != result && ERROR_OPEN_FILE != result && ERROR_READING_FILE != result; } else if (KEYINITTYPE_SERVER == initType) { fprintf(stderr, errorNotImplemented); diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index e8a59622b67d0..e1290638e7519 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -2147,9 +2147,7 @@ fil_read_first_page( /* Align the memory for a possible read from a raw device */ page = static_cast(ut_align(buf, UNIV_PAGE_SIZE)); -if (orig_space_id==18446744073709551615) { - //return NULL; -} + os_file_read(data_file, page, 0, UNIV_PAGE_SIZE, orig_space_id != ULINT_UNDEFINED ? fil_space_is_page_compressed(orig_space_id) : diff --git a/storage/xtradb/fil/fil0pageencryption.cc b/storage/xtradb/fil/fil0pageencryption.cc index bcc7aad492656..1a9d069b27fde 100644 --- a/storage/xtradb/fil/fil0pageencryption.cc +++ b/storage/xtradb/fil/fil0pageencryption.cc @@ -129,6 +129,8 @@ ulint mode uint8 key_len = 16; if (!unit_test) { KeySingleton& keys = KeySingleton::getInstance(); + if (!keys.isAvailable()) return buf; + if (keys.getKeys(encryption_key) == NULL) return buf; char* keyString = keys.getKeys(encryption_key)->key; key_len = strlen(keyString)/2; my_aes_hexToUint(keyString, (unsigned char*)&rkey, key_len); @@ -137,7 +139,11 @@ ulint mode 0x5a, 0x6f, 0x82, 0x59, 0x4f,0x5e}; if (!unit_test) { KeySingleton& keys = KeySingleton::getInstance(); - my_aes_hexToUint(keys.getKeys(encryption_key)->iv, (unsigned char*)&iv, 16); + if (!keys.isAvailable()) return buf; + if (keys.getKeys(encryption_key) == NULL) return buf; + char* ivString = keys.getKeys(encryption_key)->iv; + if (ivString == NULL) return buf; + my_aes_hexToUint(ivString, (unsigned char*)&iv, 16); } uint8 iv_len = 16; write_size = data_size; @@ -415,6 +421,8 @@ al_size); uint8 key_len = 16; if (!unit_test) { KeySingleton& keys = KeySingleton::getInstance(); + if (!keys.isAvailable()) return PAGE_ENCRYPTION_ERROR; + if (keys.getKeys(page_encryption_key) == NULL) return PAGE_ENCRYPTION_ERROR; char* keyString = keys.getKeys(page_encryption_key)->key; key_len = strlen(keyString)/2; my_aes_hexToUint(keyString, (unsigned char*)&rkey, key_len); @@ -425,6 +433,8 @@ al_size); 0x5a, 0x6f, 0x82, 0x59, 0x4f,0x5e}; if (!unit_test) { KeySingleton& keys = KeySingleton::getInstance(); + if (!keys.isAvailable()) return PAGE_ENCRYPTION_ERROR; + if (keys.getKeys(page_encryption_key) == NULL) return PAGE_ENCRYPTION_ERROR; my_aes_hexToUint(keys.getKeys(page_encryption_key)->iv, (unsigned char*)&iv, 16); } uint8 iv_len = 16; diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 941b56f6b20f3..e5638e5fec938 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -11646,6 +11646,14 @@ ha_innobase::check_table_options( " innodb_file_per_table."); return "PAGE_ENCRYPTION"; } + if (!KeySingleton::getInstance().isAvailable()) { + push_warning( + thd, Sql_condition::WARN_LEVEL_WARN, + HA_WRONG_CREATE_OPTION, + "InnoDB: PAGE_ENCRYPTION needs a key provider" + ); + return "PAGE_ENCRYPTION"; + } if (create_info->key_block_size) { push_warning( thd, Sql_condition::WARN_LEVEL_WARN, @@ -11738,6 +11746,17 @@ ha_innobase::check_table_options( options->page_encryption_key); return "PAGE_ENCRYPTION_KEY"; } + + if (!KeySingleton::getInstance().isAvailable() || KeySingleton::getInstance().getKeys(options->page_encryption_key)==NULL) { + push_warning_printf( + thd, Sql_condition::WARN_LEVEL_WARN, + HA_WRONG_CREATE_OPTION, + "InnoDB: PAGE_ENCRYPTION_KEY encryption key %lu not available", + options->page_encryption_key + ); + return "PAGE_ENCRYPTION_KEY"; + + } } diff --git a/storage/xtradb/include/KeySingleton.h b/storage/xtradb/include/KeySingleton.h index 667ba2d5be601..7dba405463b6b 100644 --- a/storage/xtradb/include/KeySingleton.h +++ b/storage/xtradb/include/KeySingleton.h @@ -50,6 +50,10 @@ class KeySingleton static KeySingleton& getInstance(const char *name, const char *url, const int initType, const char *filekey); keyentry *getKeys(int id); + + static bool isAvailable() { + return instanceInited; + } }; #endif /* KEYSINGLETON_H_ */ diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index 63677671125be..bc648a03bf201 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -4847,6 +4847,7 @@ os_aio_array_reserve_slot( slot->page_encryption_success = TRUE; } else { slot->page_encryption_success = FALSE; + ut_error; } /* Take array mutex back */ @@ -4880,7 +4881,11 @@ os_aio_array_reserve_slot( io_prep_pread(iocb, file, buf, len, aio_offset); } else { ut_a(type == OS_FILE_WRITE); - io_prep_pwrite(iocb, file, buf, len, aio_offset); + if (page_encryption && !slot->page_encryption_success) { + ut_error; + } else { + io_prep_pwrite(iocb, file, buf, len, aio_offset); + } } iocb->data = (void*) slot; @@ -5265,6 +5270,7 @@ os_aio_func( os_n_file_writes++; #ifdef WIN_ASYNC_IO if (page_encryption) { + if (!slot->page_encryption_success) goto err_exit; buffer = slot->page_buf2; } else { buffer = buf; From e673d003b157cf9626837d1a9ac524dff7de070c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Clemens=20D=C3=B6rrh=C3=B6fer?= Date: Tue, 14 Oct 2014 11:06:46 +0200 Subject: [PATCH 40/70] Singleton can now tell you if a key is present or not. --- storage/xtradb/enc/KeySingleton.cc | 4 ++++ storage/xtradb/include/KeySingleton.h | 1 + storage/xtradb/os/os0file.cc | 1 + 3 files changed, 6 insertions(+) diff --git a/storage/xtradb/enc/KeySingleton.cc b/storage/xtradb/enc/KeySingleton.cc index aea666137cce9..461ee5b892733 100644 --- a/storage/xtradb/enc/KeySingleton.cc +++ b/storage/xtradb/enc/KeySingleton.cc @@ -56,3 +56,7 @@ keyentry *KeySingleton::getKeys(int id) { return encKeys.getKeys(id); } +ibool KeySingleton::hasKey(int id) { + return encKeys.getKeys(id) != NULL; +} + diff --git a/storage/xtradb/include/KeySingleton.h b/storage/xtradb/include/KeySingleton.h index 667ba2d5be601..121b27db367a9 100644 --- a/storage/xtradb/include/KeySingleton.h +++ b/storage/xtradb/include/KeySingleton.h @@ -50,6 +50,7 @@ class KeySingleton static KeySingleton& getInstance(const char *name, const char *url, const int initType, const char *filekey); keyentry *getKeys(int id); + ibool hasKey(int id); }; #endif /* KEYSINGLETON_H_ */ diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index 957bc60d48ff5..e73829374e9a6 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -5164,6 +5164,7 @@ os_aio_func( ut_ad((n & 0xFFFFFFFFUL) == n); #endif + wake_later = mode & OS_AIO_SIMULATED_WAKE_LATER; mode = mode & (~OS_AIO_SIMULATED_WAKE_LATER); From 5c946ffaee5767c41f1739426878c1ba0f42fc71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Clemens=20D=C3=B6rrh=C3=B6fer?= Date: Tue, 14 Oct 2014 14:26:09 +0200 Subject: [PATCH 41/70] Simplified call to extract key --- storage/xtradb/fil/fil0pageencryption.cc | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/storage/xtradb/fil/fil0pageencryption.cc b/storage/xtradb/fil/fil0pageencryption.cc index d89c344a4b753..e3243f3ec66ce 100644 --- a/storage/xtradb/fil/fil0pageencryption.cc +++ b/storage/xtradb/fil/fil0pageencryption.cc @@ -344,8 +344,7 @@ ulint mode /* Get encryption key. it was not successful here to read the page encryption key from the tablespace flags, because * of unresolved deadlocks while accessing the tablespace memory structure.*/ - page_decryption_key = mach_read_from_1( - buf + page_size - FIL_PAGE_DATA_END - offset_ctrl_data); + page_decryption_key = mach_read_from_1(buf); /* Get the page type */ From 340dfd91868052c54f954bda920a541e4e0a48c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludger=20G=C3=B6ckel?= Date: Tue, 14 Oct 2014 14:37:20 +0200 Subject: [PATCH 42/70] some error handling if key file n.a. --- storage/xtradb/fil/fil0fil.cc | 55 ++++++++++++++++++++++++++++---- storage/xtradb/include/fil0fil.h | 2 ++ 2 files changed, 51 insertions(+), 6 deletions(-) diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index e1290638e7519..fcca1a54a439c 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -26,6 +26,7 @@ Created 10/25/1995 Heikki Tuuri #include "fil0fil.h" +#include "KeySingleton.h" #include #include @@ -820,8 +821,17 @@ fil_node_open_file( success = os_file_read(node->handle, page, 0, UNIV_PAGE_SIZE, space->flags); + if (fil_page_is_encrypted(page)) { + fprintf(stderr, + "InnoDB: can not decrypt %s\n", + node->name); + return false; + + } + space_id = fsp_header_get_space_id(page); flags = fsp_header_get_flags(page); + page_size = fsp_flags_get_page_size(flags); atomic_writes = fsp_flags_get_atomic_writes(flags); @@ -1336,6 +1346,16 @@ fil_space_create( ut_a(fil_system); + if (fsp_flags_is_page_encrypted(flags)) { + if (!KeySingleton::getInstance().isAvailable() || KeySingleton::getInstance().getKeys(fsp_flags_get_page_encryption_key(flags))==NULL) { + + ib_logf(IB_LOG_LEVEL_WARN, + "Tablespace '%s' can not be opened, because encryption key can not be found (space id: %lu, key %lu)\n" + , name, (ulong) id, fsp_flags_get_page_encryption_key(flags)); + return (FALSE); + } + } + /* Look for a matching tablespace and if found free it. */ do { mutex_enter(&fil_system->mutex); @@ -2074,6 +2094,7 @@ fil_check_first_page( { ulint space_id; ulint flags; + ulint cc = 0; if (srv_force_recovery >= SRV_FORCE_IGNORE_CORRUPT) { return(NULL); @@ -2081,12 +2102,17 @@ fil_check_first_page( space_id = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_ID + page); flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); + cc = fil_page_is_encrypted(page); + if (!KeySingleton::getInstance().isAvailable() && cc) { + cc = 1; + } else { + cc = 0; + if (UNIV_PAGE_SIZE != fsp_flags_get_page_size(flags)) { + fprintf(stderr, "InnoDB: Error: Current page size %lu != page size on page %lu\n", + UNIV_PAGE_SIZE, fsp_flags_get_page_size(flags)); - if (UNIV_PAGE_SIZE != fsp_flags_get_page_size(flags)) { - fprintf(stderr, "InnoDB: Error: Current page size %lu != page size on page %lu\n", - UNIV_PAGE_SIZE, fsp_flags_get_page_size(flags)); - - return("innodb-page-size mismatch"); + return("innodb-page-size mismatch"); + } } if (!space_id && !flags) { @@ -2102,9 +2128,14 @@ fil_check_first_page( } } - if (buf_page_is_corrupted( + if (!cc && buf_page_is_corrupted( false, page, fsp_flags_get_zip_size(flags))) { return("checksum mismatch"); + } else { + if (cc) { + return("can not decrypt"); + + } } if (page_get_space_id(page) == space_id @@ -4296,6 +4327,7 @@ fil_validate_single_table_tablespace( check_first_page: fsp->success = TRUE; + fsp->encryption_error = 0; if (const char* check_msg = fil_read_first_page( fsp->file, FALSE, &fsp->flags, &fsp->id, &fsp->lsn, &fsp->lsn, ULINT_UNDEFINED)) { @@ -4303,6 +4335,10 @@ fil_validate_single_table_tablespace( "%s in tablespace %s (table %s)", check_msg, fsp->filepath, tablename); fsp->success = FALSE; + if (strncmp(check_msg, "can not decrypt", strlen(check_msg))==0) { + fsp->encryption_error = 1; + return; + } } if (!fsp->success) { @@ -4445,6 +4481,13 @@ fil_load_single_table_tablespace( } if (!def.success && !remote.success) { + + if (def.encryption_error || remote.encryption_error) { + fprintf(stderr, + "InnoDB: Error: could not open single-table" + " tablespace file %s. Encryption error!\n", def.filepath); + return; + } /* The following call prints an error message */ os_file_get_last_error(true); fprintf(stderr, diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index 688e25957f8ff..ba9852fdaf462 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -223,6 +223,8 @@ struct fsp_open_info { lsn_t lsn; /*!< Flushed LSN from header page */ ulint id; /*!< Space ID */ ulint flags; /*!< Tablespace flags */ + ulint encryption_error; /*!< if an encryption error occurs */ + }; #ifndef UNIV_HOTBACKUP From 282199d14432ab9a10fa244e4b33c4e2e8c2effe Mon Sep 17 00:00:00 2001 From: unknown Date: Tue, 14 Oct 2014 15:58:02 +0200 Subject: [PATCH 43/70] made alter table working... --- storage/xtradb/handler/ha_innodb.cc | 7 +------ storage/xtradb/handler/handler0alter.cc | 7 +++++++ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index e5638e5fec938..01fbdff8137c6 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -11729,12 +11729,7 @@ ha_innobase::check_table_options( if ((ulint)options->page_encryption_key != ULINT_UNDEFINED) { if (options->page_encryption == false) { - push_warning( - thd, Sql_condition::WARN_LEVEL_WARN, - HA_WRONG_CREATE_OPTION, - "InnoDB: PAGE_ENCRYPTION_KEY requires" - " PAGE_ENCRYPTION"); - return "PAGE_ENCRYPTION_KEY"; + /* ignore for alter table...*/ } if (options->page_encryption_key < 1 || options->page_encryption_key > 255) { diff --git a/storage/xtradb/handler/handler0alter.cc b/storage/xtradb/handler/handler0alter.cc index 8097fd01e3f1e..128cb78d80202 100644 --- a/storage/xtradb/handler/handler0alter.cc +++ b/storage/xtradb/handler/handler0alter.cc @@ -280,6 +280,13 @@ ha_innobase::check_if_supported_inplace_alter( ER_ALTER_OPERATION_NOT_SUPPORTED_REASON); DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); } + + if (new_options->page_encryption != old_options->page_encryption || + new_options->page_encryption_key != old_options->page_encryption_key) { + ha_alter_info->unsupported_reason = innobase_get_err_msg( + ER_ALTER_OPERATION_NOT_SUPPORTED_REASON); + DBUG_RETURN(HA_ALTER_INPLACE_NOT_SUPPORTED); + } } if (ha_alter_info->handler_flags From ac561dbcc5c48206adc7dbee8b2437c899267eaa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Clemens=20D=C3=B6rrh=C3=B6fer?= Date: Tue, 14 Oct 2014 16:08:56 +0200 Subject: [PATCH 44/70] Code cleanup - simplification Removed unnecessary code Simplified byte calculations Using constants Removed a number of variables. Got rid of code which had no effect. --- include/my_aes.h | 2 + mysys_ssl/my_aes.cc | 2 - storage/xtradb/fil/fil0pageencryption.cc | 136 ++++++++--------------- 3 files changed, 47 insertions(+), 93 deletions(-) diff --git a/include/my_aes.h b/include/my_aes.h index c628bfa21ab82..2d49afa648771 100644 --- a/include/my_aes.h +++ b/include/my_aes.h @@ -6,6 +6,8 @@ #define AES_BAD_KEYSIZE -5 #define AES_KEY_CREATION_FAILED -10 +#define MY_AES_BLOCK_SIZE 16 /* Block size in bytes */ + /* Copyright (c) 2002, 2006 MySQL AB, 2009 Sun Microsystems, Inc. Use is subject to license terms. diff --git a/mysys_ssl/my_aes.cc b/mysys_ssl/my_aes.cc index 7e6b48e0bb4db..eb245b2a43829 100644 --- a/mysys_ssl/my_aes.cc +++ b/mysys_ssl/my_aes.cc @@ -39,8 +39,6 @@ struct MyCipherCtx enum encrypt_dir { MY_AES_ENCRYPT, MY_AES_DECRYPT }; -#define MY_AES_BLOCK_SIZE 16 /* Block size in bytes */ - /* If bad data discovered during decoding */ #define AES_BAD_DATA -1 diff --git a/storage/xtradb/fil/fil0pageencryption.cc b/storage/xtradb/fil/fil0pageencryption.cc index e3243f3ec66ce..a4a6f432ddf09 100644 --- a/storage/xtradb/fil/fil0pageencryption.cc +++ b/storage/xtradb/fil/fil0pageencryption.cc @@ -66,14 +66,11 @@ ulint mode int err = AES_OK; int key = 0; - ulint header_len = FIL_PAGE_DATA; ulint remainder = 0; ulint data_size = 0; - ulint page_size = UNIV_PAGE_SIZE; ulint orig_page_type = 0; ulint write_size = 0; ib_uint32_t checksum = 0; - ulint offset_ctrl_data = 0; fil_space_t* space = NULL; byte* tmp_buf = NULL; ulint unit_test = 0; @@ -101,10 +98,10 @@ ulint mode /* data_size -1 bytes will be encrypted at first. * data_size is the length of the cipher text.*/ - data_size = ((page_size - header_len - FIL_PAGE_DATA_END) / 16) * 16; + data_size = ((UNIV_PAGE_SIZE - FIL_PAGE_DATA - FIL_PAGE_DATA_END) / MY_AES_BLOCK_SIZE) * MY_AES_BLOCK_SIZE; /* following number of bytes are not encrypted at first */ - remainder = (page_size - header_len - FIL_PAGE_DATA_END) - (data_size - 1); + remainder = (UNIV_PAGE_SIZE - FIL_PAGE_DATA - FIL_PAGE_DATA_END) - (data_size - 1); /* read original page type */ orig_page_type = mach_read_from_2(buf + FIL_PAGE_TYPE); @@ -114,7 +111,7 @@ ulint mode } /* calculate a checksum, can be used to verify decryption */ - checksum = fil_page_encryption_calc_checksum(buf + header_len, page_size - (FIL_PAGE_DATA_END + header_len)); + checksum = fil_page_encryption_calc_checksum(buf + FIL_PAGE_DATA, UNIV_PAGE_SIZE - (FIL_PAGE_DATA_END + FIL_PAGE_DATA)); const unsigned char rkey[] = {0xbd, 0xe4, 0x72, 0xa2, 0x95, 0x67, 0x5c, 0xa9, 0x2e, 0x04, 0x67, 0xea, 0xdb, 0xc0, 0xe0, 0x23, @@ -140,8 +137,8 @@ ulint mode write_size = data_size; if (err == AES_OK) { /* 1st encryption: data_size -1 bytes starting from FIL_PAGE_DATA */ - err = my_aes_encrypt_cbc((char*) buf + header_len, data_size - 1, - (char *) out_buf + header_len, &write_size, + err = my_aes_encrypt_cbc((char*) buf + FIL_PAGE_DATA, data_size - 1, + (char *) out_buf + FIL_PAGE_DATA, &write_size, (const unsigned char *) &rkey, key_len, (const unsigned char *) &iv, iv_len); ; ut_ad(write_size == data_size); @@ -155,16 +152,16 @@ ulint mode } /* copy remaining bytes to output buffer */ - memcpy(out_buf + header_len + data_size, buf + header_len + data_size - 1, + memcpy(out_buf + FIL_PAGE_DATA + data_size, buf + FIL_PAGE_DATA + data_size - 1, remainder - offset); if (page_compressed && err == AES_OK) { - remaining_byte = mach_read_from_1(buf + header_len + data_size +1); + remaining_byte = mach_read_from_1(buf + FIL_PAGE_DATA + data_size +1); } else { //create temporary buffer for 2nd encryption tmp_buf = static_cast(ut_malloc(64)); /* 2nd encryption: 63 bytes from out_buf, result length is 64 bytes */ - err = my_aes_encrypt_cbc((char*)out_buf + page_size -FIL_PAGE_DATA_END -62, + err = my_aes_encrypt_cbc((char*)out_buf + UNIV_PAGE_SIZE -FIL_PAGE_DATA_END -62, 63, (char*)tmp_buf, &write_size, @@ -173,9 +170,9 @@ ulint mode (const unsigned char *)&iv, iv_len); ut_ad(write_size == 64); - //AES_cbc_encrypt((uchar*)out_buf + page_size -FIL_PAGE_DATA_END -62, tmp_buf, 63, &aeskey, iv, AES_ENCRYPT); + //AES_cbc_encrypt((uchar*)out_buf + UNIV_PAGE_SIZE -FIL_PAGE_DATA_END -62, tmp_buf, 63, &aeskey, iv, AES_ENCRYPT); /* copy 62 bytes from 2nd encryption to out_buf, last 2 bytes are copied later to a header field*/ - memcpy(out_buf + page_size - FIL_PAGE_DATA_END -62, tmp_buf, 62); + memcpy(out_buf + UNIV_PAGE_SIZE - FIL_PAGE_DATA_END -62, tmp_buf, 62); } /* error handling */ @@ -198,8 +195,8 @@ ulint mode } /* set up the trailer.*/ - memcpy(out_buf + (page_size -FIL_PAGE_DATA_END), - buf + (page_size - FIL_PAGE_DATA_END), FIL_PAGE_DATA_END); + memcpy(out_buf + (UNIV_PAGE_SIZE -FIL_PAGE_DATA_END), + buf + (UNIV_PAGE_SIZE - FIL_PAGE_DATA_END), FIL_PAGE_DATA_END); /* Set up the page header. Copied from input buffer*/ @@ -210,28 +207,25 @@ ulint mode /* checksum */ if (!page_compressed) { /* Set up the checksum. This is only usable to verify decryption */ - mach_write_to_3(out_buf + page_size - FIL_PAGE_DATA_END, checksum); + mach_write_to_3(out_buf + UNIV_PAGE_SIZE - FIL_PAGE_DATA_END, checksum); } else { - ulint pos_checksum = page_size - FIL_PAGE_DATA_END; + ulint pos_checksum = UNIV_PAGE_SIZE - FIL_PAGE_DATA_END; if (compressed_size + FIL_PAGE_DATA > pos_checksum) { pos_checksum = compressed_size + FIL_PAGE_DATA; - if (pos_checksum > page_size - 3) { + if (pos_checksum > UNIV_PAGE_SIZE - 3) { // checksum not supported, because no space available } else { /* Set up the checksum. This is only usable to verify decryption */ mach_write_to_3(out_buf + pos_checksum, checksum); } } else { - mach_write_to_3(out_buf + page_size - FIL_PAGE_DATA_END, checksum); + mach_write_to_3(out_buf + UNIV_PAGE_SIZE - FIL_PAGE_DATA_END, checksum); } } /* Set up the correct page type */ mach_write_to_2(out_buf + FIL_PAGE_TYPE, FIL_PAGE_PAGE_ENCRYPTED); - - offset_ctrl_data = page_size - FIL_PAGE_DATA_END; - /* checksum fields are used to store original page type, etc. * checksum check for page encrypted pages is omitted. PAGE_COMPRESSED pages does not seem to have a * Old-style checksum trailer, therefore this field is only used, if there is space. Payload length is expected as @@ -239,23 +233,18 @@ ulint mode /* Set up the encryption key. Written to the 1st byte of the checksum header field. This header is currently used to store data. */ - mach_write_to_1(out_buf + page_size - FIL_PAGE_DATA_END - offset_ctrl_data, - key); + mach_write_to_1(out_buf, key); /* store original page type. Written to 2nd and 3rd byte of the checksum header field */ - mach_write_to_2( - out_buf + page_size - FIL_PAGE_DATA_END + 1 - offset_ctrl_data, - orig_page_type); + mach_write_to_2(out_buf + 1, orig_page_type); /* write remaining bytes to checksum header byte 4 and old style checksum byte 4 */ if (!page_compressed) { - memcpy(out_buf+ page_size - FIL_PAGE_DATA_END + 3 - offset_ctrl_data, - tmp_buf + 62, 1); - memcpy(out_buf + page_size - FIL_PAGE_DATA_END +3 , tmp_buf + 63, 1); + memcpy(out_buf + 3, tmp_buf + 62, 1); + memcpy(out_buf + UNIV_PAGE_SIZE - FIL_PAGE_DATA_END +3 , tmp_buf + 63, 1); } else { /* if page is compressed, only one byte must be placed */ - memset(out_buf+ page_size - FIL_PAGE_DATA_END + 3 - offset_ctrl_data, - remaining_byte, 1); + memset(out_buf + 3, remaining_byte, 1); } #ifdef UNIV_DEBUG @@ -265,7 +254,7 @@ ulint mode #endif /* UNIV_DEBUG */ srv_stats.pages_page_encrypted.inc(); - *out_len = page_size; + *out_len = UNIV_PAGE_SIZE; /* free temporary buffer */ if (tmp_buf!=NULL) { @@ -290,15 +279,9 @@ ulint mode ) { int err = AES_OK; ulint page_decryption_key; - ulint data_size = 0; - - ulint page_size = UNIV_PAGE_SIZE; ulint orig_page_type = 0; - - ulint header_len = FIL_PAGE_DATA; ulint remainder = 0; - ulint offset_ctrl_data = 0; ulint checksum = 0; ulint stored_checksum = 0; ulint tmp_write_size = 0; @@ -306,15 +289,9 @@ ulint mode byte * in_buf; byte * tmp_buf; byte * tmp_page_buf; - ulint flags = 0; - fil_space_t* space = NULL; - ulint page_num = 0; - ulint page_encrypted = 0; ulint page_compression_flag = 0; ulint unit_test = mode & 0x01; - ulint space_id = 0; - ut_ad(buf); ut_ad(len); @@ -332,15 +309,12 @@ ulint mode return PAGE_ENCRYPTION_WRONG_PAGE_TYPE; } - space_id = mach_read_from_4(buf + FIL_PAGE_SPACE_ID); - /* checksum fields are used to store original page type, etc. * checksum check for page encrypted pages is omitted. PAGE_COMPRESSED pages does not seem to have a * Old-style checksum trailer, therefore this field is only used, if there is space. Payload is expected as * two byte value at position FIL_PAGE_DATA */ - offset_ctrl_data = page_size - FIL_PAGE_DATA_END; /* Get encryption key. it was not successful here to read the page encryption key from the tablespace flags, because * of unresolved deadlocks while accessing the tablespace memory structure.*/ @@ -348,8 +322,7 @@ ulint mode /* Get the page type */ - orig_page_type = mach_read_from_2( - buf + page_size - FIL_PAGE_DATA_END + 1 - offset_ctrl_data); + orig_page_type = mach_read_from_3(buf); if (FIL_PAGE_PAGE_COMPRESSED == orig_page_type) { if (page_compressed != NULL) { @@ -371,13 +344,12 @@ ulint mode } else { in_buf = page_buf; } - - data_size = ((page_size - header_len - FIL_PAGE_DATA_END) / 16) * 16; - remainder = (page_size - header_len - FIL_PAGE_DATA_END) - (data_size - 1); + data_size = ((UNIV_PAGE_SIZE - FIL_PAGE_DATA - FIL_PAGE_DATA_END) / MY_AES_BLOCK_SIZE) * MY_AES_BLOCK_SIZE; + remainder = (UNIV_PAGE_SIZE - FIL_PAGE_DATA - FIL_PAGE_DATA_END) - (data_size - 1); tmp_buf= static_cast(ut_malloc(64)); - tmp_page_buf = static_cast(ut_malloc(page_size)); - memset(tmp_page_buf,0, page_size); + tmp_page_buf = static_cast(ut_malloc(UNIV_PAGE_SIZE)); + memset(tmp_page_buf,0, UNIV_PAGE_SIZE); const unsigned char rkey[] = { 0xbd, 0xe4, 0x72, 0xa2, 0x95, 0x67, 0x5c, 0xa9, 0x2e, 0x04, 0x67, 0xea, 0xdb, 0xc0, 0xe0, 0x23, 0x00, 0x00, @@ -401,20 +373,20 @@ ulint mode if (!page_compression_flag) { - tmp_page_buf = static_cast(ut_malloc(page_size)); + tmp_page_buf = static_cast(ut_malloc(UNIV_PAGE_SIZE)); tmp_buf= static_cast(ut_malloc(64)); - memset(tmp_page_buf,0, page_size); + memset(tmp_page_buf,0, UNIV_PAGE_SIZE); /* 1st decryption: 64 bytes */ /* 62 bytes from data area and 2 bytes from header are copied to temporary buffer */ - memcpy(tmp_buf, buf + page_size - FIL_PAGE_DATA_END -62, 62); + memcpy(tmp_buf, buf + UNIV_PAGE_SIZE - FIL_PAGE_DATA_END - 62, 62); memcpy(tmp_buf + 62, buf + FIL_PAGE_SPACE_OR_CHKSUM + 3, 1); - memcpy(tmp_buf + 63, buf + page_size - FIL_PAGE_DATA_END +3, 1); + memcpy(tmp_buf + 63, buf + UNIV_PAGE_SIZE - FIL_PAGE_DATA_END +3, 1); if (err == AES_OK) { err = my_aes_decrypt_cbc((const char*) tmp_buf, 64, - (char *) tmp_page_buf + page_size - FIL_PAGE_DATA_END - 62, + (char *) tmp_page_buf + UNIV_PAGE_SIZE - FIL_PAGE_DATA_END - 62, &tmp_write_size, (const unsigned char *) &rkey, key_len, (const unsigned char *) &iv, iv_len); } @@ -437,7 +409,7 @@ ulint mode /* copy 1st part from buf to tmp_page_buf */ /* do not override result of 1st decryption */ memcpy(tmp_page_buf + FIL_PAGE_DATA, buf + FIL_PAGE_DATA, data_size - 60); - memset(in_buf, 0, page_size); + memset(in_buf, 0, UNIV_PAGE_SIZE); @@ -456,7 +428,7 @@ ulint mode ); ut_ad(tmp_write_size = data_size-1); - memcpy(in_buf + FIL_PAGE_DATA + data_size -1, tmp_page_buf + page_size - FIL_PAGE_DATA_END - 2, remainder - offset); + memcpy(in_buf + FIL_PAGE_DATA + data_size -1, tmp_page_buf + UNIV_PAGE_SIZE - FIL_PAGE_DATA_END - 2, remainder - offset); if (page_compression_flag) { /* the last byte was stored in position 4 of the checksum header */ memcpy(in_buf + FIL_PAGE_DATA + data_size -1 + 2, tmp_page_buf+ FIL_PAGE_SPACE_OR_CHKSUM + 3, 1); @@ -464,19 +436,19 @@ ulint mode /* calculate a checksum to verify decryption*/ - checksum = fil_page_encryption_calc_checksum(in_buf + header_len, page_size - (FIL_PAGE_DATA_END + header_len) ); + checksum = fil_page_encryption_calc_checksum(in_buf + FIL_PAGE_DATA, UNIV_PAGE_SIZE - (FIL_PAGE_DATA_END + FIL_PAGE_DATA) ); /* compare with stored checksum */ ulint compressed_size = mach_read_from_2(in_buf+ FIL_PAGE_DATA); ibool no_checksum_support = 0; - ulint pos_checksum; + ulint pos_checksum = 0; if (!page_compression_flag) { /* Read the checksum. This is only usable to verify decryption */ - stored_checksum = mach_read_from_3(buf + page_size - FIL_PAGE_DATA_END); + stored_checksum = mach_read_from_3(buf + UNIV_PAGE_SIZE - FIL_PAGE_DATA_END); } else { - pos_checksum = page_size - FIL_PAGE_DATA_END; + pos_checksum = UNIV_PAGE_SIZE - FIL_PAGE_DATA_END; if (compressed_size + FIL_PAGE_DATA > pos_checksum) { pos_checksum = compressed_size + FIL_PAGE_DATA; - if (pos_checksum > page_size - 3) { + if (pos_checksum > UNIV_PAGE_SIZE - 3) { // checksum not supported, because no space available no_checksum_support = 1; } else { @@ -485,7 +457,7 @@ ulint mode } } else { /* Read the checksum. This is only usable to verify decryption */ - stored_checksum = mach_read_from_3(buf + page_size - FIL_PAGE_DATA_END); + stored_checksum = mach_read_from_3(buf + UNIV_PAGE_SIZE - FIL_PAGE_DATA_END); } } @@ -518,11 +490,11 @@ ulint mode memcpy(in_buf, buf, FIL_PAGE_DATA); /* copy trailer */ - memcpy(in_buf + page_size - FIL_PAGE_DATA_END, - buf + page_size - FIL_PAGE_DATA_END, FIL_PAGE_DATA_END); + memcpy(in_buf + UNIV_PAGE_SIZE - FIL_PAGE_DATA_END, + buf + UNIV_PAGE_SIZE - FIL_PAGE_DATA_END, FIL_PAGE_DATA_END); /* Copy the decrypted page to the buffer pool*/ - memcpy(buf, in_buf, page_size); + memcpy(buf, in_buf, UNIV_PAGE_SIZE); /* setting original page type */ @@ -531,7 +503,7 @@ ulint mode /* calc check sums and write to the buffer, if page was not compressed */ if (!(page_compression_flag )) { - do_check_sum(page_size, buf); + do_check_sum(UNIV_PAGE_SIZE, buf); } else { /* page_compression uses BUF_NO_CHECKSUM_MAGIC as checksum */ mach_write_to_4(buf + FIL_PAGE_SPACE_OR_CHKSUM, BUF_NO_CHECKSUM_MAGIC); @@ -542,31 +514,13 @@ ulint mode } - - flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + buf); - - if (!page_compression_flag) { - - page_size = fsp_flags_get_page_size(flags); - - page_num = mach_read_from_4(buf+ 4); - page_decryption_key = FSP_FLAGS_GET_PAGE_ENCRYPTION_KEY(flags); - page_encrypted = FSP_FLAGS_GET_PAGE_ENCRYPTION(flags); - page_compression_flag = FSP_FLAGS_GET_PAGE_COMPRESSION(flags); -// fprintf(stderr,"Page num, page size, key, enc, compr: %lu, %lu, %lu, %lu %lu\n", page_num, page_size, page_encryption_key, page_encrypted, page_compression_flag); - } - // Need to free temporal buffer if no buffer was given - if (NULL == page_buf) { - ut_free(in_buf); - } - srv_stats.pages_page_decrypted.inc(); return err; } void do_check_sum( ulint page_size, byte* buf) { - ib_uint32_t checksum; + ib_uint32_t checksum = 0; /* recalculate check sum - from buf0flu.cc*/ switch ((srv_checksum_algorithm_t) srv_checksum_algorithm) { case SRV_CHECKSUM_ALGORITHM_CRC32: From b0a08301140f371c392900c3d507c46c5eecaf18 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Clemens=20D=C3=B6rrh=C3=B6fer?= Date: Tue, 14 Oct 2014 16:48:30 +0200 Subject: [PATCH 45/70] Secret file using absolut path instead of relative path. --- storage/xtradb/enc/EncKeys.cc | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/storage/xtradb/enc/EncKeys.cc b/storage/xtradb/enc/EncKeys.cc index 7a487bcf518df..bc5220fbd1460 100644 --- a/storage/xtradb/enc/EncKeys.cc +++ b/storage/xtradb/enc/EncKeys.cc @@ -103,8 +103,8 @@ int EncKeys::initKeysThroughFile(const char *name, const char *path, const char //If secret starts with FILE: interpret the secret as filename. if(memcmp(MAGIC, filekey, MAGIC_LEN) == 0) { int fk_len = strlen(filekey); - char *secretfile = (char*)malloc((len1 + (fk_len - MAGIC_LEN) + isSlash ? 1 : 2)* sizeof(char)); - sprintf(secretfile, "%s%s%s", path, isSlash ? "" : "/", filekey+MAGIC_LEN); + char *secretfile = (char*)malloc((fk_len - MAGIC_LEN)* sizeof(char)); + sprintf(secretfile, "%s", filekey+MAGIC_LEN); parseSecret(secretfile, secret); free(secretfile); } else From 8fc45be5916fd7eb02950c35e4df95f7129c40d0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludger=20G=C3=B6ckel?= Date: Wed, 15 Oct 2014 10:21:16 +0200 Subject: [PATCH 46/70] fixed decryption original page type and improved unit test! --- storage/xtradb/fil/fil0pageencryption.cc | 66 ++++++++++++------------ unittest/eperi/pageenc-t.cc | 26 +++++++++- 2 files changed, 58 insertions(+), 34 deletions(-) diff --git a/storage/xtradb/fil/fil0pageencryption.cc b/storage/xtradb/fil/fil0pageencryption.cc index f7216281fb8c4..571e0aac40f60 100644 --- a/storage/xtradb/fil/fil0pageencryption.cc +++ b/storage/xtradb/fil/fil0pageencryption.cc @@ -84,11 +84,12 @@ ulint mode if (!unit_test) { ut_ad(fil_space_is_page_encrypted(space_id)); + +#ifdef UNIV_DEBUG fil_system_enter(); space = fil_space_get_by_id(space_id); fil_system_exit(); -#ifdef UNIV_DEBUG fprintf(stderr, "InnoDB: Note: Preparing for encryption for space %lu name %s len %lu\n", space_id, fil_space_name(space), len); @@ -154,38 +155,39 @@ ulint mode (const unsigned char *) &rkey, key_len, (const unsigned char *) &iv, iv_len); ut_ad(write_size == data_size); - } - if (page_compressed) { - /* page compressed pages: only one encryption. 3 bytes remain unencrypted. 2 bytes are appended to the encrypted buffer. - * one byte is later written to the checksum header. Additionally trailer remains unencrypted (8 bytes). - */ - offset = 1; - } - /* copy remaining bytes to output buffer */ - memcpy(out_buf + FIL_PAGE_DATA + data_size, buf + FIL_PAGE_DATA + data_size - 1, - remainder - offset); - - if (page_compressed ) { - remaining_byte = mach_read_from_1(buf + FIL_PAGE_DATA + data_size +1); - } else { - //create temporary buffer for 2nd encryption - tmp_buf = static_cast(ut_malloc(64)); - /* 2nd encryption: 63 bytes from out_buf, result length is 64 bytes */ - err = my_aes_encrypt_cbc((char*)out_buf + UNIV_PAGE_SIZE -FIL_PAGE_DATA_END -62, - 63, - (char*)tmp_buf, - &write_size, - (const unsigned char *)&rkey, - key_len, - (const unsigned char *)&iv, - iv_len); - ut_ad(write_size == 64); - //AES_cbc_encrypt((uchar*)out_buf + UNIV_PAGE_SIZE -FIL_PAGE_DATA_END -62, tmp_buf, 63, &aeskey, iv, AES_ENCRYPT); - /* copy 62 bytes from 2nd encryption to out_buf, last 2 bytes are copied later to a header field*/ - memcpy(out_buf + UNIV_PAGE_SIZE - FIL_PAGE_DATA_END -62, tmp_buf, 62); - } + if (page_compressed) { + /* page compressed pages: only one encryption. 3 bytes remain unencrypted. 2 bytes are appended to the encrypted buffer. + * one byte is later written to the checksum header. Additionally trailer remains unencrypted (8 bytes). + */ + offset = 1; + } + /* copy remaining bytes to output buffer */ + memcpy(out_buf + FIL_PAGE_DATA + data_size, buf + FIL_PAGE_DATA + data_size - 1, + remainder - offset); + + if (page_compressed ) { + remaining_byte = mach_read_from_1(buf + FIL_PAGE_DATA + data_size +1); + } else { + //create temporary buffer for 2nd encryption + tmp_buf = static_cast(ut_malloc(64)); + /* 2nd encryption: 63 bytes from out_buf, result length is 64 bytes */ + err = my_aes_encrypt_cbc((char*)out_buf + UNIV_PAGE_SIZE -FIL_PAGE_DATA_END -62, + 63, + (char*)tmp_buf, + &write_size, + (const unsigned char *)&rkey, + key_len, + (const unsigned char *)&iv, + iv_len); + ut_ad(write_size == 64); + //AES_cbc_encrypt((uchar*)out_buf + UNIV_PAGE_SIZE -FIL_PAGE_DATA_END -62, tmp_buf, 63, &aeskey, iv, AES_ENCRYPT); + /* copy 62 bytes from 2nd encryption to out_buf, last 2 bytes are copied later to a header field*/ + memcpy(out_buf + UNIV_PAGE_SIZE - FIL_PAGE_DATA_END -62, tmp_buf, 62); + + } + } /* error handling */ if (err != AES_OK) { /* If error we leave the actual page as it was */ @@ -333,7 +335,7 @@ ulint mode /* Get the page type */ - orig_page_type = mach_read_from_3(buf); + orig_page_type = mach_read_from_2(buf+1); if (FIL_PAGE_PAGE_COMPRESSED == orig_page_type) { if (page_compressed != NULL) { diff --git a/unittest/eperi/pageenc-t.cc b/unittest/eperi/pageenc-t.cc index 9d0e0d8a34720..f159d38dcfb78 100644 --- a/unittest/eperi/pageenc-t.cc +++ b/unittest/eperi/pageenc-t.cc @@ -20,6 +20,20 @@ extern int summef(int a, int b); extern int summef2(int a, int b); extern int multiplikation(int a, int b); +ulint +mach_read_from_2( +/*=============*/ + const byte* b) /*!< in: pointer to 2 bytes */ +{ + return(((ulint)(b[0]) << 8) | (ulint)(b[1])); +} +ulint +mach_read_from_1( +/*=============*/ + const byte* b) /*!< in: pointer to 1 bytes */ +{ + return((ulint)(b[0])); +} extern byte* fil_encrypt_page( @@ -76,17 +90,25 @@ return buffer; } -void testIt(char* filename, ulint cmp_checksum) { +void testIt(char* filename, ulint do_not_cmp_checksum) { byte* buf = readFile(filename); byte* dest = (byte *) malloc(16384*sizeof(byte)); ulint out_len; ulint cc1 = 0; + + ulint orig_page_type = mach_read_from_2(buf + 24); fil_encrypt_page(0,buf,dest,0,255, &out_len, 1); cc1 = (buf!=dest); + if (!do_not_cmp_checksum) { + /* verify page type and enryption key*/ + cc1 = cc1 && (mach_read_from_2(dest+1) == orig_page_type); + /* 255 is the key used for unit test */ + cc1 = cc1 && (mach_read_from_1(dest) == 255); + } fil_decrypt_page(NULL, dest, 16384 ,NULL,NULL, 1); ulint a = 0; ulint b = 0; - if (cmp_checksum) { + if (do_not_cmp_checksum) { a = 4; b = 8; } From 046fba74aa582a1a1a4ff5eae03a1f3c26ad0969 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludger=20G=C3=B6ckel?= Date: Wed, 15 Oct 2014 11:51:56 +0200 Subject: [PATCH 47/70] changed checksum calculation, so that headers are included, added a unit test --- storage/xtradb/fil/fil0pageencryption.cc | 65 ++++++++++++++---------- unittest/eperi/pageenc-t.cc | 27 +++++++++- 2 files changed, 64 insertions(+), 28 deletions(-) diff --git a/storage/xtradb/fil/fil0pageencryption.cc b/storage/xtradb/fil/fil0pageencryption.cc index 571e0aac40f60..91288de817602 100644 --- a/storage/xtradb/fil/fil0pageencryption.cc +++ b/storage/xtradb/fil/fil0pageencryption.cc @@ -36,10 +36,14 @@ this program; if not, write to the Free Software Foundation, Inc., //#define UNIV_PAGEENCRIPTION_DEBUG //#define CRYPT_FF -/* calculate a 3 byte checksum to verify decryption. One byte is needed for other things */ -ulint fil_page_encryption_calc_checksum(unsigned char* buf, size_t size) { +/* calculate a 3 byte checksum to verify decryption. One byte is currently needed for other things */ +ulint fil_page_encryption_calc_checksum(unsigned char* buf) { ulint checksum = 0; - checksum = ut_fold_binary(buf, size); + checksum = ut_fold_binary(buf + FIL_PAGE_OFFSET, + FIL_PAGE_FILE_FLUSH_LSN - FIL_PAGE_OFFSET) + + ut_fold_binary(buf + FIL_PAGE_DATA, + UNIV_PAGE_SIZE - FIL_PAGE_DATA + - FIL_PAGE_END_LSN_OLD_CHKSUM); checksum = checksum & 0x0FFFFFF0UL; checksum = checksum >> 8; checksum = checksum & 0x00FFFFFFUL; @@ -112,7 +116,7 @@ ulint mode } /* calculate a checksum, can be used to verify decryption */ - checksum = fil_page_encryption_calc_checksum(buf + FIL_PAGE_DATA, UNIV_PAGE_SIZE - (FIL_PAGE_DATA_END + FIL_PAGE_DATA)); + checksum = fil_page_encryption_calc_checksum(buf); const unsigned char rkey[] = {0xbd, 0xe4, 0x72, 0xa2, 0x95, 0x67, 0x5c, 0xa9, 0x2e, 0x04, 0x67, 0xea, 0xdb, 0xc0, 0xe0, 0x23, @@ -193,7 +197,7 @@ ulint mode /* If error we leave the actual page as it was */ fprintf(stderr, - "InnoDB: Warning: Encryption failed for space %lu name %s len %lu rt %d write %lu, error: %lu\n", + "InnoDB: Warning: Encryption failed for space %lu name %s len %lu rt %d write %lu, error: %d\n", space_id, fil_space_name(space), len, err, data_size, err); fflush(stderr); srv_stats.pages_page_encryption_error.inc(); @@ -401,7 +405,7 @@ ulint mode if (err != AES_OK) { /* surely key could not be fetched */ fprintf(stderr, "InnoDB: Corruption: Page is marked as encrypted\n" - "InnoDB: but decrypt failed with error %d, encryption key: %d.\n", + "InnoDB: but decrypt failed with error %d, encryption key %d.\n", err, (int)page_decryption_key); fflush(stderr); if (NULL == page_buf) { @@ -434,7 +438,7 @@ ulint mode if (err != AES_OK) { fprintf(stderr, "InnoDB: Corruption: Page is marked as encrypted\n" "InnoDB: but decrypt failed with error %d.\n" - "InnoDB: size %lu len %lu, key%d\n", err, data_size, + "InnoDB: size %lu len %lu, key %d\n", err, data_size, len, (int)page_decryption_key); fflush(stderr); if (NULL == page_buf) { @@ -474,8 +478,6 @@ ulint mode } - /* calculate a checksum to verify decryption*/ - checksum = fil_page_encryption_calc_checksum(in_buf + FIL_PAGE_DATA, UNIV_PAGE_SIZE - (FIL_PAGE_DATA_END + FIL_PAGE_DATA) ); /* compare with stored checksum */ ulint compressed_size = mach_read_from_2(in_buf+ FIL_PAGE_DATA); ibool no_checksum_support = 0; @@ -504,20 +506,6 @@ ulint mode ut_free(tmp_page_buf); ut_free(tmp_buf); } - if (no_checksum_support) { - fprintf(stderr, "InnoDB: decrypting page can not be verified!\n"); - fflush(stderr); - - } else { - if ((stored_checksum != checksum)) { - err = PAGE_ENCRYPTION_WRONG_KEY; - // Need to free temporal buffer if no buffer was given - if (NULL == page_buf) { - ut_free(in_buf); - } - return err; - } - } #ifdef UNIV_PAGEENCRIPTION_DEBUG @@ -539,9 +527,34 @@ ulint mode mach_write_to_2(buf + FIL_PAGE_TYPE, orig_page_type); + /* calculate a checksum to verify decryption */ + checksum = fil_page_encryption_calc_checksum(buf); + + if (no_checksum_support) { + fprintf(stderr, "InnoDB: decrypting page can not be verified!\n"); + fflush(stderr); + + } else { + if ((stored_checksum != checksum)) { + err = PAGE_ENCRYPTION_WRONG_KEY; + fprintf(stderr, "InnoDB: Corruption: Page is marked as encrypted.\n" + "InnoDB: Checksum mismatch after decryption.\n" + "InnoDB: size %lu len %lu, key %d\n", data_size, + len, (int)page_decryption_key); + fflush(stderr); + // Need to free temporal buffer if no buffer was given + if (NULL == page_buf) { + ut_free(in_buf); + } + return err; + } + } - /* calc check sums and write to the buffer, if page was not compressed */ if (!(page_compression_flag )) { + /* calc check sums and write to the buffer, if page was not compressed. + * if the decryption is verified, it is assumed that the original page was restored, re-calculating the original + * checksums should be ok + */ do_check_sum(UNIV_PAGE_SIZE, buf); } else { /* page_compression uses BUF_NO_CHECKSUM_MAGIC as checksum */ @@ -558,10 +571,10 @@ ulint mode } +/* recalculate check sum - from buf0flu.cc*/ void do_check_sum( ulint page_size, byte* buf) { ib_uint32_t checksum = 0; - /* recalculate check sum - from buf0flu.cc*/ - switch ((srv_checksum_algorithm_t) srv_checksum_algorithm) { + switch ((srv_checksum_algorithm_t) srv_checksum_algorithm) { case SRV_CHECKSUM_ALGORITHM_CRC32: case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32: checksum = buf_calc_page_crc32(buf); diff --git a/unittest/eperi/pageenc-t.cc b/unittest/eperi/pageenc-t.cc index f159d38dcfb78..a66199bd6f4be 100644 --- a/unittest/eperi/pageenc-t.cc +++ b/unittest/eperi/pageenc-t.cc @@ -89,6 +89,26 @@ fclose(fileptr); // Close the file return buffer; } +void testEncryptionChecksum(char* filename) { + byte* buf = readFile(filename); + byte* dest = (byte *) malloc(16384*sizeof(byte)); + ulint out_len; + fil_encrypt_page(0,buf,dest,0,255, &out_len, 1); + dest[2000]=0xFF; + dest[2001]=0xFF; + dest[2002]=0xFF; + dest[2003]=0xFF; + + ulint result = fil_decrypt_page(NULL, dest, 16384 ,NULL,NULL, 1); + + char str[80]; + strcpy (str,"Detect decryption error in "); + strcat (str,filename ); + + + ok (result == 1, "%s encryption result %lu", (char*) str, result); + +} void testIt(char* filename, ulint do_not_cmp_checksum) { byte* buf = readFile(filename); @@ -97,15 +117,17 @@ void testIt(char* filename, ulint do_not_cmp_checksum) { ulint cc1 = 0; ulint orig_page_type = mach_read_from_2(buf + 24); - fil_encrypt_page(0,buf,dest,0,255, &out_len, 1); + byte* snd = fil_encrypt_page(0,buf,dest,0,255, &out_len, 1); cc1 = (buf!=dest); + cc1 = cc1 && (snd==dest); if (!do_not_cmp_checksum) { /* verify page type and enryption key*/ cc1 = cc1 && (mach_read_from_2(dest+1) == orig_page_type); /* 255 is the key used for unit test */ cc1 = cc1 && (mach_read_from_1(dest) == 255); } - fil_decrypt_page(NULL, dest, 16384 ,NULL,NULL, 1); + ulint result = fil_decrypt_page(NULL, dest, 16384 ,NULL,NULL, 1); + cc1 = result == 0; ulint a = 0; ulint b = 0; if (do_not_cmp_checksum) { @@ -150,6 +172,7 @@ int main() test_page_enc_dec(); + testEncryptionChecksum((char* )"xaa"); return 0; } From 923ca507afabd2f24f152b47d1967fad23ad13bc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Clemens=20D=C3=B6rrh=C3=B6fer?= Date: Thu, 16 Oct 2014 10:04:54 +0200 Subject: [PATCH 48/70] Salt now reading the correct number of bytes --- storage/xtradb/enc/EncKeys.cc | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/storage/xtradb/enc/EncKeys.cc b/storage/xtradb/enc/EncKeys.cc index 7b059ca3ab7f0..b869fbb4d5cf3 100644 --- a/storage/xtradb/enc/EncKeys.cc +++ b/storage/xtradb/enc/EncKeys.cc @@ -133,11 +133,6 @@ void EncKeys::parseSecret( const char *secretfile, char *secret ) { long file_size = ftell(fp); fseek(fp, 0L, SEEK_SET); fgets(secret, (MAX_SECRET_SIZE >= file_size)?file_size:MAX_SECRET_SIZE, fp); - fseek(fp, 0L, SEEK_SET); - for(i=0; i Date: Thu, 16 Oct 2014 10:11:53 +0200 Subject: [PATCH 49/70] backup meines zwischenstandes... --- storage/xtradb/enc/EncKeys.cc | 14 ++++------ unittest/eperi/CMakeLists.txt | 7 ++++- unittest/eperi/pageenc-t.cc | 48 ++++++++++++++++++++++++++++++----- unittest/eperi/secret.txt | 1 + 4 files changed, 53 insertions(+), 17 deletions(-) create mode 100644 unittest/eperi/secret.txt diff --git a/storage/xtradb/enc/EncKeys.cc b/storage/xtradb/enc/EncKeys.cc index 7b059ca3ab7f0..544668c9d375b 100644 --- a/storage/xtradb/enc/EncKeys.cc +++ b/storage/xtradb/enc/EncKeys.cc @@ -128,17 +128,13 @@ int EncKeys::initKeysThroughServer( const char *name, const char *path, const ch void EncKeys::parseSecret( const char *secretfile, char *secret ) { int i=0; - FILE *fp = my_fopen(secretfile, O_RDWR, MYF(MY_WME)); + FILE *fp = fopen(secretfile, "rb"); fseek(fp, 0L, SEEK_END); long file_size = ftell(fp); - fseek(fp, 0L, SEEK_SET); - fgets(secret, (MAX_SECRET_SIZE >= file_size)?file_size:MAX_SECRET_SIZE, fp); - fseek(fp, 0L, SEEK_SET); - for(i=0; i= file_size)? file_size:MAX_SECRET_SIZE, fp); + + fclose(fp); } /** diff --git a/unittest/eperi/CMakeLists.txt b/unittest/eperi/CMakeLists.txt index 766057e3d1643..95d2380f15f12 100644 --- a/unittest/eperi/CMakeLists.txt +++ b/unittest/eperi/CMakeLists.txt @@ -39,7 +39,7 @@ MY_ADD_TESTS(EperiKeySingleton MY_ADD_TESTS( pageenc EXT "cc" - LINK_LIBRARIES xtradb perfschema mysys mysys_ssl sql mysql) + LINK_LIBRARIES mysys_ssl xtradb perfschema mysys sql mysql) file(COPY @@ -73,4 +73,9 @@ file(COPY ${CMAKE_CURRENT_LIST_DIR}/compressed_full DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) + +file(COPY + ${CMAKE_CURRENT_LIST_DIR}/secret.txt + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) + endif() diff --git a/unittest/eperi/pageenc-t.cc b/unittest/eperi/pageenc-t.cc index a66199bd6f4be..e026bafb51ef3 100644 --- a/unittest/eperi/pageenc-t.cc +++ b/unittest/eperi/pageenc-t.cc @@ -19,7 +19,12 @@ typedef unsigned long int ibool; extern int summef(int a, int b); extern int summef2(int a, int b); extern int multiplikation(int a, int b); - +extern "C" { +extern int my_aes_decrypt_cbc(const char* source, unsigned long int source_length, + char* dest, unsigned long int *dest_length, + const unsigned char* key, uint8 key_length, + const unsigned char* iv, uint8 iv_length); +} ulint mach_read_from_2( /*=============*/ @@ -73,7 +78,7 @@ fil_decrypt_page( -byte* readFile(char* fileName) { +byte* readFile(char* fileName, int* fileLen) { FILE *fileptr; byte *buffer; long filelen; @@ -86,11 +91,13 @@ rewind(fileptr); // Jump back to the beginning of the file buffer = (byte *)malloc((filelen+1)*sizeof(byte)); // Enough memory for file + \0 fread(buffer, filelen, 1, fileptr); // Read in the entire file fclose(fileptr); // Close the file +if (fileLen!=NULL) + *fileLen = filelen; return buffer; } void testEncryptionChecksum(char* filename) { - byte* buf = readFile(filename); + byte* buf = readFile(filename,NULL); byte* dest = (byte *) malloc(16384*sizeof(byte)); ulint out_len; fil_encrypt_page(0,buf,dest,0,255, &out_len, 1); @@ -111,7 +118,7 @@ void testEncryptionChecksum(char* filename) { } void testIt(char* filename, ulint do_not_cmp_checksum) { - byte* buf = readFile(filename); + byte* buf = readFile(filename, NULL); byte* dest = (byte *) malloc(16384*sizeof(byte)); ulint out_len; ulint cc1 = 0; @@ -166,13 +173,40 @@ void test_page_enc_dec() { } +void +test_cbc_secret_txt() +{ + printf("%d", strlen("Salted__")); + int len = 0; + byte* buf = readFile((char*)"secret.txt",&len); + byte* dest = (byte *) malloc(16384*sizeof(byte)); + + + + ulint dest_len = 0; + unsigned char key[32] = {0xFF,0xFF,0xFF,0xEE,0xEE,0xEE,0xFF,0xFF,0xFF,0xEE,0xEE,0xEE,0xAA,0xAA,0xAA,0xAA,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0,0x0}; + uint8 k_len = 32; + unsigned char iv[16] = {0xFF,0xFF,0xFF,0xEE,0xEE,0xEE,0xFF,0xFF,0xFF,0xEE,0xEE,0xEE,0xAA,0xAA,0xAA,0xAA}; + uint8 i_len = 16; + + my_aes_decrypt_cbc((char*)buf, len, (char*)dest , &dest_len, (unsigned char*) &key, k_len, (unsigned char*) &iv, i_len); + ulint cc1 = (strncmp((char*)dest, "secret\n", dest_len)==0); + char result[10] = "123451234"; + memcpy(result,dest,10); + result[dest_len]='\0'; + ok(cc1 ,"Result is secret -> %s", result); + ulint cc2 = dest_len==7; + ok(cc2 ,"Result length is %lu.",dest_len); + +} + int main() { - - test_page_enc_dec(); - testEncryptionChecksum((char* )"xaa"); + test_cbc_secret_txt(); +// test_page_enc_dec(); +// testEncryptionChecksum((char* )"xaa"); return 0; } diff --git a/unittest/eperi/secret.txt b/unittest/eperi/secret.txt new file mode 100644 index 0000000000000..af9e26054f29d --- /dev/null +++ b/unittest/eperi/secret.txt @@ -0,0 +1 @@ +{��� �퓺���� \ No newline at end of file From 6dbd4d6e466525a8b80b9d78cb531710596ccc5e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Clemens=20D=C3=B6rrh=C3=B6fer?= Date: Thu, 16 Oct 2014 15:07:27 +0200 Subject: [PATCH 50/70] String terminators removed. --- storage/xtradb/enc/EncKeys.cc | 2 -- 1 file changed, 2 deletions(-) diff --git a/storage/xtradb/enc/EncKeys.cc b/storage/xtradb/enc/EncKeys.cc index fb6780487946f..3dc50a32314d0 100644 --- a/storage/xtradb/enc/EncKeys.cc +++ b/storage/xtradb/enc/EncKeys.cc @@ -151,7 +151,6 @@ void EncKeys::parseSecret( const char *secretfile, char *secret ) { unsigned char *key = new unsigned char[keySize32]; unsigned char *iv = new unsigned char[ivSize16]; memcpy(&salt, buf + magicSize, magicSize); - salt[magicSize] = '\0'; memcpy(_initPwd, initialPwd, strlen(initialPwd)); _initPwd[strlen(initialPwd)]= '\0'; my_bytes_to_key((unsigned char *) salt, _initPwd, key, iv); @@ -327,7 +326,6 @@ char* EncKeys::decryptFile(const char* filename, const char *secret, int *errorC unsigned char *iv = new unsigned char[ivSize16]; char *decrypted = new char[file_size]; memcpy(&salt, buffer + magicSize, magicSize); - salt[magicSize] = '\0'; my_bytes_to_key((unsigned char *) salt, secret, key, iv); unsigned long int d_size = 0; int res = my_aes_decrypt_cbc((const char*)buffer + 2 * magicSize, file_size - 2 * magicSize, From 6333c1fa42c8cbc3a422af78fd22d2ba289c73e1 Mon Sep 17 00:00:00 2001 From: unknown Date: Thu, 16 Oct 2014 15:35:51 +0200 Subject: [PATCH 51/70] fix windows issues, support unc path --- storage/xtradb/enc/EncKeys.cc | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/storage/xtradb/enc/EncKeys.cc b/storage/xtradb/enc/EncKeys.cc index fb6780487946f..6f37d40382437 100644 --- a/storage/xtradb/enc/EncKeys.cc +++ b/storage/xtradb/enc/EncKeys.cc @@ -97,7 +97,8 @@ int EncKeys::initKeysThroughFile(const char *name, const char *path, const char const char *MAGIC = "FILE:"; const short MAGIC_LEN = 5; int ret = NO_ERROR_KEY_FILE_PARSE_OK; - bool isSlash = ('/' == path[len1 - 1]); + bool isUncPath= (len1>2) ? ((strncmp("\\\\", path, 2)==0) ? TRUE : FALSE) : FALSE; + bool isSlash = ((isUncPath? '\\':'/') == path[len1 - 1]); char *secret = (char*) malloc(MAX_SECRET_SIZE +1 * sizeof(char)); char *filename = (char*) malloc((len1 + len2 + (isSlash ? 1 : 2)) * sizeof(char)); if(filekey != NULL) @@ -105,8 +106,9 @@ int EncKeys::initKeysThroughFile(const char *name, const char *path, const char //If secret starts with FILE: interpret the secret as filename. if(memcmp(MAGIC, filekey, MAGIC_LEN) == 0) { int fk_len = strlen(filekey); - char *secretfile = (char*)malloc((fk_len - MAGIC_LEN)* sizeof(char)); - sprintf(secretfile, "%s", filekey+MAGIC_LEN); + char *secretfile = (char*)malloc( (1 + fk_len - MAGIC_LEN)* sizeof(char)); + memcpy(secretfile, filekey+MAGIC_LEN, fk_len - MAGIC_LEN); + secretfile[fk_len-MAGIC_LEN] = '\0'; parseSecret(secretfile, secret); free(secretfile); } else @@ -114,7 +116,7 @@ int EncKeys::initKeysThroughFile(const char *name, const char *path, const char sprintf(secret, "%s", filekey); } } - sprintf(filename, "%s%s%s", path, isSlash ? "" : "/", name); + sprintf(filename, "%s%s%s", path, isSlash ? "" : (isUncPath ? "\\":"/"), name); ret = parseFile((const char *)filename, 254, secret); free(filename); free(secret); @@ -151,7 +153,7 @@ void EncKeys::parseSecret( const char *secretfile, char *secret ) { unsigned char *key = new unsigned char[keySize32]; unsigned char *iv = new unsigned char[ivSize16]; memcpy(&salt, buf + magicSize, magicSize); - salt[magicSize] = '\0'; + //salt[magicSize] = '\0'; memcpy(_initPwd, initialPwd, strlen(initialPwd)); _initPwd[strlen(initialPwd)]= '\0'; my_bytes_to_key((unsigned char *) salt, _initPwd, key, iv); @@ -327,7 +329,6 @@ char* EncKeys::decryptFile(const char* filename, const char *secret, int *errorC unsigned char *iv = new unsigned char[ivSize16]; char *decrypted = new char[file_size]; memcpy(&salt, buffer + magicSize, magicSize); - salt[magicSize] = '\0'; my_bytes_to_key((unsigned char *) salt, secret, key, iv); unsigned long int d_size = 0; int res = my_aes_decrypt_cbc((const char*)buffer + 2 * magicSize, file_size - 2 * magicSize, From 1e72272b5e984d6551b640bcce48b7312b0eba4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludger=20G=C3=B6ckel?= Date: Wed, 22 Oct 2014 16:26:03 +0200 Subject: [PATCH 52/70] cleanup of code --- storage/xtradb/fil/fil0fil.cc | 7 ++----- storage/xtradb/fil/fil0pageencryption.cc | 3 --- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index fcca1a54a439c..36abb003c6ad5 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -755,10 +755,7 @@ fil_node_open_file( ut_ad(mutex_own(&(system->mutex))); ut_a(node->n_pending == 0); ut_a(node->open == FALSE); - if (strcmp(node->name,"test/b")==0) { - fprintf(stderr,"file access: %s", node->name); - fflush(stderr); - } + if (node->size == 0) { /* It must be a single-table tablespace and we do not know the size of the file yet. First we open the file in the normal @@ -2624,7 +2621,7 @@ static ulint fil_check_pending_io( /*=================*/ - fil_space_t* space, /*!< in/out: Tablespace to chemismatchck */ + fil_space_t* space, /*!< in/out: Tablespace to check */ fil_node_t** node, /*!< out: Node in space list */ ulint count) /*!< in: number of attempts so far */ { diff --git a/storage/xtradb/fil/fil0pageencryption.cc b/storage/xtradb/fil/fil0pageencryption.cc index 91288de817602..a5b2440578d1e 100644 --- a/storage/xtradb/fil/fil0pageencryption.cc +++ b/storage/xtradb/fil/fil0pageencryption.cc @@ -33,8 +33,6 @@ this program; if not, write to the Free Software Foundation, Inc., #include #include -//#define UNIV_PAGEENCRIPTION_DEBUG -//#define CRYPT_FF /* calculate a 3 byte checksum to verify decryption. One byte is currently needed for other things */ ulint fil_page_encryption_calc_checksum(unsigned char* buf) { @@ -186,7 +184,6 @@ ulint mode (const unsigned char *)&iv, iv_len); ut_ad(write_size == 64); - //AES_cbc_encrypt((uchar*)out_buf + UNIV_PAGE_SIZE -FIL_PAGE_DATA_END -62, tmp_buf, 63, &aeskey, iv, AES_ENCRYPT); /* copy 62 bytes from 2nd encryption to out_buf, last 2 bytes are copied later to a header field*/ memcpy(out_buf + UNIV_PAGE_SIZE - FIL_PAGE_DATA_END -62, tmp_buf, 62); From 23df03bd3081452e5749460da8e6f9cce1807a3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludger=20G=C3=B6ckel?= Date: Thu, 23 Oct 2014 10:56:11 +0200 Subject: [PATCH 53/70] removed superfluous stuff --- sql/mysqld.cc | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 5fd24f96f1ffd..71cfc7b31c70f 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -696,7 +696,7 @@ MY_LOCALE *my_default_lc_time_names; SHOW_COMP_OPTION have_ssl, have_symlink, have_dlopen, have_query_cache; SHOW_COMP_OPTION have_geometry, have_rtree_keys; -SHOW_COMP_OPTION have_crypt, have_datacrypt, have_compress; +SHOW_COMP_OPTION have_crypt, have_compress; SHOW_COMP_OPTION have_profiling; SHOW_COMP_OPTION have_openssl; @@ -8486,11 +8486,6 @@ static int mysql_init_variables(void) #else have_crypt=SHOW_OPTION_NO; #endif -#ifdef HAVE_DATACRYPT - have_datacrypt=SHOW_OPTION_YES; -#else - have_datacrypt=SHOW_OPTION_NO; -#endif #ifdef HAVE_COMPRESS have_compress= SHOW_OPTION_YES; #else From 47a3bab3c09aad2708192fb7db667be5d61110cf Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludger=20G=C3=B6ckel?= Date: Thu, 23 Oct 2014 11:41:37 +0200 Subject: [PATCH 54/70] removed superfluous defines --- sql/mysqld.cc | 3 --- 1 file changed, 3 deletions(-) diff --git a/sql/mysqld.cc b/sql/mysqld.cc index 71cfc7b31c70f..5bf6c21058c74 100644 --- a/sql/mysqld.cc +++ b/sql/mysqld.cc @@ -3682,9 +3682,6 @@ static bool init_global_datetime_format(timestamp_type format_type, } return false; } -//#ifndef SHOW_LONG_STATUS -//#define SHOW_LONG_STATUS 15 -//#endif From a2cf68d48f35b7d2e7f5811e0171d11c91785bdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludger=20G=C3=B6ckel?= Date: Thu, 23 Oct 2014 15:37:08 +0200 Subject: [PATCH 55/70] use openssl compiler flag --- mysys_ssl/my_aes.cc | 10 ++++- storage/xtradb/enc/KeySingleton.cc | 12 ++++-- storage/xtradb/fil/fil0fil.cc | 31 ++++++++++---- storage/xtradb/fil/fil0pageencryption.cc | 47 ++++++++++++--------- storage/xtradb/handler/ha_innodb.cc | 43 +++++++++++-------- storage/xtradb/handler/ha_innodb.h | 1 - storage/xtradb/include/fil0pageencryption.h | 14 +++--- 7 files changed, 99 insertions(+), 59 deletions(-) diff --git a/mysys_ssl/my_aes.cc b/mysys_ssl/my_aes.cc index eb245b2a43829..fef60cd9040ef 100644 --- a/mysys_ssl/my_aes.cc +++ b/mysys_ssl/my_aes.cc @@ -137,9 +137,13 @@ my_aes_hexToUint(const char* in, unsigned char *out, int dest_length) void my_bytes_to_key(const unsigned char *salt, const char *secret, unsigned char *key, unsigned char *iv) { +#ifndef HAVE_OPENSSL + return; +#else const EVP_CIPHER *type = EVP_aes_256_cbc(); const EVP_MD *digest = EVP_sha1(); EVP_BytesToKey(type, digest, salt, (unsigned char*) secret, strlen(secret), 1, key, iv); +#endif } /** Crypt buffer with AES encryption algorithm. @@ -194,8 +198,12 @@ int my_aes_encrypt_cbc(const char* source, unsigned long int source_length, if (! EVP_EncryptFinal_ex(&ctx.ctx, (unsigned char *) dest + u_len, &f_len)) return AES_BAD_DATA; /* Error */ *dest_length = (unsigned long int) (u_len + f_len); -#endif + return AES_OK; +#else + /* currently Open SSL is required */ + return AES_BAD_DATA; +#endif } int my_aes_decrypt_cbc(const char* source, unsigned long int source_length, diff --git a/storage/xtradb/enc/KeySingleton.cc b/storage/xtradb/enc/KeySingleton.cc index 461ee5b892733..fedd95b9813fb 100644 --- a/storage/xtradb/enc/KeySingleton.cc +++ b/storage/xtradb/enc/KeySingleton.cc @@ -33,20 +33,26 @@ EncKeys KeySingleton::encKeys; KeySingleton & KeySingleton::getInstance() { if( !instanceInited) { - printf("Encryption / decryption keys were not initialized. " + fprintf(stderr, "Encryption / decryption keys were not initialized. " "You can not read encrypted tables or columns\n\n"); + fflush(stderr); } return theInstance; } KeySingleton & KeySingleton::getInstance(const char *name, const char *url, const int initType, const char *filekey) { - if(instanceInited) return theInstance; + if(instanceInited) return theInstance; +#ifndef HAVE_OPENSSL + instanceInited = false; +#else instanceInited = encKeys.initKeys(name, url, initType, filekey); +#endif if( !instanceInited) { - printf("Could not initialize any of the encryption / decryption keys. " + fprintf(stderr, "Could not initialize any of the encryption / decryption keys. " "You can not read encrypted tables\n\n"); + fflush(stderr); } return theInstance; diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 36abb003c6ad5..59e0c84a0f0d2 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -819,6 +819,10 @@ fil_node_open_file( space->flags); if (fil_page_is_encrypted(page)) { + /* if page is encrypted, write and error and return. + * Otherwise the server would crash if decrypting is not possible. + * This may be the case, if the keyfile could not be opened on server startup. + */ fprintf(stderr, "InnoDB: can not decrypt %s\n", node->name); @@ -1345,7 +1349,11 @@ fil_space_create( if (fsp_flags_is_page_encrypted(flags)) { if (!KeySingleton::getInstance().isAvailable() || KeySingleton::getInstance().getKeys(fsp_flags_get_page_encryption_key(flags))==NULL) { - + /* by returning here it should be avoided that + * the server crashes, if someone tries to access an + * encrypted table and the encryption key is not available. + * The table is treaded as non-existent. + */ ib_logf(IB_LOG_LEVEL_WARN, "Tablespace '%s' can not be opened, because encryption key can not be found (space id: %lu, key %lu)\n" , name, (ulong) id, fsp_flags_get_page_encryption_key(flags)); @@ -2091,7 +2099,7 @@ fil_check_first_page( { ulint space_id; ulint flags; - ulint cc = 0; + ulint page_is_encrypted = 0; if (srv_force_recovery >= SRV_FORCE_IGNORE_CORRUPT) { return(NULL); @@ -2099,11 +2107,11 @@ fil_check_first_page( space_id = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_ID + page); flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); - cc = fil_page_is_encrypted(page); - if (!KeySingleton::getInstance().isAvailable() && cc) { - cc = 1; + page_is_encrypted = fil_page_is_encrypted(page); + if (!KeySingleton::getInstance().isAvailable() && page_is_encrypted) { + page_is_encrypted = 1; } else { - cc = 0; + page_is_encrypted = 0; if (UNIV_PAGE_SIZE != fsp_flags_get_page_size(flags)) { fprintf(stderr, "InnoDB: Error: Current page size %lu != page size on page %lu\n", UNIV_PAGE_SIZE, fsp_flags_get_page_size(flags)); @@ -2125,11 +2133,14 @@ fil_check_first_page( } } - if (!cc && buf_page_is_corrupted( + if (!page_is_encrypted && buf_page_is_corrupted( false, page, fsp_flags_get_zip_size(flags))) { return("checksum mismatch"); } else { - if (cc) { + if (page_is_encrypted) { + /* this error message is interpreted by the calling method, which is + * executed if the server starts in recovery mode. + */ return("can not decrypt"); } @@ -4333,6 +4344,10 @@ fil_validate_single_table_tablespace( check_msg, fsp->filepath, tablename); fsp->success = FALSE; if (strncmp(check_msg, "can not decrypt", strlen(check_msg))==0) { + /* by returning here, it should be avoided, that the server crashes, + * if started in recovery mode and can not decrypt tables, if + * the key file can not be read. + */ fsp->encryption_error = 1; return; } diff --git a/storage/xtradb/fil/fil0pageencryption.cc b/storage/xtradb/fil/fil0pageencryption.cc index a5b2440578d1e..17d0e20ccb708 100644 --- a/storage/xtradb/fil/fil0pageencryption.cc +++ b/storage/xtradb/fil/fil0pageencryption.cc @@ -56,14 +56,14 @@ ulint fil_page_encryption_calc_checksum(unsigned char* buf) { byte* fil_encrypt_page( /*==============*/ -ulint space_id, /*!< in: tablespace id of the table. */ -byte* buf, /*!< in: buffer from which to write; in aio - this must be appropriately aligned */ -byte* out_buf, /*!< out: encrypted buffer */ -ulint len, /*!< in: length of input buffer.*/ -ulint encryption_key,/*!< in: encryption key */ -ulint* out_len, /*!< out: actual length of encrypted page */ -ulint mode + ulint space_id, /*!< in: tablespace id of the table. */ + byte* buf, /*!< in: buffer from which to write; in aio + this must be appropriately aligned */ + byte* out_buf, /*!< out: encrypted buffer */ + ulint len, /*!< in: length of input buffer.*/ + ulint encryption_key,/*!< in: encryption key */ + ulint* out_len, /*!< out: actual length of encrypted page */ + ulint mode /*!< in: calling mode. Should be 0. Can be used for unit tests */ ) { int err = AES_OK; @@ -283,13 +283,13 @@ ulint mode @return decrypted page */ ulint fil_decrypt_page( /*================*/ -byte* page_buf, /*!< in: preallocated buffer or NULL */ -byte* buf, /*!< in/out: buffer from which to read; in aio - this must be appropriately aligned */ -ulint len, /*!< in: length of output buffer.*/ -ulint* write_size, /*!< in/out: Actual payload size of the decrypted data. */ -ibool* page_compressed, /*!(ut_malloc(len + 16)); - //in_buf = static_cast(ut_malloc(UNIV_PAGE_SIZE)); + /* it was request to align this buffer */ + in_buffer = static_cast(ut_malloc(2 * (len + 16))); + in_buf = static_cast(ut_align(in_buffer, len + 16)); + } else { in_buf = page_buf; } @@ -406,7 +409,7 @@ ulint mode err, (int)page_decryption_key); fflush(stderr); if (NULL == page_buf) { - ut_free(in_buf); + ut_free(in_buffer); } return err; } @@ -439,7 +442,7 @@ ulint mode len, (int)page_decryption_key); fflush(stderr); if (NULL == page_buf) { - ut_free(in_buf); + ut_free(in_buffer); } return err; } @@ -541,7 +544,7 @@ ulint mode fflush(stderr); // Need to free temporal buffer if no buffer was given if (NULL == page_buf) { - ut_free(in_buf); + ut_free(in_buffer); } return err; } @@ -569,7 +572,9 @@ ulint mode /* recalculate check sum - from buf0flu.cc*/ -void do_check_sum( ulint page_size, byte* buf) { +void do_check_sum( + ulint page_size, + byte* buf) { ib_uint32_t checksum = 0; switch ((srv_checksum_algorithm_t) srv_checksum_algorithm) { case SRV_CHECKSUM_ALGORITHM_CRC32: diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 01fbdff8137c6..5ab722f108691 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -106,6 +106,9 @@ this program; if not, write to the Free Software Foundation, Inc., #include "page0zip.h" #include "fil0pagecompress.h" +#include "KeySingleton.h" + + #define thd_get_trx_isolation(X) ((enum_tx_isolation)thd_tx_isolation(X)) #ifdef MYSQL_DYNAMIC_PLUGIN @@ -3415,8 +3418,8 @@ innobase_init( ut_a(DATA_MYSQL_TRUE_VARCHAR == (ulint)MYSQL_TYPE_VARCHAR); - //FF - KeySingleton& keysingleton = KeySingleton::getInstance( + + KeySingleton::getInstance( innobase_data_encryption_providername, innobase_data_encryption_providerurl, innobase_data_encryption_providertype, innobase_data_encryption_filekey); @@ -3502,6 +3505,15 @@ innobase_init( goto error; } } +#ifndef HAVE_OPENSSL + if (innobase_data_encryption_providertype != 0) { + sql_print_error("InnoDB: innobase_data_encryption_providertype = %lu unsupported.\n" + "InnoDB: Open SSL not available. \n", + innobase_data_encryption_providertype); + goto error; + + } +#endif #ifndef HAVE_LZ4 if (innodb_compression_algorithm == PAGE_LZ4_ALGORITHM) { @@ -3570,13 +3582,6 @@ innobase_init( srv_data_home = (innobase_data_home_dir ? innobase_data_home_dir : default_path); - printf("\ninnobase_data_home_dir = %s,\n innobase_data_encryption_providerurl = %s," - "\n Type = %u, innobase_data_encryption_providername = %s\n\n", - innobase_data_home_dir, innobase_data_encryption_providerurl - ? innobase_data_encryption_providerurl : "ist NULL", - innobase_data_encryption_providertype, innobase_data_encryption_providername - ? innobase_data_encryption_providername : "ist NULL"); -// fflush(stdout); /* Set default InnoDB data file size to 12 MB and let it be auto-extending. Thus users can use InnoDB in >= 4.0 without having @@ -11638,6 +11643,15 @@ ha_innobase::check_table_options( ha_table_option_struct *options= table->s->option_struct; atomic_writes_t awrites = (atomic_writes_t)options->atomic_writes; if (options->page_encryption) { +#ifndef HAVE_OPENSSL + push_warning( + thd, Sql_condition::WARN_LEVEL_WARN, + HA_WRONG_CREATE_OPTION, + "InnoDB: Open SSL is required for PAGE_ENCRYPTION" + ); + return "PAGE_ENCRYPTION"; + +#else if (!use_tablespace) { push_warning( thd, Sql_condition::WARN_LEVEL_WARN, @@ -11654,14 +11668,7 @@ ha_innobase::check_table_options( ); return "PAGE_ENCRYPTION"; } - if (create_info->key_block_size) { - push_warning( - thd, Sql_condition::WARN_LEVEL_WARN, - HA_WRONG_CREATE_OPTION, - "InnoDB: PAGE_ENCRYPTION table can't have" - " key_block_size"); - return "PAGE_ENCRYPTION"; - } +#endif } /* Check page compression requirements */ @@ -11729,7 +11736,7 @@ ha_innobase::check_table_options( if ((ulint)options->page_encryption_key != ULINT_UNDEFINED) { if (options->page_encryption == false) { - /* ignore for alter table...*/ + /* ignore this to allow alter table without changing page_encryption_key ...*/ } if (options->page_encryption_key < 1 || options->page_encryption_key > 255) { diff --git a/storage/xtradb/handler/ha_innodb.h b/storage/xtradb/handler/ha_innodb.h index bf8f09e010dff..9401b973ca869 100644 --- a/storage/xtradb/handler/ha_innodb.h +++ b/storage/xtradb/handler/ha_innodb.h @@ -26,7 +26,6 @@ this program; if not, write to the Free Software Foundation, Inc., #include "dict0stats.h" -#include "KeySingleton.h" /* Structure defines translation table between mysql index and innodb diff --git a/storage/xtradb/include/fil0pageencryption.h b/storage/xtradb/include/fil0pageencryption.h index bdb964cf792b0..89d063870c996 100644 --- a/storage/xtradb/include/fil0pageencryption.h +++ b/storage/xtradb/include/fil0pageencryption.h @@ -66,15 +66,15 @@ operation. byte* fil_encrypt_page( /*==============*/ - ulint space_id, /*!< in: tablespace id of the - table. */ + ulint space_id, /*!< in: tablespace id of the + table. */ byte* buf, /*!< in: buffer from which to write; in aio - this must be appropriately aligned */ + this must be appropriately aligned */ byte* out_buf, /*!< out: compressed buffer */ ulint len, /*!< in: length of input buffer.*/ ulint compression_level, /*!< in: compression level */ - ulint* out_len, /*!< out: actual length of compressed page */ - ulint mode /*!< in: calling mode. Should be 0. */ + ulint* out_len, /*!< out: actual length of compressed page */ + ulint mode /*!< in: calling mode. Should be 0. */ ); /****************************************************************//** @@ -86,11 +86,11 @@ fil_decrypt_page( /*================*/ byte* page_buf, /*!< in: preallocated buffer or NULL */ byte* buf, /*!< out: buffer from which to read; in aio - this must be appropriately aligned */ + this must be appropriately aligned */ ulint len, /*!< in: length of output buffer.*/ ulint* write_size, /*!< in/out: Actual payload size of the decrypted data. */ ibool* page_compressed, /*! Date: Thu, 23 Oct 2014 15:39:30 +0200 Subject: [PATCH 56/70] added a comment --- storage/xtradb/fil/fil0fil.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 59e0c84a0f0d2..9da0d41c256f5 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -1352,7 +1352,7 @@ fil_space_create( /* by returning here it should be avoided that * the server crashes, if someone tries to access an * encrypted table and the encryption key is not available. - * The table is treaded as non-existent. + * The the table is treaded as non-existent. */ ib_logf(IB_LOG_LEVEL_WARN, "Tablespace '%s' can not be opened, because encryption key can not be found (space id: %lu, key %lu)\n" From 9607b8f78f9e1a047e654349f07fb27e999aaa65 Mon Sep 17 00:00:00 2001 From: Ludger Goeckel Date: Fri, 24 Oct 2014 15:28:13 +0200 Subject: [PATCH 57/70] introduced yassl for encryption, improved compilier flags for windows compilation with yassl, added unit test for aes enc/dec, removed compiler flags from xtradb, because visual studio says, they do not exist --- mysys_ssl/my_aes.cc | 159 ++++++++++- storage/xtradb/enc/KeySingleton.cc | 4 - storage/xtradb/handler/ha_innodb.cc | 23 +- unittest/eperi/CMakeLists.txt | 26 +- .../eperi/{eperi_aes-t.c => eperi_aes-t.cc} | 248 ++++++++---------- 5 files changed, 278 insertions(+), 182 deletions(-) rename unittest/eperi/{eperi_aes-t.c => eperi_aes-t.cc} (60%) diff --git a/mysys_ssl/my_aes.cc b/mysys_ssl/my_aes.cc index fef60cd9040ef..3b76d8dee4cde 100644 --- a/mysys_ssl/my_aes.cc +++ b/mysys_ssl/my_aes.cc @@ -21,6 +21,7 @@ #if defined(HAVE_YASSL) #include "aes.hpp" #include "openssl/ssl.h" +#include "crypto_wrapper.hpp" #elif defined(HAVE_OPENSSL) #include #include @@ -137,9 +138,63 @@ my_aes_hexToUint(const char* in, unsigned char *out, int dest_length) void my_bytes_to_key(const unsigned char *salt, const char *secret, unsigned char *key, unsigned char *iv) { -#ifndef HAVE_OPENSSL +#ifdef HAVE_YASSL +#ifndef ___min +#define ___min(a,b) (((a) < (b)) ? (a) : (b)) +#endif + /* + the yassl function has no support for SHA1. + Reason unknown. + */ + int keyLen = 32; + int ivLen = 16; + int EVP_SALT_SZ = 8; + const int SHA_LEN = 20; + yaSSL::SHA myMD; + uint digestSz = myMD.get_digestSize(); + unsigned char digest[SHA_LEN]; // max size + int sz = strlen(secret); + int count = 1; + int keyLeft = keyLen; + int ivLeft = ivLen; + int keyOutput = 0; + + while (keyOutput < (keyLen + ivLen)) { + int digestLeft = digestSz; + // D_(i - 1) + if (keyOutput) // first time D_0 is empty + myMD.update(digest, digestSz); + // data + myMD.update((yaSSL::byte* )secret, sz); + // salt + if (salt) + myMD.update(salt, EVP_SALT_SZ); + myMD.get_digest(digest); + // count + for (int j = 1; j < count; j++) { + myMD.update(digest, digestSz); + myMD.get_digest(digest); + } + + if (keyLeft) { + int store = ___min(keyLeft, static_cast(digestSz)); + memcpy(&key[keyLen - keyLeft], digest, store); + + keyOutput += store; + keyLeft -= store; + digestLeft -= store; + } + + if (ivLeft && digestLeft) { + int store = ___min(ivLeft, digestLeft); + memcpy(&iv[ivLen - ivLeft], &digest[digestSz - digestLeft], store); + + keyOutput += store; + ivLeft -= store; + } + } return; -#else +#elif HAVE_OPENSSL const EVP_CIPHER *type = EVP_aes_256_cbc(); const EVP_MD *digest = EVP_sha1(); EVP_BytesToKey(type, digest, salt, (unsigned char*) secret, strlen(secret), 1, key, iv); @@ -153,19 +208,62 @@ my_bytes_to_key(const unsigned char *salt, const char *secret, unsigned char *ke @param source [in] Pointer to data for encryption @param source_length [in] Size of encryption data @param dest [out] Buffer to place encrypted data (must be large enough) - @param key [in] Key to be used for encryption + @param dest_length [out] Pointer to size of encrypted data + @param key [in] Key to be used for encryption @param key_length [in] Length of the key. Will handle keys of any length + @param key [in] Iv to be used for encryption + @param key_length [in] Length of the iv. should be 16. @return - >= 0 Size of encrypted data - < 0 Error + > 0 error + 0 no error */ int my_aes_encrypt_cbc(const char* source, unsigned long int source_length, char* dest, unsigned long int *dest_length, const unsigned char* key, uint8 key_length, const unsigned char* iv, uint8 iv_length) { -#if defined(HAVE_OPENSSL) +#ifdef HAVE_YASSL + TaoCrypt::AES_CBC_Encryption enc; + /* 128 bit block used for padding */ + uint8 block[MY_AES_BLOCK_SIZE]; + int num_blocks; /* number of complete blocks */ + int i; + switch(key_length) { + case 16: + break; + case 24: + break; + case 32: + break; + default: + return AES_BAD_KEYSIZE; + } + + enc.SetKey((const TaoCrypt::byte *) key, key_length, (const TaoCrypt::byte *) iv); + + num_blocks = source_length / MY_AES_BLOCK_SIZE; + + for (i = num_blocks; i > 0; i--) /* Encode complete blocks */ + { + enc.Process((TaoCrypt::byte *) dest, (const TaoCrypt::byte *) source, + MY_AES_BLOCK_SIZE); + source += MY_AES_BLOCK_SIZE; + dest += MY_AES_BLOCK_SIZE; + } + + /* Encode the rest. We always have incomplete block */ + char pad_len = MY_AES_BLOCK_SIZE - (source_length - + MY_AES_BLOCK_SIZE * num_blocks); + memcpy(block, source, 16 - pad_len); + memset(block + MY_AES_BLOCK_SIZE - pad_len, pad_len, pad_len); + + enc.Process((TaoCrypt::byte *) dest, (const TaoCrypt::byte *) block, + MY_AES_BLOCK_SIZE); + + *dest_length = MY_AES_BLOCK_SIZE * (num_blocks + 1); + return AES_OK; +#elif defined(HAVE_OPENSSL) MyCipherCtx ctx; int u_len, f_len; /* The real key to be used for encryption */ @@ -211,7 +309,54 @@ int my_aes_decrypt_cbc(const char* source, unsigned long int source_length, const unsigned char* key, uint8 key_length, const unsigned char* iv, uint8 iv_length) { -#if defined(HAVE_OPENSSL) +#ifdef HAVE_YASSL + TaoCrypt::AES_CBC_Decryption dec; + /* 128 bit block used for padding */ + uint8 block[MY_AES_BLOCK_SIZE]; + int num_blocks; /* Number of complete blocks */ + int i; + switch(key_length) { + case 16: + break; + case 24: + break; + case 32: + break; + default: + return AES_BAD_KEYSIZE; + } + + dec.SetKey((const TaoCrypt::byte *) key, key_length, iv); + + num_blocks = source_length / MY_AES_BLOCK_SIZE; + + if ((source_length != num_blocks * MY_AES_BLOCK_SIZE) || num_blocks == 0 ) + /* Input size has to be even and at least one block */ + return AES_BAD_DATA; + + /* Decode all but last blocks */ + for (i = num_blocks - 1; i > 0; i--) + { + dec.Process((TaoCrypt::byte *) dest, (const TaoCrypt::byte *) source, + MY_AES_BLOCK_SIZE); + source += MY_AES_BLOCK_SIZE; + dest += MY_AES_BLOCK_SIZE; + } + + dec.Process((TaoCrypt::byte *) block, (const TaoCrypt::byte *) source, + MY_AES_BLOCK_SIZE); + + /* Use last char in the block as size */ + uint pad_len = (uint) (uchar) block[MY_AES_BLOCK_SIZE - 1]; + + if (pad_len > MY_AES_BLOCK_SIZE) + return AES_BAD_DATA; + /* We could also check whole padding but we do not really need this */ + + memcpy(dest, block, MY_AES_BLOCK_SIZE - pad_len); + *dest_length = MY_AES_BLOCK_SIZE * num_blocks - pad_len; + return AES_OK; +#elif defined(HAVE_OPENSSL) MyCipherCtx ctx; int u_len, f_len; diff --git a/storage/xtradb/enc/KeySingleton.cc b/storage/xtradb/enc/KeySingleton.cc index fedd95b9813fb..279a5c5982f9f 100644 --- a/storage/xtradb/enc/KeySingleton.cc +++ b/storage/xtradb/enc/KeySingleton.cc @@ -44,11 +44,7 @@ KeySingleton & KeySingleton::getInstance(const char *name, const char *url, const int initType, const char *filekey) { if(instanceInited) return theInstance; -#ifndef HAVE_OPENSSL - instanceInited = false; -#else instanceInited = encKeys.initKeys(name, url, initType, filekey); -#endif if( !instanceInited) { fprintf(stderr, "Could not initialize any of the encryption / decryption keys. " "You can not read encrypted tables\n\n"); diff --git a/storage/xtradb/handler/ha_innodb.cc b/storage/xtradb/handler/ha_innodb.cc index 5ab722f108691..eaa5f1baaac67 100644 --- a/storage/xtradb/handler/ha_innodb.cc +++ b/storage/xtradb/handler/ha_innodb.cc @@ -3422,7 +3422,7 @@ innobase_init( KeySingleton::getInstance( innobase_data_encryption_providername, innobase_data_encryption_providerurl, innobase_data_encryption_providertype, innobase_data_encryption_filekey); - + #ifndef DBUG_OFF static const char test_filename[] = "-@"; char test_tablename[sizeof test_filename @@ -3505,16 +3505,7 @@ innobase_init( goto error; } } -#ifndef HAVE_OPENSSL - if (innobase_data_encryption_providertype != 0) { - sql_print_error("InnoDB: innobase_data_encryption_providertype = %lu unsupported.\n" - "InnoDB: Open SSL not available. \n", - innobase_data_encryption_providertype); - goto error; - - } -#endif - + #ifndef HAVE_LZ4 if (innodb_compression_algorithm == PAGE_LZ4_ALGORITHM) { sql_print_error("InnoDB: innodb_compression_algorithm = %lu unsupported.\n" @@ -11643,15 +11634,6 @@ ha_innobase::check_table_options( ha_table_option_struct *options= table->s->option_struct; atomic_writes_t awrites = (atomic_writes_t)options->atomic_writes; if (options->page_encryption) { -#ifndef HAVE_OPENSSL - push_warning( - thd, Sql_condition::WARN_LEVEL_WARN, - HA_WRONG_CREATE_OPTION, - "InnoDB: Open SSL is required for PAGE_ENCRYPTION" - ); - return "PAGE_ENCRYPTION"; - -#else if (!use_tablespace) { push_warning( thd, Sql_condition::WARN_LEVEL_WARN, @@ -11668,7 +11650,6 @@ ha_innobase::check_table_options( ); return "PAGE_ENCRYPTION"; } -#endif } /* Check page compression requirements */ diff --git a/unittest/eperi/CMakeLists.txt b/unittest/eperi/CMakeLists.txt index 999e05fd132f5..7c6ecb4609b95 100644 --- a/unittest/eperi/CMakeLists.txt +++ b/unittest/eperi/CMakeLists.txt @@ -25,21 +25,29 @@ INCLUDE_DIRECTORIES(${CMAKE_SOURCE_DIR}/sql ) SET(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -D SINGLETON_TEST_DATA=\\\"${CMAKE_SOURCE_DIR}/unittest/eperi\\\" ") + +MY_ADD_TESTS(eperi_aes + EXT "cc" + LINK_LIBRARIES mysys_ssl) + + if (WIN32) + else() -MY_ADD_TESTS(eperi) -#MY_ADD_TESTS(eperi_aes -# LINK_LIBRARIES mysys_ssl dbug) +MY_ADD_TESTS( + pageenc + EXT "cc" + LINK_LIBRARIES mysys_ssl xtradb perfschema mysys sql mysql) MY_ADD_TESTS(EperiKeySingleton EXT "cc" LINK_LIBRARIES xtradb pcre mysys_ssl) + + +MY_ADD_TESTS(eperi) -MY_ADD_TESTS( - pageenc - EXT "cc" - LINK_LIBRARIES mysys_ssl xtradb perfschema mysys sql mysql) +endif() file(COPY @@ -89,6 +97,4 @@ file(COPY DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) file(COPY ${CMAKE_CURRENT_LIST_DIR}/secret256.enc - DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) - -endif() + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) \ No newline at end of file diff --git a/unittest/eperi/eperi_aes-t.c b/unittest/eperi/eperi_aes-t.cc similarity index 60% rename from unittest/eperi/eperi_aes-t.c rename to unittest/eperi/eperi_aes-t.cc index ca67ffb79656a..c7bd9fd65c19a 100644 --- a/unittest/eperi/eperi_aes-t.c +++ b/unittest/eperi/eperi_aes-t.cc @@ -1,23 +1,34 @@ #define EP_UNIT_TEST 1 #define UNIV_INLINE +#define AES_OK 0 +#define AES_BAD_DATA -1 +#define AES_BAD_KEYSIZE -5 +#define AES_KEY_CREATION_FAILED -10 + +#define MY_AES_BLOCK_SIZE 16 /* Block size in bytes */ -#ifndef __WIN__ typedef unsigned char byte; typedef unsigned long int ulint; typedef unsigned long int ibool; #include -#include #include #include -#include "../../storage/xtradb/include/fil0pageencryption.h" - - - - - +extern "C" { +extern int my_aes_decrypt_cbc(const char* source, unsigned long int source_length, + char* dest, unsigned long int *dest_length, + const unsigned char* key, uint8 key_length, + const unsigned char* iv, uint8 iv_length); + + +extern int my_aes_encrypt_cbc(const char* source, unsigned long int source_length, + char* dest, unsigned long int *dest_length, + const unsigned char* key, uint8 key_length, + const unsigned char* iv, uint8 iv_length); +extern void my_bytes_to_key(const unsigned char *salt, const char *secret, unsigned char *key, unsigned char *iv); +} #define MY_AES_TEST_TEXTBLOCK "abcdefghijklmnopqrstuvwxyz\ ABCDEFGHIJKLMNOPQRSTUVW\ @@ -176,19 +187,21 @@ test_wrong_key() } void -test_cbc() +test_cbc128() { + char expected[]= + { + 0x51 , 0xBC , 0xF9 , 0x96 , 0xCB , 0x6A , 0x6D , 0x18 , 0x08 , 0xE1 , 0x08 , 0xC5 , 0x07 , 0x78 , 0x70 , 0xA6, + 0x15 , 0x3E , 0x41 , 0x34 , 0xEC , 0x5E , 0xA2 , 0x67 , 0x52 , 0x51 , 0x87 , 0x61 , 0x8A , 0x15 , 0xE0 , 0xD7, + 0x1D , 0x9A , 0x5B , 0x4A , 0xF9 , 0x9F , 0x13 , 0xEE , 0x3B , 0x77 , 0x1E , 0xD1 , 0xF6 , 0x54 , 0xAD , 0xFE + }; plan(1); int i; - char source[20]; - for(i=0; i<20; i++) { - source[i] = 5; - } - ulint s_len = 20; - char dest[32]; - for(i=0; i<32; i++){ - dest[i]=0; - } + char* source = "int i = memcmp(decbuf,inbuf,16);"; + ulint s_len = strlen(source); + char* dest = (char* ) malloc(100); + char* result = (char*) malloc(100); + ulint dest_len = 0; unsigned char key[16] = {0x58, 0x3b, 0xe7, 0xf3, 0x34, 0xf8, 0x5e, 0x7d, 0x9d, 0xdb, 0x36, 0x2e, 0x9a, 0xc3, 0x81, 0x51}; @@ -196,145 +209,92 @@ test_cbc() unsigned char iv[16] = {0x33, 0x25, 0xcc, 0x3f, 0x02, 0x20, 0x3f, 0xb6, 0xb8, 0x49, 0x99, 0x00, 0x42, 0xe5, 0x8b, 0xcb}; uint8 i_len = 16; - int ec = my_aes_encrypt_cbc(source, s_len, (char*) &dest, &dest_len, (unsigned char*) &key, k_len,(unsigned char*) &iv, i_len); + int ec = my_aes_encrypt_cbc(source, s_len, dest, &dest_len, (unsigned char*) &key, k_len,(unsigned char*) &iv, i_len); ok(ec == AES_OK, "Checking return code."); - for(i=0; i<20; i++) { - source[i] = 0; - } - my_aes_decrypt_cbc(dest , dest_len, (char*)&source, &dest_len, (unsigned char*) &key, k_len, (unsigned char*) &iv, i_len); - ok(strcmp(source, "Beam me up, Scotty."),"Decrypted text is identical to original text."); + ok(memcmp(expected,dest,48)==0, "expected cipher text"); + ok(memcmp(source,dest,32)!=0,"plain and cipher text differ"); + ec = my_aes_decrypt_cbc(dest , dest_len, result, &dest_len, (unsigned char*) &key, k_len, (unsigned char*) &iv, i_len); + ok((dest_len == s_len) && (ec == AES_OK) && (strncmp(result, "int i = memcmp(decbuf,inbuf,16);",dest_len)==0),"Decrypted text is identical to original text."); + free(result); + free(dest); } void -test_cbc_resultsize() +test_cbc192() { - plan(2); - char *source = (char*) malloc(5000*sizeof(char)); - source = "abcdefghijklmnopqrstfjdklfkjdsljsdlkfjsaklföjsfölkdsjfölsd" - "kjklösjsdklfjdsklöfjsdalökfjdsklöjfölksdjfklösdajfklösdaj"; - ulint s_len = (ulint) strlen(source); - char* dest = (char *) malloc(2 * s_len * sizeof(char)); - ulint d_len = 0; - unsigned char key[16] = {0x58, 0x3b, 0xe7, 0xf3, 0x34, 0xf8, - 0x5e, 0x7d, 0x9d, 0xdb, 0x36, 0x2e, 0x9a, 0xc3, 0x81, 0x51}; - uint8 k_len = 16; - unsigned char iv[16] = {0x33, 0x25, 0xcc, 0x3f, 0x02, 0x20, 0x3f, 0xb6, 0xb8, - 0x49, 0x99, 0x00, 0x42, 0xe5, 0x8b, 0xcb}; - uint8 i_len = 16; - my_aes_encrypt_cbc(source, s_len, dest, &d_len, (unsigned char*)&key, k_len, (unsigned char *)&iv, i_len); - ok(d_len==128, "Destination length ok."); -} - -void test_cbc_enc_dec() { - unsigned char inbuf[1024]="Hello,world!"; -unsigned char encbuf[1024]; - -unsigned char key32[] = {0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa}; -unsigned char deckey32[] = {0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa} -; -unsigned char iv[] = {0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}; -unsigned char deciv[] = {0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}; - -AES_KEY aeskey; -AES_KEY aesdeckey; - -//Now enrypt -memset(encbuf, 0, sizeof(encbuf)); -AES_set_encrypt_key(key32, 32*8, &aeskey); -AES_cbc_encrypt(inbuf, encbuf, 16, &aeskey, iv, AES_ENCRYPT); - -//Now decrypt -unsigned char decbuf[1024]; -memset(decbuf, 0, sizeof(decbuf)); - -AES_set_decrypt_key(deckey32, 32*8, &aesdeckey); -AES_cbc_encrypt(encbuf, decbuf, 16, &aesdeckey, deciv, AES_DECRYPT); + plan(1); + int i; + char* source = "int i = memcmp(decbuf,inbuf,16);"; + ulint s_len = strlen(source); + char* dest = (char* ) malloc(100); + char* result = (char*) malloc(100); -int i = memcmp(decbuf,inbuf,16); -ok (i==0, "in==out"); + ulint dest_len = 0; + unsigned char key[24] = {0x58, 0x3b, 0xe7, 0xf3, 0x34, 0xf8, + 0x5e, 0x7d, 0x9d, 0xdb, 0x36, 0x2e, 0x9a, 0xc3, 0x81, 0x51, + 0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08}; + uint8 k_len = 24; + unsigned char iv[16] = {0x33, 0x25, 0xcc, 0x3f, 0x02, 0x20, 0x3f, 0xb6, 0xb8, + 0x49, 0x99, 0x00, 0x42, 0xe5, 0x8b, 0xcb}; + uint8 i_len = 16; + int ec = my_aes_encrypt_cbc(source, s_len, dest, &dest_len, (unsigned char*) &key, k_len,(unsigned char*) &iv, i_len); + ok(ec == AES_OK, "Checking return code."); + ec = my_aes_decrypt_cbc(dest , dest_len, result, &dest_len, (unsigned char*) &key, k_len, (unsigned char*) &iv, i_len); + ok((dest_len == s_len) && (ec == AES_OK) && (strncmp(result, "int i = memcmp(decbuf,inbuf,16);",dest_len)==0),"Decrypted text is identical to original text."); + free(result); + free(dest); } -void test_cbc_enc_dec2() { - unsigned char inbuf[1024]="Hello,world!"; -unsigned char encbuf[1024]; - -unsigned char key32[] = {0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa}; -unsigned char deckey32[] = {0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa} -; -unsigned char iv[] = {0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}; -unsigned char deciv[] = {0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}; - -AES_KEY aeskey; -AES_KEY aesdeckey; +void +test_cbc256() +{ + char expected[] = { + 0x81, 0x22, 0x05, 0xA7, 0x3E, 0x9D, 0xB2, 0x18, 0x7F, 0xE2, 0x5C, 0xB4, 0xBD, 0xCD, 0xFB, 0x9B, + 0xB6, 0xEF, 0x64, 0x2C, 0xF4, 0x53, 0x9B, 0x29, 0x98, 0x3A, 0xD6, 0xDE, 0xB2, 0x65, 0xEF, 0x85, + 0xEF, 0x4B, 0xDA, 0x8F, 0xD9, 0xEB, 0xD7, 0x07, 0x80, 0x03, 0x0E, 0x7C, 0x55, 0x2E, 0x97, 0x47 + }; + plan(1); + int i; + char* source = "int i = memcmp(decbuf,inbuf,16);"; + ulint s_len = strlen(source); + char* dest = (char* ) malloc(100); + char* result = (char*) malloc(100); -//Now enrypt -memset(encbuf, 0, sizeof(encbuf)); -AES_set_encrypt_key(key32, 32*8, &aeskey); -AES_cbc_encrypt(inbuf, encbuf, 16, &aeskey, iv, AES_ENCRYPT); + ulint dest_len = 0; + unsigned char key[32] = {0x58, 0x3b, 0xe7, 0xf3, 0x34, 0xf8, + 0x5e, 0x7d, 0x9d, 0xdb, 0x36, 0x2e, 0x9a, 0xc3, 0x81, 0x51, + 0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08, + 0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08}; + uint8 k_len = 32; + unsigned char iv[16] = {0x33, 0x25, 0xcc, 0x3f, 0x02, 0x20, 0x3f, 0xb6, 0xb8, + 0x49, 0x99, 0x00, 0x42, 0xe5, 0x8b, 0xcb}; + uint8 i_len = 16; + int ec = my_aes_encrypt_cbc(source, s_len, dest, &dest_len, (unsigned char*) &key, k_len,(unsigned char*) &iv, i_len); -//Now decrypt -unsigned char decbuf[1024]; -memset(decbuf, 0, sizeof(decbuf)); -AES_set_decrypt_key(deckey32, 32*8, &aesdeckey); -AES_cbc_encrypt(encbuf, decbuf, 16, &aesdeckey, deciv, AES_DECRYPT); -dump_buffer(16, decbuf); -dump_buffer(16, encbuf); + ok(ec == AES_OK, "Checking return code."); + ok(memcmp(expected,dest,48)==0,"Excepted cipher text - aes 256 cbc"); -int i = memcmp(decbuf,inbuf,16); -ok (i==0, "in==out"); + ec = my_aes_decrypt_cbc(dest , dest_len, result, &dest_len, (unsigned char*) &key, k_len, (unsigned char*) &iv, i_len); + ok((dest_len == s_len) && (ec == AES_OK) && (strncmp(result, "int i = memcmp(decbuf,inbuf,16);",dest_len)==0),"Decrypted text is identical to original text."); + free(result); + free(dest); } -void test_cbc_enc_() { - unsigned char inbuf[1024]="Hello,world!"; -unsigned char encbuf[1024]; - -unsigned char key32[] = {0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa}; -unsigned char deckey32[] = {0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa,0xaa} -; -unsigned char iv[] = {0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}; -unsigned char deciv[] = {0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00}; -AES_KEY aeskey; -AES_KEY aesdeckey; -//Now enrypt -memset(encbuf, 0, sizeof(encbuf)); -AES_set_encrypt_key(key32, 32*8, &aeskey); -AES_cbc_encrypt(inbuf, encbuf, 16, &aeskey, iv, AES_ENCRYPT); -//Now decrypt -unsigned char decbuf[1024]; -memset(decbuf, 0, sizeof(decbuf)); -AES_set_decrypt_key(deckey32, 32*8, &aesdeckey); -AES_cbc_encrypt(encbuf, decbuf, 16, &aesdeckey, deciv, AES_DECRYPT); -dump_buffer(16, decbuf); -dump_buffer(16, encbuf); -int i = memcmp(decbuf,inbuf,16); -ok (i==0, "in==out"); - -} -void test_page_enc_dec() { - unsigned char* buf = readFile("xaa"); - char* dest = (char *) malloc(16384*sizeof(char)); - //fil_encrypt_page(0,buf,dest,0,0,NULL,1); - - //fil_decrypt_page(NULL, dest, 0,NULL,1); - - ulint i = memcmp(buf,dest, 16384); - ok (i==0, "in==out"); -} /* * Test if bytes for AES Key and IV are generated in the same way as in openssl commandline. @@ -342,10 +302,15 @@ void test_page_enc_dec() { void test_bytes_to_key() { + + char expected[32] = { + 0x2E, 0xFF , 0xB7 , 0x1D , 0xDB , 0x97 , 0xA8 , 0x3A , 0x03 , 0x5A , 0x06 , 0xDF , 0xB0 , 0xDD , 0x72 , 0x29, + 0xA6, 0xD9 , 0x1F , 0xFB , 0xE6 , 0x06 , 0x3B , 0x4B , 0x81 , 0x23 , 0x85 , 0x45 , 0x71 , 0x28 , 0xFF , 0x1F + }; plan(2); unsigned char salt[] = {0x0c, 0x3b, 0x72, 0x1b, 0xfe, 0x07, 0xe2, 0xb3}; char *secret = "secret"; - char key[32]; + unsigned char* key = (unsigned char*)malloc(32 * sizeof(char)); unsigned char iv[16]; unsigned char keyresult[32] = {0x2E, 0xFF, 0xB7, 0x1D, 0xDB, 0x97, 0xA8, 0x3A, 0x03, 0x5A, 0x06, 0xDF, 0xB0, 0xDD, 0x72, 0x29, @@ -354,31 +319,34 @@ test_bytes_to_key() unsigned char ivresult[16] = {0x61, 0xFF, 0xC8, 0x27, 0x5B, 0x46, 0x4C, 0xBD, 0x55, 0x82, 0x0E, 0x54, 0x8F, 0xE4, 0x44, 0xD9}; - my_bytes_to_key((unsigned char*) &salt, secret, (unsigned char*) &key, (unsigned char*) &iv); - + my_bytes_to_key((unsigned char*) &salt, secret, (unsigned char*) key, (unsigned char*) &iv); + dump_buffer(32, key); ok(memcmp(key, &keyresult, 32) == 0, "BytesToKey key generated successfully."); ok(memcmp(iv, &ivresult, 16) == 0, "BytesToKey iv generated successfully."); + // following should ensure, that yassl and openssl calculate the same! + ok(memcmp(expected,key,32)==0, "expected result"); + free(key); } int main(int argc __attribute__((unused)),char *argv[]) { + test_cbc128(); + test_cbc192(); + test_cbc256(); + + test_bytes_to_key(); + /* test_cbc(); test_cbc_large(); test_cbc_keysizes(); test_cbc_wrong_keylength(); test_cbc_resultsize(); - test_cbc_enc_dec(); test_wrong_key(); test_bytes_to_key(); - + */ return 0; } -#else -int -main(int argc ,char *argv[]) { - return 0; -} -#endif + From 7d587116752777058143835c2771931f5a29029f Mon Sep 17 00:00:00 2001 From: Ludger Goeckel Date: Fri, 24 Oct 2014 15:52:39 +0200 Subject: [PATCH 58/70] fix ut_align call, removed keys from output (debug) --- storage/xtradb/enc/EncKeys.cc | 2 +- storage/xtradb/fil/fil0pageencryption.cc | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/storage/xtradb/enc/EncKeys.cc b/storage/xtradb/enc/EncKeys.cc index 1df23f75c3347..dc992a2066df1 100644 --- a/storage/xtradb/enc/EncKeys.cc +++ b/storage/xtradb/enc/EncKeys.cc @@ -367,7 +367,7 @@ void EncKeys::printKeyEntry( ulint id) } else { #ifdef UNIV_DEBUG - fprintf(stderr, "Key: id:%3u \tiv:%s \tkey:%s\n", entry->id, entry->iv, entry->key); + fprintf(stderr, "Key: id:%3u \tiv:%d bytes\tkey:%d bytes\n", entry->id, strlen(entry->iv)/2, strlen(entry->key)/2); #else fprintf(stderr, "Key: id:%3u\n", entry->id); #endif //UNIV_DEBUG diff --git a/storage/xtradb/fil/fil0pageencryption.cc b/storage/xtradb/fil/fil0pageencryption.cc index 17d0e20ccb708..ed5b6251ce310 100644 --- a/storage/xtradb/fil/fil0pageencryption.cc +++ b/storage/xtradb/fil/fil0pageencryption.cc @@ -354,9 +354,9 @@ ulint fil_decrypt_page( "InnoDB: FIL: Note: Decryption buffer not given, allocating...\n"); fflush(stderr); #endif /* UNIV_PAGEENCRIPTION_DEBUG */ - /* it was request to align this buffer */ - in_buffer = static_cast(ut_malloc(2 * (len + 16))); - in_buf = static_cast(ut_align(in_buffer, len + 16)); + /* it was requested to align this buffer */ + in_buffer = static_cast(ut_malloc(2 * (len))); + in_buf = static_cast(ut_align(in_buffer, len)); } else { in_buf = page_buf; From de74ef107de2e12a7a3513a6b0c720f7e3cf1f4d Mon Sep 17 00:00:00 2001 From: Ludger Goeckel Date: Mon, 27 Oct 2014 11:06:28 +0100 Subject: [PATCH 59/70] improved cbc enc/dec with no padding support --- include/my_aes.h | 62 ++++++--- mysys_ssl/my_aes.cc | 60 +++++++- unittest/eperi/eperi_aes-t.cc | 254 +++++++++++++++++++--------------- 3 files changed, 235 insertions(+), 141 deletions(-) diff --git a/include/my_aes.h b/include/my_aes.h index 2d49afa648771..049cad3fd4175 100644 --- a/include/my_aes.h +++ b/include/my_aes.h @@ -35,20 +35,30 @@ C_MODE_START #define AES_KEY_LENGTH 128 /* Must be 128 192 or 256 */ -/* - my_aes_encrypt_cbc- Crypt buffer with AES encryption algorithm using cbc mode. - source - Pointer to data for encryption - source_length - size of encryption data - dest - buffer to place encrypted data (must be large enough) - key - Key to be used for encryption - kel_length - Length of the key. Will handle keys of any length - - returns - size of encrypted data, or negative in case of error. +/** + Crypt buffer with AES encryption algorithm. + + SYNOPSIS + my_aes_encrypt() + @param source [in] Pointer to data for encryption + @param source_length [in] Size of encryption data + @param dest [out] Buffer to place encrypted data (must be large enough) + @param dest_length [out] Pointer to size of encrypted data + @param key [in] Key to be used for encryption + @param key_length [in] Length of the key. 16, 24 or 32 + @param iv [in] Iv to be used for encryption + @param iv_length [in] Length of the iv. should be 16. + @param noPadding [in] if set to true, no padding is used, input data size must be a mulitple of the AES block size + + @return + != 0 error + 0 no error */ int my_aes_encrypt_cbc(const char* source, unsigned long int source_length, char* dest, unsigned long int *dest_length, const unsigned char* key, uint8 key_length, - const unsigned char* iv, uint8 iv_length); + const unsigned char* iv, uint8 iv_length, + int noPadding); /** @@ -89,22 +99,30 @@ void my_aes_hexToUint(const char* in, int my_aes_encrypt(const char *source, int source_length, char *dest, const char *key, int key_length); -/* - my_aes_decrypt_cbc - DeCrypt buffer with AES encryption algorithm using - cbc Mode. - source - Pointer to data for decryption - source_length - size of encrypted data - dest - buffer to place decrypted data (must be large enough) - key - Key to be used for decryption - kel_length - Length of the key. Will handle keys of any length - - returns - size of original data, or negative in case of error. +/** + AES decryption - CBC mode + + SYNOPSIS + my_aes_encrypt() + @param source [in] Pointer to data to decrypt + @param source_length [in] Size of data + @param dest [out] Buffer to place decrypted data (must be large enough) + @param dest_length [out] Pointer to size of decrypted data + @param key [in] Key to be used for decryption + @param key_length [in] Length of the key. 16, 24 or 32 + @param iv [in] Iv to be used for encryption + @param iv_length [in] Length of the iv. should be 16. + @param noPadding [in] if set to true, no padding is used, input data size must be a mulitple of the AES block size + + @return + != 0 error + 0 no error */ - int my_aes_decrypt_cbc(const char* source, unsigned long int source_length, char* dest, unsigned long int *dest_length, const unsigned char* key, uint8 key_length, - const unsigned char* iv, uint8 iv_length); + const unsigned char* iv, uint8 iv_length, + int noPadding); /* my_aes_decrypt - DeCrypt buffer with AES encryption algorithm. diff --git a/mysys_ssl/my_aes.cc b/mysys_ssl/my_aes.cc index 3b76d8dee4cde..02526c5dbd35f 100644 --- a/mysys_ssl/my_aes.cc +++ b/mysys_ssl/my_aes.cc @@ -210,19 +210,24 @@ my_bytes_to_key(const unsigned char *salt, const char *secret, unsigned char *ke @param dest [out] Buffer to place encrypted data (must be large enough) @param dest_length [out] Pointer to size of encrypted data @param key [in] Key to be used for encryption - @param key_length [in] Length of the key. Will handle keys of any length - @param key [in] Iv to be used for encryption - @param key_length [in] Length of the iv. should be 16. + @param key_length [in] Length of the key. 16, 24 or 32 + @param iv [in] Iv to be used for encryption + @param iv_length [in] Length of the iv. should be 16. + @param noPadding [in] if set to true, no padding is used, input data size must be a mulitple of the AES block size @return - > 0 error + != 0 error 0 no error */ int my_aes_encrypt_cbc(const char* source, unsigned long int source_length, char* dest, unsigned long int *dest_length, const unsigned char* key, uint8 key_length, - const unsigned char* iv, uint8 iv_length) + const unsigned char* iv, uint8 iv_length, + int noPadding) { + if (noPadding) { + if (source_length % 16 !=0) return AES_BAD_DATA; + } #ifdef HAVE_YASSL TaoCrypt::AES_CBC_Encryption enc; /* 128 bit block used for padding */ @@ -252,6 +257,12 @@ int my_aes_encrypt_cbc(const char* source, unsigned long int source_length, dest += MY_AES_BLOCK_SIZE; } + if (noPadding) { + *dest_length = MY_AES_BLOCK_SIZE * (num_blocks); + return AES_OK; + + } + /* Encode the rest. We always have incomplete block */ char pad_len = MY_AES_BLOCK_SIZE - (source_length - MY_AES_BLOCK_SIZE * num_blocks); @@ -286,6 +297,9 @@ int my_aes_encrypt_cbc(const char* source, unsigned long int source_length, if (! EVP_EncryptInit_ex(&ctx.ctx, cipher, engine, key, iv)) return AES_BAD_DATA; /* Error */ + if (noPadding) { + EVP_CIPHER_CTX_set_padding(&ctx.ctx, 0); + } EVP_CIPHER_CTX_key_length(&ctx.ctx); OPENSSL_assert(EVP_CIPHER_CTX_key_length(&ctx.ctx) == key_length); OPENSSL_assert(EVP_CIPHER_CTX_iv_length(&ctx.ctx) == iv_length); @@ -304,11 +318,35 @@ int my_aes_encrypt_cbc(const char* source, unsigned long int source_length, #endif } + +/** + AES decryption - CBC mode + + SYNOPSIS + my_aes_encrypt() + @param source [in] Pointer to data to decrypt + @param source_length [in] Size of data + @param dest [out] Buffer to place decrypted data (must be large enough) + @param dest_length [out] Pointer to size of decrypted data + @param key [in] Key to be used for decryption + @param key_length [in] Length of the key. 16, 24 or 32 + @param iv [in] Iv to be used for encryption + @param iv_length [in] Length of the iv. should be 16. + @param noPadding [in] if set to true, no padding is used, input data size must be a mulitple of the AES block size + + @return + != 0 error + 0 no error +*/ int my_aes_decrypt_cbc(const char* source, unsigned long int source_length, char* dest, unsigned long int *dest_length, const unsigned char* key, uint8 key_length, - const unsigned char* iv, uint8 iv_length) + const unsigned char* iv, uint8 iv_length, + int noPadding) { + if (noPadding) { + if (source_length % 16 !=0) return AES_BAD_DATA; + } #ifdef HAVE_YASSL TaoCrypt::AES_CBC_Decryption dec; /* 128 bit block used for padding */ @@ -345,6 +383,12 @@ int my_aes_decrypt_cbc(const char* source, unsigned long int source_length, dec.Process((TaoCrypt::byte *) block, (const TaoCrypt::byte *) source, MY_AES_BLOCK_SIZE); + + if (noPadding) { + memcpy(dest, block, MY_AES_BLOCK_SIZE); + *dest_length = MY_AES_BLOCK_SIZE * num_blocks; + return AES_OK; + } /* Use last char in the block as size */ uint pad_len = (uint) (uchar) block[MY_AES_BLOCK_SIZE - 1]; @@ -379,6 +423,9 @@ int my_aes_decrypt_cbc(const char* source, unsigned long int source_length, if (! EVP_DecryptInit_ex(&ctx.ctx, cipher, engine, key, iv)) return AES_BAD_DATA; /* Error */ + if (noPadding) { + EVP_CIPHER_CTX_set_padding(&ctx.ctx, 0); + } OPENSSL_assert(EVP_CIPHER_CTX_key_length(&ctx.ctx) == key_length); OPENSSL_assert(EVP_CIPHER_CTX_iv_length(&ctx.ctx) == iv_length); OPENSSL_assert(EVP_CIPHER_CTX_block_size(&ctx.ctx) == 16); @@ -401,6 +448,7 @@ my_aes_encrypt(const char* source, int source_length, char* dest, { #if defined(HAVE_YASSL) TaoCrypt::AES_ECB_Encryption enc; + /* 128 bit block used for padding */ uint8 block[MY_AES_BLOCK_SIZE]; int num_blocks; /* number of complete blocks */ diff --git a/unittest/eperi/eperi_aes-t.cc b/unittest/eperi/eperi_aes-t.cc index c7bd9fd65c19a..13c17b16ea245 100644 --- a/unittest/eperi/eperi_aes-t.cc +++ b/unittest/eperi/eperi_aes-t.cc @@ -20,13 +20,15 @@ extern "C" { extern int my_aes_decrypt_cbc(const char* source, unsigned long int source_length, char* dest, unsigned long int *dest_length, const unsigned char* key, uint8 key_length, - const unsigned char* iv, uint8 iv_length); + const unsigned char* iv, uint8 iv_length, + int noPadding); extern int my_aes_encrypt_cbc(const char* source, unsigned long int source_length, char* dest, unsigned long int *dest_length, const unsigned char* key, uint8 key_length, - const unsigned char* iv, uint8 iv_length); + const unsigned char* iv, uint8 iv_length, + int noPadding); extern void my_bytes_to_key(const unsigned char *salt, const char *secret, unsigned char *key, unsigned char *iv); } @@ -81,109 +83,44 @@ fclose(fileptr); // Close the file return buffer; } -void -test_cbc_wrong_keylength() -{ - plan(2); - char* source = "Joshua: Shall we play a game"; - ulint s_len = (ulint)strlen(source); - unsigned char key[24] = {0x89, 0x9c, 0x0e, 0xcb, 0x59, 0x2b, 0x2c, - 0xee, 0x46, 0xe6, 0x41, 0x91, 0xb6, 0xe6, 0xde, 0x9b, 0x97, - 0xd8, 0xa8, 0xee, 0xa4, 0x3b, 0xef, 0x78 }; - uint8 k_len = 6; - unsigned char iv[16] = {0xf0, 0x97, 0x40, 0x07, 0xd6, 0x19, 0x46, 0x6b, - 0x9e, 0xbf, 0x8d, 0x4f, 0x6e, 0x30, 0x2a, 0xa3}; - uint8 i_len = 16; - char* dest = (char *) malloc(2*s_len*sizeof(char)); - unsigned long int dest_len = 0; - int rc = my_aes_encrypt_cbc(source, s_len, dest, &dest_len,(unsigned char*) &key, k_len,(unsigned char*) &iv, i_len); - ok(rc == -5, "Encryption - wrong keylength was detected."); - rc = my_aes_decrypt_cbc(source, s_len, dest, &dest_len,(unsigned char*) &key, k_len,(unsigned char*) &iv, i_len); - ok(rc == -5, "Decryption - wrong keylength was detected."); -} -void -test_cbc_keysizes() -{ - plan(2); - char* source = MY_AES_TEST_JOSHUA; - ulint s_len = (ulint)strlen(source); - unsigned char key[24] = {0x89, 0x9c, 0x0e, 0xcb, 0x59, 0x2b, 0x2c, - 0xee, 0x46, 0xe6, 0x41, 0x91, 0xb6, 0xe6, 0xde, 0x9b, 0x97, - 0xd8, 0xa8, 0xee, 0xa4, 0x3b, 0xef, 0x78 }; - uint8 k_len = 24; - unsigned char iv[16] = {0xf0, 0x97, 0x40, 0x07, 0xd6, 0x19, 0x46, 0x6b, - 0x9e, 0xbf, 0x8d, 0x4f, 0x6e, 0x30, 0x2a, 0xa3}; - uint8 i_len = 16; - char* dest = (char *) malloc(2*s_len*sizeof(char)); - ulint dest_len = 0; - my_aes_encrypt_cbc(source, s_len, dest, &dest_len, key, k_len, iv, i_len); - source = (char *) malloc(strlen(MY_AES_TEST_TEXTBLOCK) * sizeof(char)); - my_aes_decrypt_cbc(dest , strlen(dest), source, &dest_len,(unsigned char*) &key, k_len,(unsigned char*) &iv, i_len); - ok(strcmp(source, MY_AES_TEST_JOSHUA),"Decrypted text is identical to original text."); - - unsigned char key2[32] = {0x7b, 0x3b, 0x8d, 0xa9, 0x4b, 0x77, - 0xf9, 0x1a, 0x6e, 0x05, 0x03, 0x7b, - 0x21, 0xad, 0x5f, 0x6e, 0x86, 0xbd, - 0x46, 0x57, 0xc4, 0x5d, 0x97, 0xbc, - 0xb5, 0xa3}; - k_len = 32; - dest = (char *) malloc(2*s_len*sizeof(char)); - my_aes_encrypt_cbc(source, s_len, dest, &dest_len, (unsigned char*) &key2, k_len,(unsigned char*) &iv, i_len); - source = (char *) malloc(strlen(MY_AES_TEST_TEXTBLOCK) * sizeof(char)); - my_aes_decrypt_cbc(dest , strlen(dest), source, &dest_len, (unsigned char*) &key2, k_len, (unsigned char*) &iv, i_len); - ok(strcmp(source, MY_AES_TEST_JOSHUA),"Decrypted text is identical to original text."); - free(source); - free(dest); -} + + void -test_cbc_large() +test_cbc128noPadding() { + char expected[]= + { + 0x51 , 0xBC , 0xF9 , 0x96 , 0xCB , 0x6A , 0x6D , 0x18 , 0x08 , 0xE1 , 0x08 , 0xC5 , 0x07 , 0x78 , 0x70 , 0xA6, + 0x15 , 0x3E , 0x41 , 0x34 , 0xEC , 0x5E , 0xA2 , 0x67 , 0x52 , 0x51 , 0x87 , 0x61 , 0x8A , 0x15 , 0xE0 , 0xD7, + 0x1D , 0x9A , 0x5B , 0x4A , 0xF9 , 0x9F , 0x13 , 0xEE , 0x3B , 0x77 , 0x1E , 0xD1 , 0xF6 , 0x54 , 0xAD , 0xFE + }; plan(1); - char* source = MY_AES_TEST_TEXTBLOCK; - ulint s_len = (ulint)strlen(source); + int i; + char* source = "int i = memcmp(decbuf,inbuf,16);"; + ulint s_len = strlen(source); + char* dest = (char* ) malloc(100); + char* result = (char*) malloc(100); - unsigned char key[16] = {0x3c, 0x5d, 0xc9, 0x15, 0x3a, 0x6f, 0xe5, 0xf2, - 0x25, 0x16, 0xe2, 0x17, 0xc1, 0x60, 0x3b, 0xf7}; + ulint dest_len = 0; + unsigned char key[16] = {0x58, 0x3b, 0xe7, 0xf3, 0x34, 0xf8, + 0x5e, 0x7d, 0x9d, 0xdb, 0x36, 0x2e, 0x9a, 0xc3, 0x81, 0x51}; uint8 k_len = 16; - unsigned char iv[16] = {0xf0, 0x97, 0x40, 0x00, 0x7d, 0x61, 0x94, 0x66, - 0xb9, 0xeb, 0xf8, 0xd4, 0x6e, 0x30, 0x2a, 0xa3}; + unsigned char iv[16] = {0x33, 0x25, 0xcc, 0x3f, 0x02, 0x20, 0x3f, 0xb6, 0xb8, + 0x49, 0x99, 0x00, 0x42, 0xe5, 0x8b, 0xcb}; uint8 i_len = 16; - char* dest = (char *) malloc( 2* s_len * sizeof(char)); - ulint dest_len = 0; - my_aes_encrypt_cbc(source, s_len, dest, &dest_len, key, k_len, iv, i_len); - source = (char *) malloc(strlen(MY_AES_TEST_TEXTBLOCK) * sizeof(char)); - my_aes_decrypt_cbc(dest , strlen(dest), source, &dest_len, (unsigned char*) &key, k_len, (unsigned char*) &iv, i_len); - ok(strcmp(source, MY_AES_TEST_TEXTBLOCK),"Decrypted text is identical to original text."); - free(source); - free(dest); -} + int ec = my_aes_encrypt_cbc(source, s_len, dest, &dest_len, (unsigned char*) &key, k_len,(unsigned char*) &iv, i_len, 1); + ok(ec == AES_OK, "Checking return code."); + ok(memcmp(expected,dest,dest_len)==0, "expected cipher text"); + ok(memcmp(source,dest,32)!=0,"plain and cipher text differ"); + ok(dest_len == s_len, "input length = output length, cbc 128 no padding"); -void -test_wrong_key() -{ - plan(1); - char* source = MY_AES_TEST_TEXTBLOCK; - ulint s_len = (ulint)strlen(source); - unsigned char key[16] = {0x3c, 0x5d, 0xc9, 0x15, 0x3a, 0x6f, 0xe5, 0xf2, - 0x25, 0x16, 0xe2, 0x17, 0xc1, 0x60, 0x3b, 0xf7}; - uint8 k_len = 16; - unsigned char iv[16] = {0xf0, 0x97, 0x40, 0x00, 0x7d, 0x61, 0x94, 0x66, - 0xb9, 0xeb, 0xf8, 0xd4, 0x6e, 0x30, 0x2a, 0xa3}; - uint8 i_len = 16; - char* dest = (char *) malloc( 2* s_len * sizeof(char)); - ulint dest_len = 0; - my_aes_encrypt_cbc(source, s_len, dest, &dest_len, (unsigned char*) &key, k_len,(unsigned char*) &iv, i_len); - - iv[0] = 0xf1; - //"F1A74007D619455B9EBF8D4F6E302AA3"; - source = (char *) malloc(strlen(MY_AES_TEST_TEXTBLOCK) * sizeof(char)); - my_aes_decrypt_cbc(dest , strlen(dest), source, &dest_len, key, k_len, iv, i_len); - ok(strcmp(source, MY_AES_TEST_TEXTBLOCK) != 0,"Using wrong iv results in wrong decryption."); - free(source); - free(dest); + ec = my_aes_decrypt_cbc(dest , dest_len, result, &dest_len, (unsigned char*) &key, k_len, (unsigned char*) &iv, i_len, 1); + ok((dest_len == s_len) && (ec == AES_OK) && (strncmp(result, "int i = memcmp(decbuf,inbuf,16);",dest_len)==0),"Decrypted text is identical to original text."); + free(result); + free(dest); } void @@ -209,19 +146,19 @@ test_cbc128() unsigned char iv[16] = {0x33, 0x25, 0xcc, 0x3f, 0x02, 0x20, 0x3f, 0xb6, 0xb8, 0x49, 0x99, 0x00, 0x42, 0xe5, 0x8b, 0xcb}; uint8 i_len = 16; - int ec = my_aes_encrypt_cbc(source, s_len, dest, &dest_len, (unsigned char*) &key, k_len,(unsigned char*) &iv, i_len); + int ec = my_aes_encrypt_cbc(source, s_len, dest, &dest_len, (unsigned char*) &key, k_len,(unsigned char*) &iv, i_len, 0); ok(ec == AES_OK, "Checking return code."); ok(memcmp(expected,dest,48)==0, "expected cipher text"); ok(memcmp(source,dest,32)!=0,"plain and cipher text differ"); - ec = my_aes_decrypt_cbc(dest , dest_len, result, &dest_len, (unsigned char*) &key, k_len, (unsigned char*) &iv, i_len); + ec = my_aes_decrypt_cbc(dest , dest_len, result, &dest_len, (unsigned char*) &key, k_len, (unsigned char*) &iv, i_len, 0); ok((dest_len == s_len) && (ec == AES_OK) && (strncmp(result, "int i = memcmp(decbuf,inbuf,16);",dest_len)==0),"Decrypted text is identical to original text."); free(result); free(dest); } void -test_cbc192() +test_cbc192noPadding() { plan(1); @@ -239,10 +176,37 @@ test_cbc192() unsigned char iv[16] = {0x33, 0x25, 0xcc, 0x3f, 0x02, 0x20, 0x3f, 0xb6, 0xb8, 0x49, 0x99, 0x00, 0x42, 0xe5, 0x8b, 0xcb}; uint8 i_len = 16; - int ec = my_aes_encrypt_cbc(source, s_len, dest, &dest_len, (unsigned char*) &key, k_len,(unsigned char*) &iv, i_len); - ok(ec == AES_OK, "Checking return code."); + int ec = my_aes_encrypt_cbc(source, s_len, dest, &dest_len, (unsigned char*) &key, k_len,(unsigned char*) &iv, i_len, 1); + ok (ec == AES_OK, "Checking return code."); + ok (dest_len == s_len, "input length = output length, cbc 192 no padding"); + ec = my_aes_decrypt_cbc(dest , dest_len, result, &dest_len, (unsigned char*) &key, k_len, (unsigned char*) &iv, i_len, 1); + ok((dest_len == s_len) && (ec == AES_OK) && (strncmp(result, "int i = memcmp(decbuf,inbuf,16);",dest_len)==0),"Decrypted text is identical to original text."); + free(result); + free(dest); +} - ec = my_aes_decrypt_cbc(dest , dest_len, result, &dest_len, (unsigned char*) &key, k_len, (unsigned char*) &iv, i_len); +void +test_cbc192() +{ + + plan(1); + int i; + char* source = "int i = memcmp(decbuf,inbuf,16);"; + ulint s_len = strlen(source); + char* dest = (char* ) malloc(100); + char* result = (char*) malloc(100); + + ulint dest_len = 0; + unsigned char key[24] = {0x58, 0x3b, 0xe7, 0xf3, 0x34, 0xf8, + 0x5e, 0x7d, 0x9d, 0xdb, 0x36, 0x2e, 0x9a, 0xc3, 0x81, 0x51, + 0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08}; + uint8 k_len = 24; + unsigned char iv[16] = {0x33, 0x25, 0xcc, 0x3f, 0x02, 0x20, 0x3f, 0xb6, 0xb8, + 0x49, 0x99, 0x00, 0x42, 0xe5, 0x8b, 0xcb}; + uint8 i_len = 16; + int ec = my_aes_encrypt_cbc(source, s_len, dest, &dest_len, (unsigned char*) &key, k_len,(unsigned char*) &iv, i_len, 1); + ok (ec == AES_OK, "Checking return code."); + ec = my_aes_decrypt_cbc(dest , dest_len, result, &dest_len, (unsigned char*) &key, k_len, (unsigned char*) &iv, i_len, 1); ok((dest_len == s_len) && (ec == AES_OK) && (strncmp(result, "int i = memcmp(decbuf,inbuf,16);",dest_len)==0),"Decrypted text is identical to original text."); free(result); free(dest); @@ -272,13 +236,52 @@ test_cbc256() unsigned char iv[16] = {0x33, 0x25, 0xcc, 0x3f, 0x02, 0x20, 0x3f, 0xb6, 0xb8, 0x49, 0x99, 0x00, 0x42, 0xe5, 0x8b, 0xcb}; uint8 i_len = 16; - int ec = my_aes_encrypt_cbc(source, s_len, dest, &dest_len, (unsigned char*) &key, k_len,(unsigned char*) &iv, i_len); + int ec = my_aes_encrypt_cbc(source, s_len, dest, &dest_len, (unsigned char*) &key, k_len,(unsigned char*) &iv, i_len, 0); ok(ec == AES_OK, "Checking return code."); ok(memcmp(expected,dest,48)==0,"Excepted cipher text - aes 256 cbc"); - ec = my_aes_decrypt_cbc(dest , dest_len, result, &dest_len, (unsigned char*) &key, k_len, (unsigned char*) &iv, i_len); + ec = my_aes_decrypt_cbc(dest , dest_len, result, &dest_len, (unsigned char*) &key, k_len, (unsigned char*) &iv, i_len, 0); + + ok((dest_len == s_len) && (ec == AES_OK) && (strncmp(result, "int i = memcmp(decbuf,inbuf,16);",dest_len)==0),"Decrypted text is identical to original text."); + free(result); + free(dest); +} + + +void +test_cbc256noPadding() +{ + char expected[] = { + 0x81, 0x22, 0x05, 0xA7, 0x3E, 0x9D, 0xB2, 0x18, 0x7F, 0xE2, 0x5C, 0xB4, 0xBD, 0xCD, 0xFB, 0x9B, + 0xB6, 0xEF, 0x64, 0x2C, 0xF4, 0x53, 0x9B, 0x29, 0x98, 0x3A, 0xD6, 0xDE, 0xB2, 0x65, 0xEF, 0x85, + 0xEF, 0x4B, 0xDA, 0x8F, 0xD9, 0xEB, 0xD7, 0x07, 0x80, 0x03, 0x0E, 0x7C, 0x55, 0x2E, 0x97, 0x47 + }; + plan(1); + int i; + char* source = "int i = memcmp(decbuf,inbuf,16);"; + ulint s_len = strlen(source); + char* dest = (char* ) malloc(100); + char* result = (char*) malloc(100); + + ulint dest_len = 0; + unsigned char key[32] = {0x58, 0x3b, 0xe7, 0xf3, 0x34, 0xf8, + 0x5e, 0x7d, 0x9d, 0xdb, 0x36, 0x2e, 0x9a, 0xc3, 0x81, 0x51, + 0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08, + 0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08}; + uint8 k_len = 32; + unsigned char iv[16] = {0x33, 0x25, 0xcc, 0x3f, 0x02, 0x20, 0x3f, 0xb6, 0xb8, + 0x49, 0x99, 0x00, 0x42, 0xe5, 0x8b, 0xcb}; + uint8 i_len = 16; + int ec = my_aes_encrypt_cbc(source, s_len, dest, &dest_len, (unsigned char*) &key, k_len,(unsigned char*) &iv, i_len, 1); + + + ok(ec == AES_OK, "Checking return code."); + ok(memcmp(expected,dest,dest_len)==0,"Excepted cipher text - aes 256 cbc"); + ok(s_len==dest_len,"input length = output length, cbc 256 no padding"); + + ec = my_aes_decrypt_cbc(dest , dest_len, result, &dest_len, (unsigned char*) &key, k_len, (unsigned char*) &iv, i_len, 1); ok((dest_len == s_len) && (ec == AES_OK) && (strncmp(result, "int i = memcmp(decbuf,inbuf,16);",dest_len)==0),"Decrypted text is identical to original text."); free(result); @@ -287,8 +290,36 @@ test_cbc256() +void +test_cbc256noPaddingWrongInputSize() +{ + char expected[] = { + 0x81, 0x22, 0x05, 0xA7, 0x3E, 0x9D, 0xB2, 0x18, 0x7F, 0xE2, 0x5C, 0xB4, 0xBD, 0xCD, 0xFB, 0x9B, + 0xB6, 0xEF, 0x64, 0x2C, 0xF4, 0x53, 0x9B, 0x29, 0x98, 0x3A, 0xD6, 0xDE, 0xB2, 0x65, 0xEF, 0x85, + 0xEF, 0x4B, 0xDA, 0x8F, 0xD9, 0xEB, 0xD7, 0x07, 0x80, 0x03, 0x0E, 0x7C, 0x55, 0x2E, 0x97, 0x47 + }; + plan(1); + int i; + char* source = "int i = memcmp(decbuf,inbuf,16);sdfsd"; + ulint s_len = strlen(source); + char* dest = (char* ) malloc(100); + char* result = (char*) malloc(100); + ulint dest_len = 0; + unsigned char key[32] = {0x58, 0x3b, 0xe7, 0xf3, 0x34, 0xf8, + 0x5e, 0x7d, 0x9d, 0xdb, 0x36, 0x2e, 0x9a, 0xc3, 0x81, 0x51, + 0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08, + 0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08}; + uint8 k_len = 32; + unsigned char iv[16] = {0x33, 0x25, 0xcc, 0x3f, 0x02, 0x20, 0x3f, 0xb6, 0xb8, + 0x49, 0x99, 0x00, 0x42, 0xe5, 0x8b, 0xcb}; + uint8 i_len = 16; + int ec = my_aes_encrypt_cbc(source, s_len, dest, &dest_len, (unsigned char*) &key, k_len,(unsigned char*) &iv, i_len, 1); + ok (ec== AES_BAD_DATA, "wrong input size detected"); + free(result); + free(dest); +} @@ -320,7 +351,6 @@ test_bytes_to_key() 0x55, 0x82, 0x0E, 0x54, 0x8F, 0xE4, 0x44, 0xD9}; my_bytes_to_key((unsigned char*) &salt, secret, (unsigned char*) key, (unsigned char*) &iv); - dump_buffer(32, key); ok(memcmp(key, &keyresult, 32) == 0, "BytesToKey key generated successfully."); ok(memcmp(iv, &ivresult, 16) == 0, "BytesToKey iv generated successfully."); // following should ensure, that yassl and openssl calculate the same! @@ -331,20 +361,18 @@ test_bytes_to_key() int main(int argc __attribute__((unused)),char *argv[]) { + test_cbc256noPadding(); + test_cbc192noPadding(); + test_cbc128noPadding(); + test_cbc256noPaddingWrongInputSize(); + test_cbc128(); test_cbc192(); test_cbc256(); test_bytes_to_key(); - /* - test_cbc(); - test_cbc_large(); - test_cbc_keysizes(); - test_cbc_wrong_keylength(); - test_cbc_resultsize(); - test_wrong_key(); - test_bytes_to_key(); - */ + + return 0; } From 4bf6a7c53acf6860b16942e9d1df0c2c300f859d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludger=20G=C3=B6ckel?= Date: Tue, 28 Oct 2014 17:51:26 +0100 Subject: [PATCH 60/70] added unit tests for page_encryption, added support for row_format=compressed --- storage/xtradb/enc/EncKeys.cc | 4 +- storage/xtradb/fil/fil0fil.cc | 12 +- storage/xtradb/fil/fil0pageencryption.cc | 426 ++++++++++---------- storage/xtradb/include/fil0fil.h | 9 + storage/xtradb/include/fil0pageencryption.h | 16 +- unittest/eperi/CMakeLists.txt | 54 ++- unittest/eperi/compressed_6bytes_av | Bin 0 -> 16384 bytes unittest/eperi/compressed_full | Bin 16384 -> 16384 bytes unittest/eperi/eperi_aes-t.cc | 14 +- unittest/eperi/pageenc-t.cc | 77 +++- unittest/eperi/row_format_compactaa | Bin 0 -> 16384 bytes unittest/eperi/row_format_compactab | Bin 0 -> 16384 bytes unittest/eperi/row_format_compactac | Bin 0 -> 16384 bytes unittest/eperi/row_format_compactad | Bin 0 -> 16384 bytes unittest/eperi/row_format_compressedaa | Bin 0 -> 8192 bytes unittest/eperi/row_format_compressedab | Bin 0 -> 8192 bytes unittest/eperi/row_format_compressedac | Bin 0 -> 8192 bytes unittest/eperi/row_format_compressedad | Bin 0 -> 8192 bytes unittest/eperi/row_format_compressedae | Bin 0 -> 8192 bytes unittest/eperi/row_format_compressedaf | Bin 0 -> 8192 bytes unittest/eperi/row_format_compressedag | Bin 0 -> 8192 bytes unittest/eperi/row_format_compressedah | Bin 0 -> 8192 bytes unittest/eperi/row_format_dynamicaa | Bin 0 -> 16384 bytes unittest/eperi/row_format_dynamicab | Bin 0 -> 16384 bytes unittest/eperi/row_format_dynamicac | Bin 0 -> 16384 bytes unittest/eperi/row_format_dynamicad | Bin 0 -> 16384 bytes unittest/eperi/row_format_redundantaa | Bin 0 -> 16384 bytes unittest/eperi/row_format_redundantab | Bin 0 -> 16384 bytes unittest/eperi/row_format_redundantac | Bin 0 -> 16384 bytes unittest/eperi/row_format_redundantad | Bin 0 -> 16384 bytes 30 files changed, 358 insertions(+), 254 deletions(-) create mode 100644 unittest/eperi/compressed_6bytes_av create mode 100644 unittest/eperi/row_format_compactaa create mode 100644 unittest/eperi/row_format_compactab create mode 100644 unittest/eperi/row_format_compactac create mode 100644 unittest/eperi/row_format_compactad create mode 100644 unittest/eperi/row_format_compressedaa create mode 100644 unittest/eperi/row_format_compressedab create mode 100644 unittest/eperi/row_format_compressedac create mode 100644 unittest/eperi/row_format_compressedad create mode 100644 unittest/eperi/row_format_compressedae create mode 100644 unittest/eperi/row_format_compressedaf create mode 100644 unittest/eperi/row_format_compressedag create mode 100644 unittest/eperi/row_format_compressedah create mode 100644 unittest/eperi/row_format_dynamicaa create mode 100644 unittest/eperi/row_format_dynamicab create mode 100644 unittest/eperi/row_format_dynamicac create mode 100644 unittest/eperi/row_format_dynamicad create mode 100644 unittest/eperi/row_format_redundantaa create mode 100644 unittest/eperi/row_format_redundantab create mode 100644 unittest/eperi/row_format_redundantac create mode 100644 unittest/eperi/row_format_redundantad diff --git a/storage/xtradb/enc/EncKeys.cc b/storage/xtradb/enc/EncKeys.cc index dc992a2066df1..c61acc825df8f 100644 --- a/storage/xtradb/enc/EncKeys.cc +++ b/storage/xtradb/enc/EncKeys.cc @@ -158,7 +158,7 @@ void EncKeys::parseSecret( const char *secretfile, char *secret ) { my_bytes_to_key((unsigned char *) salt, _initPwd, key, iv); unsigned long int d_size = 0; int res = my_aes_decrypt_cbc((const char*)buf + 2 * magicSize, bytes_to_read - 2 * magicSize, - secret, &d_size, key, keySize32, iv, ivSize16); + secret, &d_size, key, keySize32, iv, ivSize16, 0); if (d_size>EncKeys::MAX_SECRET_SIZE) { d_size = EncKeys::MAX_SECRET_SIZE; } @@ -331,7 +331,7 @@ char* EncKeys::decryptFile(const char* filename, const char *secret, int *errorC my_bytes_to_key((unsigned char *) salt, secret, key, iv); unsigned long int d_size = 0; int res = my_aes_decrypt_cbc((const char*)buffer + 2 * magicSize, file_size - 2 * magicSize, - decrypted, &d_size, key, keySize32, iv, ivSize16); + decrypted, &d_size, key, keySize32, iv, ivSize16, 0); if(0 != res) { *errorCode = ERROR_FALSE_FILE_KEY; delete[] buffer; buffer = NULL; diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index 9da0d41c256f5..c3325f2225112 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -819,7 +819,7 @@ fil_node_open_file( space->flags); if (fil_page_is_encrypted(page)) { - /* if page is encrypted, write and error and return. + /* if page is encrypted, write an error and return. * Otherwise the server would crash if decrypting is not possible. * This may be the case, if the keyfile could not be opened on server startup. */ @@ -6899,6 +6899,16 @@ fil_space_name( return (space->name); } +/*******************************************************************//** +Return space flags */ +ulint +fil_space_flags( +/*===========*/ + fil_space_t* space) /*!< in: space */ +{ + return (space->flags); +} + /*******************************************************************//** Return page type name */ const char* diff --git a/storage/xtradb/fil/fil0pageencryption.cc b/storage/xtradb/fil/fil0pageencryption.cc index ed5b6251ce310..681b57dc7d91a 100644 --- a/storage/xtradb/fil/fil0pageencryption.cc +++ b/storage/xtradb/fil/fil0pageencryption.cc @@ -27,6 +27,7 @@ this program; if not, write to the Free Software Foundation, Inc., #include "fil0pageencryption.h" #include "fsp0pageencryption.h" #include "my_dbug.h" +#include "page0zip.h" #include "buf0checksum.h" @@ -34,24 +35,56 @@ this program; if not, write to the Free Software Foundation, Inc., #include -/* calculate a 3 byte checksum to verify decryption. One byte is currently needed for other things */ -ulint fil_page_encryption_calc_checksum(unsigned char* buf) { - ulint checksum = 0; - checksum = ut_fold_binary(buf + FIL_PAGE_OFFSET, - FIL_PAGE_FILE_FLUSH_LSN - FIL_PAGE_OFFSET) - + ut_fold_binary(buf + FIL_PAGE_DATA, - UNIV_PAGE_SIZE - FIL_PAGE_DATA - - FIL_PAGE_END_LSN_OLD_CHKSUM); - checksum = checksum & 0x0FFFFFF0UL; - checksum = checksum >> 8; - checksum = checksum & 0x00FFFFFFUL; - - return checksum; +/* + * derived from libFLAC, which is gpl v2 + */ +byte crc_table[] = { + 0x00,0x07,0x0E,0x09,0x1C,0x1B,0x12,0x15,0x38,0x3F,0x36,0x31,0x24,0x23,0x2A,0x2D,0x70,0x77,0x7E,0x79, + 0x6C,0x6B,0x62,0x65,0x48,0x4F,0x46,0x41,0x54,0x53,0x5A,0x5D,0xE0,0xE7,0xEE,0xE9,0xFC,0xFB,0xF2,0xF5, + 0xD8,0xDF,0xD6,0xD1,0xC4,0xC3,0xCA,0xCD,0x90,0x97,0x9E,0x99,0x8C,0x8B,0x82,0x85,0xA8,0xAF,0xA6,0xA1, + 0xB4,0xB3,0xBA,0xBD,0xC7,0xC0,0xC9,0xCE,0xDB,0xDC,0xD5,0xD2,0xFF,0xF8,0xF1,0xF6,0xE3,0xE4,0xED,0xEA, + 0xB7,0xB0,0xB9,0xBE,0xAB,0xAC,0xA5,0xA2,0x8F,0x88,0x81,0x86,0x93,0x94,0x9D,0x9A,0x27,0x20,0x29,0x2E, + 0x3B,0x3C,0x35,0x32,0x1F,0x18,0x11,0x16,0x03,0x04,0x0D,0x0A,0x57,0x50,0x59,0x5E,0x4B,0x4C,0x45,0x42, + 0x6F,0x68,0x61,0x66,0x73,0x74,0x7D,0x7A,0x89,0x8E,0x87,0x80,0x95,0x92,0x9B,0x9C,0xB1,0xB6,0xBF,0xB8, + 0xAD,0xAA,0xA3,0xA4,0xF9,0xFE,0xF7,0xF0,0xE5,0xE2,0xEB,0xEC,0xC1,0xC6,0xCF,0xC8,0xDD,0xDA,0xD3,0xD4, + 0x69,0x6E,0x67,0x60,0x75,0x72,0x7B,0x7C,0x51,0x56,0x5F,0x58,0x4D,0x4A,0x43,0x44,0x19,0x1E,0x17,0x10, + 0x05,0x02,0x0B,0x0C,0x21,0x26,0x2F,0x28,0x3D,0x3A,0x33,0x34,0x4E,0x49,0x40,0x47,0x52,0x55,0x5C,0x5B, + 0x76,0x71,0x78,0x7F,0x6A,0x6D,0x64,0x63,0x3E,0x39,0x30,0x37,0x22,0x25,0x2C,0x2B,0x06,0x01,0x08,0x0F, + 0x1A,0x1D,0x14,0x13,0xAE,0xA9,0xA0,0xA7,0xB2,0xB5,0xBC,0xBB,0x96,0x91,0x98,0x9F,0x8A,0x8D,0x84,0x83, + 0xDE,0xD9,0xD0,0xD7,0xC2,0xC5,0xCC,0xCB,0xE6,0xE1,0xE8,0xEF,0xFA,0xFD,0xF4,0xF3 + +}; + + +byte fil_page_encryption_calc_checksum(unsigned char* buf, ulint len) { + byte crc = 0; + for (ulint i = 0; i < len; i++) + crc = crc_table[(crc ^ buf[i]) & 0xff]; + return crc; } /****************************************************************//** For page encrypted pages encrypt the page before actual write operation. + + Pages are encrypted with AES/CBC/NoPadding algorithm. + "No padding" is used to ensure, that the encrypted page does not exceed the page size. + If "no padding" is used, the input for encryption must be of size (multiple * AES blocksize). AES Blocksize is usually 16 (bytes). + + Since the length of the payload is not a mulitple of the AES blocksize, + and to ensure that every byte of the payload is encrypted, two encryption operations are done. + Each time with a block of adequate size as input. + + Each encrypted page receives a new page type for PAGE_ENCRYPTION. + The original page type (2 bytes) is stored in the Checksum header of the page. + Additionally the encryption key identifier is stored in the Checksum Header. This uses 1 byte. + Checksum verification for encrypted pages is disabled. + + To be able to verify decryption in a later stage, a checksum is stored in the "Old-style Checksum" at the end of the page. + + Since with page_compressed enabled pages, these field does not seem to be availabe, this checksum is only usable, if there is + enough space at the end of the page. + @return encrypted page to be written*/ byte* fil_encrypt_page( @@ -68,30 +101,25 @@ fil_encrypt_page( int err = AES_OK; int key = 0; - ulint remainder = 0; ulint data_size = 0; ulint orig_page_type = 0; ulint write_size = 0; - ib_uint32_t checksum = 0; fil_space_t* space = NULL; byte* tmp_buf = NULL; ulint unit_test = 0; - ibool page_compressed = 0 ; ut_ad(buf);ut_ad(out_buf); key = encryption_key; ulint offset = 0; - byte remaining_byte = 0 ; - - unit_test = 0x01 & mode; + unit_test = mode ? 1 : 0; if (!unit_test) { ut_ad(fil_space_is_page_encrypted(space_id)); - -#ifdef UNIV_DEBUG fil_system_enter(); space = fil_space_get_by_id(space_id); fil_system_exit(); +#ifdef UNIV_DEBUG + fprintf(stderr, "InnoDB: Note: Preparing for encryption for space %lu name %s len %lu\n", space_id, fil_space_name(space), len); @@ -99,22 +127,17 @@ fil_encrypt_page( } - /* data_size -1 bytes will be encrypted at first. - * data_size is the length of the cipher text.*/ - data_size = ((UNIV_PAGE_SIZE - FIL_PAGE_DATA - FIL_PAGE_DATA_END) / MY_AES_BLOCK_SIZE) * MY_AES_BLOCK_SIZE; + byte checksum_byte = fil_page_encryption_calc_checksum(buf + FIL_PAGE_DATA, len - FIL_PAGE_DATA); + + /* data_size bytes will be encrypted at first. + * data_size will be the length of the cipher text since no padding is used.*/ + data_size = ((len - FIL_PAGE_DATA - FIL_PAGE_DATA_END) / MY_AES_BLOCK_SIZE) * MY_AES_BLOCK_SIZE; - /* following number of bytes are not encrypted at first */ - remainder = (UNIV_PAGE_SIZE - FIL_PAGE_DATA - FIL_PAGE_DATA_END) - (data_size - 1); /* read original page type */ orig_page_type = mach_read_from_2(buf + FIL_PAGE_TYPE); - if (orig_page_type == FIL_PAGE_PAGE_COMPRESSED) { - page_compressed = 1; - } - /* calculate a checksum, can be used to verify decryption */ - checksum = fil_page_encryption_calc_checksum(buf); const unsigned char rkey[] = {0xbd, 0xe4, 0x72, 0xa2, 0x95, 0x67, 0x5c, 0xa9, 0x2e, 0x04, 0x67, 0xea, 0xdb, 0xc0, 0xe0, 0x23, @@ -149,49 +172,40 @@ fil_encrypt_page( } } - write_size = data_size; - /* 1st encryption: data_size -1 bytes starting from FIL_PAGE_DATA */ + /* 1st encryption: data_size bytes starting from FIL_PAGE_DATA */ if (err == AES_OK) { - err = my_aes_encrypt_cbc((char*) buf + FIL_PAGE_DATA, data_size - 1, + err = my_aes_encrypt_cbc((char*) buf + FIL_PAGE_DATA, data_size, (char *) out_buf + FIL_PAGE_DATA, &write_size, (const unsigned char *) &rkey, key_len, - (const unsigned char *) &iv, iv_len); + (const unsigned char *) &iv, iv_len, 1); ut_ad(write_size == data_size); + if (err == AES_OK) { + /* copy remaining bytes from input buffer to output buffer. + * Note, that this copies the final 8 bytes of a page, which consists of the + * Old-style checksum and the "Low 32 bits of LSN */ + memcpy(out_buf + FIL_PAGE_DATA + data_size , buf + FIL_PAGE_DATA + data_size , len - FIL_PAGE_DATA -data_size); - if (page_compressed) { - /* page compressed pages: only one encryption. 3 bytes remain unencrypted. 2 bytes are appended to the encrypted buffer. - * one byte is later written to the checksum header. Additionally trailer remains unencrypted (8 bytes). - */ - offset = 1; - } - /* copy remaining bytes to output buffer */ - memcpy(out_buf + FIL_PAGE_DATA + data_size, buf + FIL_PAGE_DATA + data_size - 1, - remainder - offset); - - if (page_compressed ) { - remaining_byte = mach_read_from_1(buf + FIL_PAGE_DATA + data_size +1); - } else { //create temporary buffer for 2nd encryption tmp_buf = static_cast(ut_malloc(64)); /* 2nd encryption: 63 bytes from out_buf, result length is 64 bytes */ - err = my_aes_encrypt_cbc((char*)out_buf + UNIV_PAGE_SIZE -FIL_PAGE_DATA_END -62, - 63, + err = my_aes_encrypt_cbc((char*)out_buf + len -offset -64, + 64, (char*)tmp_buf, &write_size, (const unsigned char *)&rkey, key_len, (const unsigned char *)&iv, - iv_len); + iv_len, 1); ut_ad(write_size == 64); - /* copy 62 bytes from 2nd encryption to out_buf, last 2 bytes are copied later to a header field*/ - memcpy(out_buf + UNIV_PAGE_SIZE - FIL_PAGE_DATA_END -62, tmp_buf, 62); - + /* copy 64 bytes from 2nd encryption to out_buf*/ + memcpy(out_buf + len - offset -64, tmp_buf, 64); } + } /* error handling */ if (err != AES_OK) { - /* If error we leave the actual page as it was */ + /* If an error occurred we leave the actual page as it was */ fprintf(stderr, "InnoDB: Warning: Encryption failed for space %lu name %s len %lu rt %d write %lu, error: %d\n", @@ -208,43 +222,17 @@ fil_encrypt_page( return (buf); } - /* set up the trailer.*/ - memcpy(out_buf + (UNIV_PAGE_SIZE -FIL_PAGE_DATA_END), - buf + (UNIV_PAGE_SIZE - FIL_PAGE_DATA_END), FIL_PAGE_DATA_END); /* Set up the page header. Copied from input buffer*/ memcpy(out_buf, buf, FIL_PAGE_DATA); - ulint compressed_size = mach_read_from_2(buf+ FIL_PAGE_DATA); - /* checksum */ - if (!page_compressed) { - /* Set up the checksum. This is only usable to verify decryption */ - mach_write_to_3(out_buf + UNIV_PAGE_SIZE - FIL_PAGE_DATA_END, checksum); - } else { - ulint pos_checksum = UNIV_PAGE_SIZE - FIL_PAGE_DATA_END; - if (compressed_size + FIL_PAGE_DATA > pos_checksum) { - pos_checksum = compressed_size + FIL_PAGE_DATA; - if (pos_checksum > UNIV_PAGE_SIZE - 3) { - // checksum not supported, because no space available - } else { - /* Set up the checksum. This is only usable to verify decryption */ - mach_write_to_3(out_buf + pos_checksum, checksum); - } - } else { - mach_write_to_3(out_buf + UNIV_PAGE_SIZE - FIL_PAGE_DATA_END, checksum); - } - } - /* Set up the correct page type */ mach_write_to_2(out_buf + FIL_PAGE_TYPE, FIL_PAGE_PAGE_ENCRYPTED); - /* checksum fields are used to store original page type, etc. - * checksum check for page encrypted pages is omitted. PAGE_COMPRESSED pages does not seem to have a - * Old-style checksum trailer, therefore this field is only used, if there is space. Payload length is expected as - * two byte value at position FIL_PAGE_DATA */ - + /* The 1st checksum field is used to store original page type, etc. + * checksum check for page encrypted pages is omitted. /* Set up the encryption key. Written to the 1st byte of the checksum header field. This header is currently used to store data. */ mach_write_to_1(out_buf, key); @@ -252,14 +240,8 @@ fil_encrypt_page( /* store original page type. Written to 2nd and 3rd byte of the checksum header field */ mach_write_to_2(out_buf + 1, orig_page_type); - /* write remaining bytes to checksum header byte 4 and old style checksum byte 4 */ - if (!page_compressed) { - memcpy(out_buf + 3, tmp_buf + 62, 1); - memcpy(out_buf + UNIV_PAGE_SIZE - FIL_PAGE_DATA_END +3 , tmp_buf + 63, 1); - } else { - /* if page is compressed, only one byte must be placed */ - memset(out_buf + 3, remaining_byte, 1); - } + /* set byte 4 of checksum field to checksum byte */ + memset(out_buf + 3, checksum_byte, 1); #ifdef UNIV_DEBUG /* Verify */ @@ -268,7 +250,7 @@ fil_encrypt_page( #endif /* UNIV_DEBUG */ srv_stats.pages_page_encrypted.inc(); - *out_len = UNIV_PAGE_SIZE; + *out_len = len; /* free temporary buffer */ if (tmp_buf!=NULL) { @@ -280,14 +262,22 @@ fil_encrypt_page( /****************************************************************//** For page encrypted pages decrypt the page after actual read operation. + + See fil_encrypt_page for details. + + if the decryption can be verified, original page should be completely restored, alongside. This includes original page type, checksum fields at page start and end. + If the verification fails, an error is raised. + If page_compressed is active it may not be possible to verify decryption, since the verification checksum can be missing. In this + case the verification result is not evaluated. + @return decrypted page */ ulint fil_decrypt_page( /*================*/ byte* page_buf, /*!< in: preallocated buffer or NULL */ byte* buf, /*!< in/out: buffer from which to read; in aio this must be appropriately aligned */ - ulint len, /*!< in: length of output buffer.*/ - ulint* write_size, /*!< in/out: Actual payload size of the decrypted data. */ + ulint len, /*!< in: length buffer, which should be decrypted.*/ + ulint* write_size, /*!< out: size of the decrypted data. If no error occurred equal to len */ ibool* page_compressed, /*!(ut_malloc(64)); - tmp_page_buf = static_cast(ut_malloc(UNIV_PAGE_SIZE)); - memset(tmp_page_buf,0, UNIV_PAGE_SIZE); + tmp_page_buf = static_cast(ut_malloc(len)); + memset(tmp_page_buf,0, len); const unsigned char rkey[] = {0xbd, 0xe4, 0x72, 0xa2, 0x95, 0x67, 0x5c, 0xa9, 0x2e, 0x04, 0x67, 0xea, 0xdb, 0xc0,0xe0, 0x23, @@ -403,7 +391,7 @@ ulint fil_decrypt_page( } if (err != AES_OK) { - /* surely key could not be fetched */ + /* surely key could not be determined. */ fprintf(stderr, "InnoDB: Corruption: Page is marked as encrypted\n" "InnoDB: but decrypt failed with error %d, encryption key %d.\n", err, (int)page_decryption_key); @@ -415,50 +403,49 @@ ulint fil_decrypt_page( } - if (!page_compression_flag) { - tmp_page_buf = static_cast(ut_malloc(UNIV_PAGE_SIZE)); - tmp_buf= static_cast(ut_malloc(64)); - memset(tmp_page_buf,0, UNIV_PAGE_SIZE); + tmp_page_buf = static_cast(ut_malloc(len)); + tmp_buf= static_cast(ut_malloc(64)); + memset(tmp_page_buf,0, len); - /* 1st decryption: 64 bytes */ - /* 62 bytes from data area and 2 bytes from header are copied to temporary buffer */ - memcpy(tmp_buf, buf + UNIV_PAGE_SIZE - FIL_PAGE_DATA_END - 62, 62); - memcpy(tmp_buf + 62, buf + FIL_PAGE_SPACE_OR_CHKSUM + 3, 1); - memcpy(tmp_buf + 63, buf + UNIV_PAGE_SIZE - FIL_PAGE_DATA_END +3, 1); + /* 1st decryption: 64 bytes */ + /* 64 bytes from data area are copied to temporary buffer. + * These are the last 64 of the (encrypted) payload */ + memcpy(tmp_buf, buf + len - offset - 64, 64); + if (err == AES_OK) { + err = my_aes_decrypt_cbc((const char*) tmp_buf, 64, + (char *) tmp_page_buf + len - offset - 64, + &tmp_write_size, (const unsigned char *) &rkey, key_len, + (const unsigned char *) &iv, iv_len, 1); + } + ut_ad(tmp_write_size == 64); - if (err == AES_OK) { - err = my_aes_decrypt_cbc((const char*) tmp_buf, 64, - (char *) tmp_page_buf + UNIV_PAGE_SIZE - FIL_PAGE_DATA_END - 62, - &tmp_write_size, (const unsigned char *) &rkey, key_len, - (const unsigned char *) &iv, iv_len); - } - /* If decrypt fails it means that page is corrupted or has an unknown key */ - if (err != AES_OK) { - fprintf(stderr, "InnoDB: Corruption: Page is marked as encrypted\n" - "InnoDB: but decrypt failed with error %d.\n" - "InnoDB: size %lu len %lu, key %d\n", err, data_size, - len, (int)page_decryption_key); - fflush(stderr); - if (NULL == page_buf) { - ut_free(in_buffer); - } - return err; + /* If decrypt fails it means that page is corrupted or has an unknown key */ + if (err != AES_OK) { + fprintf(stderr, "InnoDB: Corruption: Page is marked as encrypted\n" + "InnoDB: but decrypt failed with error %d.\n" + "InnoDB: size %lu len %lu, key %d\n", err, data_size, + len, (int)page_decryption_key); + fflush(stderr); + if (NULL == page_buf) { + ut_free(in_buffer); } + return err; + } - ut_ad(tmp_write_size == 63); + ut_ad(tmp_write_size == 64); - /* copy 1st part from buf to tmp_page_buf */ - /* do not override result of 1st decryption */ - memcpy(tmp_page_buf + FIL_PAGE_DATA, buf + FIL_PAGE_DATA, data_size - 60); - memset(in_buf, 0, UNIV_PAGE_SIZE); + /* copy 1st part of payload from buf to tmp_page_buf */ + /* do not override result of 1st decryption */ + memcpy(tmp_page_buf + FIL_PAGE_DATA, buf + FIL_PAGE_DATA, len -offset -64 - FIL_PAGE_DATA); + + + /* fill target buffer with zeros */ + memset(in_buf, 0, len); - } else { - tmp_page_buf = buf; - } err = my_aes_decrypt_cbc((char*) tmp_page_buf + FIL_PAGE_DATA, data_size, @@ -467,45 +454,25 @@ ulint fil_decrypt_page( (const unsigned char *)&rkey, key_len, (const unsigned char *)&iv, - iv_len - ); - ut_ad(tmp_write_size = data_size-1); - - memcpy(in_buf + FIL_PAGE_DATA + data_size -1, tmp_page_buf + UNIV_PAGE_SIZE - FIL_PAGE_DATA_END - 2, remainder - offset); - if (page_compression_flag) { - /* the last byte was stored in position 4 of the checksum header */ - memcpy(in_buf + FIL_PAGE_DATA + data_size -1 + 2, tmp_page_buf+ FIL_PAGE_SPACE_OR_CHKSUM + 3, 1); - } + iv_len, + 1); + ut_ad(tmp_write_size = data_size); + + /* copy remaining bytes from tmp_page_buf to in_buf. + */ + ulint bytes_to_copy = len - FIL_PAGE_DATA - data_size - offset; + memcpy(in_buf + FIL_PAGE_DATA + data_size, tmp_page_buf + FIL_PAGE_DATA + data_size, bytes_to_copy); + + /* apart from header data everything is now in in_buf */ - /* compare with stored checksum */ - ulint compressed_size = mach_read_from_2(in_buf+ FIL_PAGE_DATA); - ibool no_checksum_support = 0; - ulint pos_checksum = 0; - if (!page_compression_flag) { - /* Read the checksum. This is only usable to verify decryption */ - stored_checksum = mach_read_from_3(buf + UNIV_PAGE_SIZE - FIL_PAGE_DATA_END); - } else { - pos_checksum = UNIV_PAGE_SIZE - FIL_PAGE_DATA_END; - if (compressed_size + FIL_PAGE_DATA > pos_checksum) { - pos_checksum = compressed_size + FIL_PAGE_DATA; - if (pos_checksum > UNIV_PAGE_SIZE - 3) { - // checksum not supported, because no space available - no_checksum_support = 1; - } else { - /* Read the checksum. This is only usable to verify decryption */ - stored_checksum = mach_read_from_3(buf + pos_checksum); - } - } else { - /* Read the checksum. This is only usable to verify decryption */ - stored_checksum = mach_read_from_3(buf + UNIV_PAGE_SIZE - FIL_PAGE_DATA_END); - } - } - if (!page_compression_flag) { - ut_free(tmp_page_buf); - ut_free(tmp_buf); - } + + + + ut_free(tmp_page_buf); + ut_free(tmp_buf); + #ifdef UNIV_PAGEENCRIPTION_DEBUG @@ -516,56 +483,69 @@ ulint fil_decrypt_page( /* copy header */ memcpy(in_buf, buf, FIL_PAGE_DATA); - /* copy trailer */ - memcpy(in_buf + UNIV_PAGE_SIZE - FIL_PAGE_DATA_END, - buf + UNIV_PAGE_SIZE - FIL_PAGE_DATA_END, FIL_PAGE_DATA_END); /* Copy the decrypted page to the buffer pool*/ - memcpy(buf, in_buf, UNIV_PAGE_SIZE); + memcpy(buf, in_buf, len); /* setting original page type */ mach_write_to_2(buf + FIL_PAGE_TYPE, orig_page_type); - /* calculate a checksum to verify decryption */ - checksum = fil_page_encryption_calc_checksum(buf); - if (no_checksum_support) { - fprintf(stderr, "InnoDB: decrypting page can not be verified!\n"); - fflush(stderr); + ulint pageno = mach_read_from_4(buf + FIL_PAGE_OFFSET); + ulint flags = 0; + ulint zip_size = 0; + if (pageno == 0 ) { + flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + buf); } else { - if ((stored_checksum != checksum)) { - err = PAGE_ENCRYPTION_WRONG_KEY; - fprintf(stderr, "InnoDB: Corruption: Page is marked as encrypted.\n" - "InnoDB: Checksum mismatch after decryption.\n" - "InnoDB: size %lu len %lu, key %d\n", data_size, - len, (int)page_decryption_key); - fflush(stderr); - // Need to free temporal buffer if no buffer was given - if (NULL == page_buf) { - ut_free(in_buffer); + if (unit_test) { + /* in simple unit test, the tablespace memory hash is n.a. */ + if ((mode & 0x01) != 0x01) { + zip_size = mode; } - return err; + } else { + ulint space_id = mach_read_from_4(buf + FIL_PAGE_ARCH_LOG_NO_OR_SPACE_ID); + fil_system_enter(); + space = fil_space_get_by_id(space_id); + flags = fil_space_flags(space); + fil_system_exit(); } } + if (!unit_test || pageno==0) { + zip_size = fsp_flags_get_zip_size(flags); + } + + if (write_size!=NULL) { + *write_size = len; + } - if (!(page_compression_flag )) { + + byte checksum_byte = fil_page_encryption_calc_checksum(buf + FIL_PAGE_DATA, len - FIL_PAGE_DATA); + if (checksum_byte != stored_checksum_byte) { + err = PAGE_ENCRYPTION_WRONG_KEY; + fprintf(stderr, "InnoDB: Corruption: Page is marked as encrypted\n" + "InnoDB: but decryption verification failed with error %d, encryption key %d.\n", + err, (int)page_decryption_key); + fflush(stderr); + + return err; + } + + if (!(page_compression_flag)) { /* calc check sums and write to the buffer, if page was not compressed. * if the decryption is verified, it is assumed that the original page was restored, re-calculating the original * checksums should be ok */ - do_check_sum(UNIV_PAGE_SIZE, buf); + do_check_sum(len, zip_size, buf); } else { /* page_compression uses BUF_NO_CHECKSUM_MAGIC as checksum */ mach_write_to_4(buf + FIL_PAGE_SPACE_OR_CHKSUM, BUF_NO_CHECKSUM_MAGIC); - if (!no_checksum_support) { - /* reset the checksum bytes - if used */ - memset(buf + pos_checksum, 0, 3); - } - } + + + srv_stats.pages_page_decrypted.inc(); return err; } @@ -574,9 +554,20 @@ ulint fil_decrypt_page( /* recalculate check sum - from buf0flu.cc*/ void do_check_sum( ulint page_size, + ulint zip_size, byte* buf) { ib_uint32_t checksum = 0; - switch ((srv_checksum_algorithm_t) srv_checksum_algorithm) { + + if (zip_size) { + checksum = page_zip_calc_checksum(buf,zip_size, + static_cast( + srv_checksum_algorithm)); + + mach_write_to_4(buf + FIL_PAGE_SPACE_OR_CHKSUM, checksum); + return; + } + + switch ((srv_checksum_algorithm_t) srv_checksum_algorithm) { case SRV_CHECKSUM_ALGORITHM_CRC32: case SRV_CHECKSUM_ALGORITHM_STRICT_CRC32: checksum = buf_calc_page_crc32(buf); @@ -594,20 +585,7 @@ void do_check_sum( is added and not handled here */ } mach_write_to_4(buf + FIL_PAGE_SPACE_OR_CHKSUM, checksum); - /* We overwrite the first 4 bytes of the end lsn field to store - the old formula checksum. Since it depends also on the field - FIL_PAGE_SPACE_OR_CHKSUM, it has to be calculated after storing the - new formula checksum. */ - if (srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_STRICT_INNODB - || srv_checksum_algorithm == SRV_CHECKSUM_ALGORITHM_INNODB) { - checksum = (ib_uint32_t) (buf_calc_page_old_checksum(buf)); - /* In other cases we use the value assigned from above. - If CRC32 is used then it is faster to use that checksum - (calculated above) instead of calculating another one. - We can afford to store something other than - buf_calc_page_old_checksum() or BUF_NO_CHECKSUM_MAGIC in - this field because the file will not be readable by old - versions of MySQL/InnoDB anyway (older than MySQL 5.6.3) */ - } - mach_write_to_4(buf + page_size - FIL_PAGE_END_LSN_OLD_CHKSUM, checksum); + + /* old style checksum is omitted */ + } diff --git a/storage/xtradb/include/fil0fil.h b/storage/xtradb/include/fil0fil.h index ba9852fdaf462..718047368a7c0 100644 --- a/storage/xtradb/include/fil0fil.h +++ b/storage/xtradb/include/fil0fil.h @@ -1115,6 +1115,15 @@ fil_space_name( fil_space_t* space); /*!< in: space */ #endif +/*******************************************************************//** +Return space flags */ +ulint +fil_space_flags( +/*===========*/ + fil_space_t* space); /*!< in: space */ + + + /****************************************************************//** Does error handling when a file operation fails. @return TRUE if we should retry the operation */ diff --git a/storage/xtradb/include/fil0pageencryption.h b/storage/xtradb/include/fil0pageencryption.h index 89d063870c996..5fd4f5b4974b4 100644 --- a/storage/xtradb/include/fil0pageencryption.h +++ b/storage/xtradb/include/fil0pageencryption.h @@ -84,19 +84,19 @@ operation. ulint fil_decrypt_page( /*================*/ - byte* page_buf, /*!< in: preallocated buffer or NULL */ - byte* buf, /*!< out: buffer from which to read; in aio + byte* page_buf, /*!< in: preallocated buffer or NULL */ + byte* buf, /*!< out: buffer from which to read; in aio this must be appropriately aligned */ - ulint len, /*!< in: length of output buffer.*/ - ulint* write_size, /*!< in/out: Actual payload size of the decrypted data. */ - ibool* page_compressed, /*!2l|aj+low6S=E@XJ zv6;s#gG3T^G-#Da&eoL7tI<+Wie_42F6WB!YoCMo|35s>|JwsUc(e}`O9Vk|F>U5o zCkQ`of2r^9Mi6EMi|7d-D`@;QBagO^u^(LalUH7Ow<60toZaPN?O!9@X~3|lhYc%cO-xE9&KCLC7h4WoJa|i?9xcv^H41m#i*+-;4Xo!W4EK#0`*csM6J>pd z*rtA2*_qrZ6JzQ~voOuI*Kx%E#%F{myy~C;2-$x5gK5*A{nJi4zi1M|{%g zNhdpehi4il^kn$mN6qQyoE__ut&)5VCvTg~CbsAV70GOm_qyQdQW8VmV24~9 zrG1oe;uhPvHRdS>YhGq#X~g*`TYia8&mir#hI5TCEMrd=k@4jWT`yN;OHG^cy2#RF zQhCKIcbZu1t6UvoUe&t`>)0yADQbL?Ip82oWX=|nlNAh|f-7>^dU=R8qv5FdLNQyl z9QZADPY_kg{c5RPw;(607`kq*$c{1`A`5I?gkKhV*>UD6Iy+wDbLz7?I+tP!X`H_e z<3d2tn-?Sgp@N+CfOWNLhfF?4Z$tiHfz2<+f&c^{009U<00Izz00bZa0SG_<0uX=z z1Rwwb2tWV=5P$##AOHafKmY;|fB*y_009U<00Izz00bZa0SG_<0uX=z1Rwwb2tWV= O5P$##{-uEF&+(t;VKBV_ literal 0 HcmV?d00001 diff --git a/unittest/eperi/compressed_full b/unittest/eperi/compressed_full index 5c9ffc29c60f712c35e68ac203633d7639a3f1c8..9404efe4cce4a6c44fbac22187d51701b5e00417 100644 GIT binary patch delta 14 VcmZo@U~Fh$oS@EleWS(=djKaG1%dzo delta 14 VcmZo@U~Fh$oS@ElbEC!$djKaQ1%v> 24); + b[1] = (byte)(n >> 16); + b[2] = (byte)(n >> 8); + b[3] = (byte) n; +} ulint mach_read_from_2( /*=============*/ @@ -98,10 +111,11 @@ return buffer; } void testEncryptionChecksum(char* filename) { - byte* buf = readFile(filename,NULL); + int fl = 0; + byte* buf = readFile(filename,&fl); byte* dest = (byte *) malloc(16384*sizeof(byte)); ulint out_len; - fil_encrypt_page(0,buf,dest,0,255, &out_len, 1); + fil_encrypt_page(0,buf,dest,fl,255, &out_len, 1); dest[2000]=0xFF; dest[2001]=0xFF; dest[2002]=0xFF; @@ -119,13 +133,17 @@ void testEncryptionChecksum(char* filename) { } void testIt(char* filename, ulint do_not_cmp_checksum) { - byte* buf = readFile(filename, NULL); + int fl = 0; + byte* buf = readFile(filename, &fl); + + byte* dest = (byte *) malloc(16384*sizeof(byte)); + ulint out_len; ulint cc1 = 0; ulint orig_page_type = mach_read_from_2(buf + 24); - byte* snd = fil_encrypt_page(0,buf,dest,0,255, &out_len, 1); + byte* snd = fil_encrypt_page(0,buf,dest,fl,255, &out_len, fl==8192 ? fl : 1); cc1 = (buf!=dest); cc1 = cc1 && (snd==dest); if (!do_not_cmp_checksum) { @@ -134,25 +152,34 @@ void testIt(char* filename, ulint do_not_cmp_checksum) { /* 255 is the key used for unit test */ cc1 = cc1 && (mach_read_from_1(dest) == 255); } - ulint result = fil_decrypt_page(NULL, dest, 16384 ,NULL,NULL, 1); - cc1 = result == 0; + ulint write_size = 0; + ulint result = fil_decrypt_page(NULL, dest, out_len,&write_size,NULL, out_len==8192 ? out_len : 1); + cc1 = (result == 0) && (write_size==fl); ulint a = 0; ulint b = 0; if (do_not_cmp_checksum) { a = 4; - b = 8; + b = 0; } - ulint i = memcmp((buf + a),(dest +a), (size_t)(16384 - (a+b))); + ulint i = memcmp((buf + a),(dest +a), (size_t)(write_size - (a+b))); char str[80]; strcpy (str,"File "); strcat (str,filename ); + cc1 = (i==0) && (cc1); + if (!cc1) { + dump_buffer(fl, buf); + + dump_buffer(write_size, dest); + } - ok (i==0 && cc1, "%s", (char*) str); + ok (cc1, "%s", (char*) str); } void test_page_enc_dec() { char compressed[] = "compressed"; char compressed_full[] = "compressed_full"; + char compressed_6bytes_av[] = "compressed_6bytes_av"; + char xaa[] = "xaa"; char xab[] = "xab"; char xac[] = "xac"; @@ -160,15 +187,40 @@ void test_page_enc_dec() { char xae[] = "xae"; char xaf[] = "xaf"; + + + testIt("row_format_compressedaa", 0); + testIt("row_format_compressedab", 0); + testIt("row_format_compressedac", 0); + testIt("row_format_compressedad", 0); + + testIt("row_format_dynamicaa", 0); + testIt("row_format_dynamicab", 0); + testIt("row_format_dynamicac", 0); + testIt("row_format_dynamicad", 0); + + testIt("row_format_redundantaa", 0); + testIt("row_format_redundantab", 0); + testIt("row_format_redundantac", 0); + testIt("row_format_redundantad", 0); + + testIt("row_format_compactaa", 0); + testIt("row_format_compactab", 0); + testIt("row_format_compactac", 0); + testIt("row_format_compactad", 0); + + testIt(compressed,0); testIt(compressed_full,0); + testIt(compressed_6bytes_av,0); + testIt(xaa,0); testIt(xab,0); testIt(xac,0); testIt(xad,0); - +// empty pages testIt(xae,1); testIt(xaf,1); @@ -230,6 +282,7 @@ void testSecret256_PlainFile() { } void testSecrets() { + testShortSecret_EncryptedFile(); testShortSecret_PlainFile(); testLongSecret_PlainFile(); @@ -241,9 +294,11 @@ void testSecrets() { int main() { + + testSecrets(); test_page_enc_dec(); - testEncryptionChecksum((char* )"xaa"); + testEncryptionChecksum((char* )"xaa"); return 0; } diff --git a/unittest/eperi/row_format_compactaa b/unittest/eperi/row_format_compactaa new file mode 100644 index 0000000000000000000000000000000000000000..9a18f6a204284afc2e6f9921392cbf2a10bdb0c0 GIT binary patch literal 16384 zcmeI(u?@m75CzanK!}v5SDDV8AhF&}g literal 0 HcmV?d00001 diff --git a/unittest/eperi/row_format_compactab b/unittest/eperi/row_format_compactab new file mode 100644 index 0000000000000000000000000000000000000000..aa80ac8567f2cd8c7f243179e1dd9d86a37c9418 GIT binary patch literal 16384 zcmeI(F%5t~5Cp)bzyL-NfC}`GP>{SX6rceG)Zg;eV}Notxn!@Me$TCzBtyF|%d=+e z@07cDg#ZBp1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N j0t5&UAV7cs0RjXF5FkK+009C72oU&D;M$KlZRh_2oJ$Ct literal 0 HcmV?d00001 diff --git a/unittest/eperi/row_format_compactac b/unittest/eperi/row_format_compactac new file mode 100644 index 0000000000000000000000000000000000000000..e740ed5e07640622da7b17af5e7444aeba3e43df GIT binary patch literal 16384 zcmeI(yA8rH5CG7Fhy}6+5(}UdjFT>1#$XG^UU9% zZ*eygAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ g009C72oNAZfB*pk1PBlyK!CtM2=tfZHPS!-4;RBe-2eap literal 0 HcmV?d00001 diff --git a/unittest/eperi/row_format_compactad b/unittest/eperi/row_format_compactad new file mode 100644 index 0000000000000000000000000000000000000000..7b4f3a73c044a51b35141666ebc135f33b5d6abe GIT binary patch literal 16384 zcmeI(y=p>15C+g$ja3?nm87zkG-*>?r}P0r1R?&Q*jR|Ywcy+2QPNAQ+sn=FMZ{+a zXN%cyhGmM=@B8lMHzL~g_2eA!b~%r}n^ekE{6s5``*t(gMJL~>$#*5C=3V5M)h)fI z+&)j|lj&?T%keQ{y;-h4OBd0ttFmBQ%ENGQ%3HrucOR{BnK%Rp5FkK+009C72oNAZ zfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&U SAVA>13oPSfap?ETKmR}N=pq&X literal 0 HcmV?d00001 diff --git a/unittest/eperi/row_format_compressedaa b/unittest/eperi/row_format_compressedaa new file mode 100644 index 0000000000000000000000000000000000000000..793f51085a076dd1188317e9d61f789b9f905457 GIT binary patch literal 8192 zcmeIu!3{t_5QX8fM4a6;AaQndQHT5Xq6!75Krwn@hFCH!_$QgS+1c64H&1;OU&Gc& zm4>zKVtrSs$Ee+D7us@YJ*n5sX0p8L-t8f`aC7??q)llnekZK<=~LGdfdB#sAb{4FUuR l5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAVA8^c?$0_J literal 0 HcmV?d00001 diff --git a/unittest/eperi/row_format_compressedad b/unittest/eperi/row_format_compressedad new file mode 100644 index 0000000000000000000000000000000000000000..7bbfd6045d7ee3a4fee9094fb22df17657ff5346 GIT binary patch literal 8192 zcmeI%Ee^s!5QgEeKUhLg+@QV|g{bEUNUl)Rw7PnehN~dB2nvCy>;YJ2CxoEU?}*ti zlNs~aosYXoQV~C8S=wKl=H@B2t?tSd`obrjR`zC|f7{FVWSh%3)#u?6o~P3GVzl#% z>fCQy^->yB2q1s}0tg_000IagfB*srAbO^S*=I= J!armib#EHB8@>Pl literal 0 HcmV?d00001 diff --git a/unittest/eperi/row_format_compressedae b/unittest/eperi/row_format_compressedae new file mode 100644 index 0000000000000000000000000000000000000000..6d17cf9d15fb9f4a2358a2d079f3b8c755d005fa GIT binary patch literal 8192 zcmeIu0Sy2E0K%a6Pi+o2h(KY$fB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM GyblZ@00031 literal 0 HcmV?d00001 diff --git a/unittest/eperi/row_format_compressedaf b/unittest/eperi/row_format_compressedaf new file mode 100644 index 0000000000000000000000000000000000000000..6d17cf9d15fb9f4a2358a2d079f3b8c755d005fa GIT binary patch literal 8192 zcmeIu0Sy2E0K%a6Pi+o2h(KY$fB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM GyblZ@00031 literal 0 HcmV?d00001 diff --git a/unittest/eperi/row_format_compressedag b/unittest/eperi/row_format_compressedag new file mode 100644 index 0000000000000000000000000000000000000000..6d17cf9d15fb9f4a2358a2d079f3b8c755d005fa GIT binary patch literal 8192 zcmeIu0Sy2E0K%a6Pi+o2h(KY$fB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM GyblZ@00031 literal 0 HcmV?d00001 diff --git a/unittest/eperi/row_format_compressedah b/unittest/eperi/row_format_compressedah new file mode 100644 index 0000000000000000000000000000000000000000..6d17cf9d15fb9f4a2358a2d079f3b8c755d005fa GIT binary patch literal 8192 zcmeIu0Sy2E0K%a6Pi+o2h(KY$fB^#r3>YwAz<>b*1`HT5V8DO@0|pEjFkrxd0RsjM GyblZ@00031 literal 0 HcmV?d00001 diff --git a/unittest/eperi/row_format_dynamicaa b/unittest/eperi/row_format_dynamicaa new file mode 100644 index 0000000000000000000000000000000000000000..66256ed2601b4e77e21098fa4f6b7ee3fbc2c0cb GIT binary patch literal 16384 zcmeI(u?Ye}5Czcj6cNnK3{3=UFfqAGEX6)7z-~+g1M4u<8O2k!1K|zKUv73bdHHP) z%YC%9z20Ls3>R^W>UT}^*k!M>C)x4H`ec5NY8J>)mUTso$6FJ~NNE zVKBP@{uiGn#&23y1lqDUJFy_0O~{E^SE_t4*( zh&ZOU#^pIiwWQ~GO(`Ni{XBmt>%VJT`RQ>T(zz<{*(>I{mlx+czTdW$=lb`myWCsc zjRXh~AV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ e009C72oNAZfB*pk1PBly@DBoAeQzWE^M38Hp|OAk{CrC z%3{B1CPjb%0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ l009C72oNAZfB*pk1PBlyK!5-N0tEhDposNzG5a+C{J(h09-IIG literal 0 HcmV?d00001 diff --git a/unittest/eperi/row_format_redundantaa b/unittest/eperi/row_format_redundantaa new file mode 100644 index 0000000000000000000000000000000000000000..66dfbd857b65740c41b404cb31d50818567caabc GIT binary patch literal 16384 zcmeI(u?@m75CzanK!}(-aDkMzE+M#`E4favatjfXDlTC84nEv(Uw_Mblq6HXFMEA6kGJF+ zT_He#009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+009C72oNAZfB*pk f1PBlyK!5-N0t5&UAV7cs0RjYm6xh!DN#FT5&&>$B literal 0 HcmV?d00001 diff --git a/unittest/eperi/row_format_redundantac b/unittest/eperi/row_format_redundantac new file mode 100644 index 0000000000000000000000000000000000000000..4fe6880bbf7dc064bc9be2c50f66f4fc5d40e82d GIT binary patch literal 16384 zcmeI(yA8rH5CG7Fh=`hXSRq?bB_m}I#151clnlTU!B*mcQlyQ9-buD~{>bOo`_>=t z5wTD08|QZy)tsKG=NIQXe&4p1=lZv*yWCsc zjRXh~AV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF5FkK+ e009C72oNAZfB*pk1PBly@DBoYbGb(P=l=m-fIhAO literal 0 HcmV?d00001 diff --git a/unittest/eperi/row_format_redundantad b/unittest/eperi/row_format_redundantad new file mode 100644 index 0000000000000000000000000000000000000000..914f69b78bfdc349e662ec4b1ba8fcdcc6bdbe85 GIT binary patch literal 16384 zcmeI(Jqp4=5C+hRL5Qu`c>sG`YmX8kLPShbdNy0J^Z@o+dI8N};~9J(?9S{m+q`~P z*X1H2($AQ-kJ!BS+e_bd+dLjIiF3P#Ovn0Dl3^~|sdxXP>(*I&okm?8c178gF-sy# zBdX?j%Db}=AV7cs0RjXF5FkK+009C72oNAZfB*pk1PBlyK!5-N0t5&UAV7cs0RjXF n5FkK+009C72oNAZfB*pk1PBlyK!5;&e-@}CuWrwK_s{ Date: Wed, 29 Oct 2014 16:47:16 +0100 Subject: [PATCH 61/70] page type 8 and 9 are not encrypted anymore to support recovery for row_format=compressed --- storage/xtradb/enc/EncKeys.cc | 42 ++++++++++ storage/xtradb/fil/fil0fil.cc | 12 +-- storage/xtradb/fil/fil0pageencryption.cc | 82 +++++++++++++------- storage/xtradb/include/fil0pageencryption.h | 25 ++++-- storage/xtradb/include/fsp0pageencryption.ic | 47 +++++++++++ storage/xtradb/os/os0file.cc | 8 +- unittest/eperi/pageenc-t.cc | 22 ++++-- 7 files changed, 190 insertions(+), 48 deletions(-) diff --git a/storage/xtradb/enc/EncKeys.cc b/storage/xtradb/enc/EncKeys.cc index c61acc825df8f..ee6060fea9b4b 100644 --- a/storage/xtradb/enc/EncKeys.cc +++ b/storage/xtradb/enc/EncKeys.cc @@ -17,6 +17,48 @@ @file EncKeys.cc A class to keep keys for encryption/decryption. +How it works... +The location and usage can be configured via the configuration file. +Example + +[mysqld] +... +innodb_data_encryption_providertype = 1 +innodb_data_encryption_providername = keys.enc +innodb_data_encryption_providerurl = /home/mdb/ +innodb_data_encryption_filekey = secret +... + +As provider type currently only value 1 is supported, which means, the keys are read from a file. +The filename is set up via the innodb_data_encryption_providername configuration value. +innodb_data_encryption_providerurl is used to configure the path to this file. This is usually +a folder name. +Examples: +innodb_data_encryption_providerurl = \\\\unc (windows share) +innodb_data_encryption_providerurl = e:/tmp/ (windows path) +innodb_data_encryption_providerurl = /tmp (linux path) + +The key file contains AES keys and initialization vectors as hex-encoded Strings. +Supported are keys of size 128, 192 or 256 bits. IV consists of 16 bytes. + +The key file should be encrypted and the key to decrypt the file can be given with the +innodb_data_encryption_filekey parameter. + +The file key can also be located if FILE: is prepended to the key. Then the following part is interpreted +as absolut to the file containing the file key. This file can optionally be encrypted, currently with a fix key. +Example: +innodb_data_encryption_filekey = FILE:y:/secret256.enc + +If the key file can not be read at server startup, for example if the file key is not present, +page_encryption feature is not availabe and access to page_encryption tables is not possible. + +Example files can be found inside the unittest/eperi folder. + +Open SSL command line utility can be used to create an encrypted key file. +Examples: +openssl enc –aes-256-cbc –md sha1 –k secret –in keys.txt –out keys.enc +openssl enc –aes-256-cbc –md sha1 –k –in secret –out secret.enc + Created 09/15/2014 ***********************************************************************/ #ifdef __WIN__ diff --git a/storage/xtradb/fil/fil0fil.cc b/storage/xtradb/fil/fil0fil.cc index c3325f2225112..5397cc611a2ab 100644 --- a/storage/xtradb/fil/fil0fil.cc +++ b/storage/xtradb/fil/fil0fil.cc @@ -818,10 +818,10 @@ fil_node_open_file( success = os_file_read(node->handle, page, 0, UNIV_PAGE_SIZE, space->flags); - if (fil_page_is_encrypted(page)) { - /* if page is encrypted, write an error and return. + if (fil_page_can_not_decrypt(page)) { + /* if page is (still) encrypted, write an error and return. * Otherwise the server would crash if decrypting is not possible. - * This may be the case, if the keyfile could not be opened on server startup. + * This may be the case, if the key file could not be opened on server startup. */ fprintf(stderr, "InnoDB: can not decrypt %s\n", @@ -2107,8 +2107,10 @@ fil_check_first_page( space_id = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_ID + page); flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + page); - page_is_encrypted = fil_page_is_encrypted(page); - if (!KeySingleton::getInstance().isAvailable() && page_is_encrypted) { + /* Note: the 1st page is usually not encrypted. If the Key Provider or the encryption key is not available, the + * check for reading the first page should intentionally fail with "can not decrypt" message. */ + page_is_encrypted = fil_page_can_not_decrypt(page); + if ((!KeySingleton::getInstance().isAvailable() || (page_is_encrypted == PAGE_ENCRYPTION_KEY_MISSING)) && page_is_encrypted) { page_is_encrypted = 1; } else { page_is_encrypted = 0; diff --git a/storage/xtradb/fil/fil0pageencryption.cc b/storage/xtradb/fil/fil0pageencryption.cc index 681b57dc7d91a..d7bcdff2c0862 100644 --- a/storage/xtradb/fil/fil0pageencryption.cc +++ b/storage/xtradb/fil/fil0pageencryption.cc @@ -55,7 +55,7 @@ byte crc_table[] = { }; - +/* this calculates a crc-8 checksum byte */ byte fil_page_encryption_calc_checksum(unsigned char* buf, ulint len) { byte crc = 0; for (ulint i = 0; i < len; i++) @@ -68,22 +68,24 @@ byte fil_page_encryption_calc_checksum(unsigned char* buf, ulint len) { operation. Pages are encrypted with AES/CBC/NoPadding algorithm. + "No padding" is used to ensure, that the encrypted page does not exceed the page size. If "no padding" is used, the input for encryption must be of size (multiple * AES blocksize). AES Blocksize is usually 16 (bytes). - Since the length of the payload is not a mulitple of the AES blocksize, + Everything in the page is encrypted except for the 38 byte FIL header. + Since the length of the payload is not a multiple of the AES blocksize, and to ensure that every byte of the payload is encrypted, two encryption operations are done. Each time with a block of adequate size as input. + 1st block contains everything from beginning of payload bytes except for the remainder. + 2nd block is of size 64 and contains the remainder and the last (64 - sizeof(remainder)) bytes of the encrypted 1st block. Each encrypted page receives a new page type for PAGE_ENCRYPTION. - The original page type (2 bytes) is stored in the Checksum header of the page. + The original page type (2 bytes) is stored in the Checksum header of the page (position FIL_PAGE_SPACE_OR_CHKSUM). Additionally the encryption key identifier is stored in the Checksum Header. This uses 1 byte. - Checksum verification for encrypted pages is disabled. + Checksum verification for encrypted pages is disabled. This checksum should be restored after decryption. - To be able to verify decryption in a later stage, a checksum is stored in the "Old-style Checksum" at the end of the page. + To be able to verify decryption in a later stage, a 1-byte checksum at position 4 of the FIL_PAGE_SPACE_OR_CHKSUM header is stored. - Since with page_compressed enabled pages, these field does not seem to be availabe, this checksum is only usable, if there is - enough space at the end of the page. @return encrypted page to be written*/ byte* @@ -96,6 +98,7 @@ fil_encrypt_page( ulint len, /*!< in: length of input buffer.*/ ulint encryption_key,/*!< in: encryption key */ ulint* out_len, /*!< out: actual length of encrypted page */ + ulint* errorCode, /*!< out: an error code. set, if page is intentionally not encrypted */ ulint mode /*!< in: calling mode. Should be 0. Can be used for unit tests */ ) { @@ -112,6 +115,8 @@ fil_encrypt_page( ulint offset = 0; unit_test = mode ? 1 : 0; + *errorCode = AES_OK; + if (!unit_test) { ut_ad(fil_space_is_page_encrypted(space_id)); fil_system_enter(); @@ -125,7 +130,15 @@ fil_encrypt_page( space_id, fil_space_name(space), len); #endif /* UNIV_DEBUG */ } + /* read original page type */ + orig_page_type = mach_read_from_2(buf + FIL_PAGE_TYPE); + if ((orig_page_type == FIL_PAGE_TYPE_FSP_HDR) || (orig_page_type == FIL_PAGE_TYPE_XDES) ) { + memcpy(out_buf, buf, len); + + *errorCode = PAGE_ENCRYPTION_WILL_NOT_ENCRYPT; + return (out_buf); + } byte checksum_byte = fil_page_encryption_calc_checksum(buf + FIL_PAGE_DATA, len - FIL_PAGE_DATA); @@ -134,8 +147,6 @@ fil_encrypt_page( data_size = ((len - FIL_PAGE_DATA - FIL_PAGE_DATA_END) / MY_AES_BLOCK_SIZE) * MY_AES_BLOCK_SIZE; - /* read original page type */ - orig_page_type = mach_read_from_2(buf + FIL_PAGE_TYPE); @@ -149,7 +160,7 @@ fil_encrypt_page( if (!keys.isAvailable()) { err = AES_KEY_CREATION_FAILED; } else if (keys.getKeys(encryption_key) == NULL) { - err = AES_KEY_CREATION_FAILED; + err = PAGE_ENCRYPTION_KEY_MISSING; } else { char* keyString = keys.getKeys(encryption_key)->key; key_len = strlen(keyString)/2; @@ -164,7 +175,7 @@ fil_encrypt_page( if (!keys.isAvailable()) { err = AES_KEY_CREATION_FAILED; } else if (keys.getKeys(encryption_key) == NULL) { - err = AES_KEY_CREATION_FAILED; + err = PAGE_ENCRYPTION_KEY_MISSING; } else { char* ivString = keys.getKeys(encryption_key)->iv; if (ivString == NULL) return buf; @@ -218,6 +229,7 @@ fil_encrypt_page( if (tmp_buf!=NULL) { ut_free(tmp_buf); } + *errorCode = err; return (buf); } @@ -233,15 +245,16 @@ fil_encrypt_page( /* The 1st checksum field is used to store original page type, etc. * checksum check for page encrypted pages is omitted. + */ /* Set up the encryption key. Written to the 1st byte of the checksum header field. This header is currently used to store data. */ - mach_write_to_1(out_buf, key); + mach_write_to_1(out_buf + FIL_PAGE_SPACE_OR_CHKSUM, key); /* store original page type. Written to 2nd and 3rd byte of the checksum header field */ - mach_write_to_2(out_buf + 1, orig_page_type); + mach_write_to_2(out_buf + FIL_PAGE_SPACE_OR_CHKSUM + 1, orig_page_type); /* set byte 4 of checksum field to checksum byte */ - memset(out_buf + 3, checksum_byte, 1); + memset(out_buf + FIL_PAGE_SPACE_OR_CHKSUM + 3, checksum_byte, 1); #ifdef UNIV_DEBUG /* Verify */ @@ -263,12 +276,12 @@ fil_encrypt_page( For page encrypted pages decrypt the page after actual read operation. - See fil_encrypt_page for details. + See fil_encrypt_page for details, how the encryption works. + + If the decryption can be verified, original page should be completely restored. + This includes original page type, 4-byte checksum field at page start. + Decryption is verified against a 1-byte checksum built over the plain data bytes. If this verification fails, an error state is returned.. - if the decryption can be verified, original page should be completely restored, alongside. This includes original page type, checksum fields at page start and end. - If the verification fails, an error is raised. - If page_compressed is active it may not be possible to verify decryption, since the verification checksum can be missing. In this - case the verification result is not evaluated. @return decrypted page */ ulint fil_decrypt_page( @@ -300,8 +313,17 @@ ulint fil_decrypt_page( ut_ad(len); /* Before actual decrypt, make sure that page type is correct */ - if (mach_read_from_2(buf + FIL_PAGE_TYPE) != FIL_PAGE_PAGE_ENCRYPTED) + ulint current_page_type = mach_read_from_2(buf + FIL_PAGE_TYPE); + if ((current_page_type == FIL_PAGE_TYPE_FSP_HDR) || (current_page_type == FIL_PAGE_TYPE_XDES)) { + /* assumed as unencrypted */ + if (write_size!=NULL) { + * write_size = len; + } + return AES_OK; + } + if (current_page_type != FIL_PAGE_PAGE_ENCRYPTED) { + if (mach_read_from_2(buf + FIL_PAGE_TYPE) != FIL_PAGE_PAGE_ENCRYPTED) fprintf(stderr, "InnoDB: Corruption: We try to decrypt corrupted page\n" "InnoDB: CRC %lu type %lu.\n" "InnoDB: len %lu\n", @@ -319,14 +341,14 @@ ulint fil_decrypt_page( /* read page encryption key */ - page_decryption_key = mach_read_from_1(buf); + page_decryption_key = mach_read_from_1(buf + FIL_PAGE_SPACE_OR_CHKSUM); /* Get the page type */ - orig_page_type = mach_read_from_2(buf+1); + orig_page_type = mach_read_from_2(buf + FIL_PAGE_SPACE_OR_CHKSUM + 1); /* read checksum byte */ - byte stored_checksum_byte = mach_read_from_1(buf+3); + byte stored_checksum_byte = mach_read_from_1(buf + FIL_PAGE_SPACE_OR_CHKSUM + 3); if (FIL_PAGE_PAGE_COMPRESSED == orig_page_type) { if (page_compressed != NULL) { @@ -366,7 +388,7 @@ ulint fil_decrypt_page( if (!keys.isAvailable()) { err = PAGE_ENCRYPTION_ERROR; } else if (keys.getKeys(page_decryption_key) == NULL) { - err = PAGE_ENCRYPTION_ERROR; + err = PAGE_ENCRYPTION_KEY_MISSING; } else { char* keyString = keys.getKeys(page_decryption_key)->key; key_len = strlen(keyString)/2; @@ -375,21 +397,23 @@ ulint fil_decrypt_page( } - const unsigned char iv[] = {0x2d, 0x1a, 0xf8, 0xd3, 0x97, 0x4e, 0x0b, 0xd3, 0xef, 0xed, 0x5a, 0x6f, 0x82, 0x59, 0x4f,0x5e}; + + uint8 iv_len = 16; if (!unit_test) { KeySingleton& keys = KeySingleton::getInstance(); if (!keys.isAvailable()) { err = PAGE_ENCRYPTION_ERROR; } else if (keys.getKeys(page_decryption_key) == NULL) { - err = PAGE_ENCRYPTION_ERROR; + err = PAGE_ENCRYPTION_KEY_MISSING; } else { my_aes_hexToUint(keys.getKeys(page_decryption_key)->iv, (unsigned char*)&iv, 16); } } + if (err != AES_OK) { /* surely key could not be determined. */ fprintf(stderr, "InnoDB: Corruption: Page is marked as encrypted\n" @@ -496,11 +520,13 @@ ulint fil_decrypt_page( ulint pageno = mach_read_from_4(buf + FIL_PAGE_OFFSET); ulint flags = 0; ulint zip_size = 0; + /* please note, that pane with number 0 is not encrypted */ if (pageno == 0 ) { + flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + buf); } else { if (unit_test) { - /* in simple unit test, the tablespace memory hash is n.a. */ + /* in simple unit test, the tablespace memory cache is n.a. */ if ((mode & 0x01) != 0x01) { zip_size = mode; } @@ -533,7 +559,7 @@ ulint fil_decrypt_page( } if (!(page_compression_flag)) { - /* calc check sums and write to the buffer, if page was not compressed. + /* calc check sums and write to the buffer, if page is not of type PAGE_COMPRESSED. * if the decryption is verified, it is assumed that the original page was restored, re-calculating the original * checksums should be ok */ diff --git a/storage/xtradb/include/fil0pageencryption.h b/storage/xtradb/include/fil0pageencryption.h index 5fd4f5b4974b4..469eda76f55d8 100644 --- a/storage/xtradb/include/fil0pageencryption.h +++ b/storage/xtradb/include/fil0pageencryption.h @@ -19,15 +19,17 @@ this program; if not, write to the Free Software Foundation, Inc., #ifndef fil0pageencryption_h #define fil0pageencryption_h -#ifndef EP_UNIT_TEST -#include "fsp0fsp.h" -#include "fsp0pageencryption.h" -#endif - #define PAGE_ENCRYPTION_WRONG_KEY 1 #define PAGE_ENCRYPTION_WRONG_PAGE_TYPE 2 #define PAGE_ENCRYPTION_ERROR 3 +#define PAGE_ENCRYPTION_KEY_MISSING 4 #define PAGE_ENCRYPTION_OK 0 +#define PAGE_ENCRYPTION_WILL_NOT_ENCRYPT 5 + +#include "fsp0fsp.h" +#include "fsp0pageencryption.h" + + @@ -39,7 +41,7 @@ Created 08/25/2014 ***********************************************************************/ -/*******************************************************************//** +/******************************PAGE_ENCRYPTION_ERROR*************************************//** Returns the page encryption flag of the space, or false if the space is not encrypted. The tablespace must be cached in the memory cache. @return true if page encrypted, false if not or space not found */ @@ -59,6 +61,16 @@ fil_page_is_encrypted( const byte *buf); /*!< in: page */ +/*******************************************************************//** +Find out whether the page can be decrypted +@return true if page can be decrypted, false if not. */ +UNIV_INLINE +ulint +fil_page_can_not_decrypt( +/*===================*/ + const byte *buf); /*!< in: page */ + + /****************************************************************//** For page encrypted pages encrypt the page before actual write operation. @@ -74,6 +86,7 @@ fil_encrypt_page( ulint len, /*!< in: length of input buffer.*/ ulint compression_level, /*!< in: compression level */ ulint* out_len, /*!< out: actual length of compressed page */ + ulint* errorCode, /*!< out: an error code. set, if page is intentionally not encrypted */ ulint mode /*!< in: calling mode. Should be 0. */ ); diff --git a/storage/xtradb/include/fsp0pageencryption.ic b/storage/xtradb/include/fsp0pageencryption.ic index 43288297972e7..04180e89027dc 100644 --- a/storage/xtradb/include/fsp0pageencryption.ic +++ b/storage/xtradb/include/fsp0pageencryption.ic @@ -24,6 +24,8 @@ Created 08/28/2014 ***********************************************************************/ #include "fsp0fsp.h" +#include "KeySingleton.h" +#include "fil0pageencryption.h" @@ -113,3 +115,48 @@ fil_page_is_encrypted( { return(mach_read_from_2(buf+FIL_PAGE_TYPE) == FIL_PAGE_PAGE_ENCRYPTED); } + + +/*******************************************************************//** +Find out whether the page can be decrypted. +This is the case, if the page is already decrypted and is not the first page of the table space. +If the page is already decrypted it is not of the FIL_PAGE_PAGE_ENCRYPTED type. +if it is the first page of the table space, it is assumed that a page can be decrypted if the +key found in the flags (part of the 1st page) can be read from the key provider. +The case, if the key changed, is currently not caught. +The function for decrypting the page should already be executed before this. +@return PAGE_ENCRYPTION_KEY_MISSING if key provider is available, but key is not available + PAGE_ENCRYPTION_ERROR if other error occurred + 0 if decryption should be possible +*/ +UNIV_INLINE +ulint +fil_page_can_not_decrypt( +/*===================*/ + const byte *buf) /*!< in: page */ +{ + ulint page_type = mach_read_from_2(buf+FIL_PAGE_TYPE); + if (page_type == FIL_PAGE_TYPE_FSP_HDR) { + ulint flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + buf); + if (fsp_flags_is_page_encrypted(flags)) { + if (!KeySingleton::getInstance().isAvailable() || + !KeySingleton::getInstance().hasKey(fsp_flags_get_page_encryption_key(flags))) { + /* accessing table would surely fail, because no key or no key provider available */ + if (KeySingleton::getInstance().isAvailable() && + !KeySingleton::getInstance().hasKey(fsp_flags_get_page_encryption_key(flags))) { + return PAGE_ENCRYPTION_KEY_MISSING; + } + return PAGE_ENCRYPTION_ERROR; + } + } + } + if(page_type == FIL_PAGE_PAGE_ENCRYPTED) { + ulint key = mach_read_from_1(buf + FIL_PAGE_SPACE_OR_CHKSUM); + if (KeySingleton::getInstance().isAvailable() && + !KeySingleton::getInstance().hasKey(key)) { + return PAGE_ENCRYPTION_KEY_MISSING; + } + return PAGE_ENCRYPTION_ERROR; + } + return 0; +} diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index 93b623aa3e7d1..d93a3a332ac4f 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -45,7 +45,6 @@ Created 10/21/1995 Heikki Tuuri #include "fil0fil.h" #include "fsp0fsp.h" #include "fil0pagecompress.h" -#include "fsp0pageencryption.h" #include "fil0pageencryption.h" #include "buf0buf.h" #include "btr0types.h" @@ -4825,6 +4824,7 @@ os_aio_array_reserve_slot( then we encrypt the page */ if (message1 && type == OS_FILE_WRITE && page_encryption ) { ulint real_len = len; + ulint ec = 0; byte* tmp = NULL; /* Release the array mutex while encrypting */ @@ -4837,11 +4837,11 @@ os_aio_array_reserve_slot( } ut_ad(slot->page_buf2); - tmp = fil_encrypt_page(fil_node_get_space_id(slot->message1), (byte *)buf, slot->page_buf2, len, page_encryption_key, &real_len, 0); + + tmp = fil_encrypt_page(fil_node_get_space_id(slot->message1), (byte *)buf, slot->page_buf2, len, page_encryption_key, &real_len, &ec, 0); /* If encryption succeeded, set up the length and buffer */ - //TODO we do not need to reset len since we do not alter any content size - if (tmp != buf) { + if (tmp != buf || (ec == PAGE_ENCRYPTION_WILL_NOT_ENCRYPT)) { len = real_len; buf = slot->page_buf2; slot->len = real_len; diff --git a/unittest/eperi/pageenc-t.cc b/unittest/eperi/pageenc-t.cc index 34ce0ceaeecae..4166bb78d793b 100644 --- a/unittest/eperi/pageenc-t.cc +++ b/unittest/eperi/pageenc-t.cc @@ -16,6 +16,10 @@ typedef unsigned long int ibool; #include #include +#define FIL_PAGE_TYPE_FSP_HDR 8 /*!< File space header */ +#define FIL_PAGE_TYPE_XDES 9 /*!< Extent descriptor page */ +#define PAGE_ENCRYPTION_WILL_NOT_ENCRYPT 5 + extern int summef(int a, int b); extern int summef2(int a, int b); @@ -65,6 +69,7 @@ fil_encrypt_page( ulint len, /*!< in: length of input buffer.*/ ulint compression_level, /*!< in: compression level */ ulint* out_len, /*!< out: actual length of compressed page */ + ulint* errorCode, /*!< out: an error code. set, if page is intentionally not encrypted */ ulint mode /*!< in: calling mode */ ); @@ -115,7 +120,8 @@ void testEncryptionChecksum(char* filename) { byte* buf = readFile(filename,&fl); byte* dest = (byte *) malloc(16384*sizeof(byte)); ulint out_len; - fil_encrypt_page(0,buf,dest,fl,255, &out_len, 1); + ulint ec = 0; + fil_encrypt_page(0,buf,dest,fl,255, &out_len, &ec, 1); dest[2000]=0xFF; dest[2001]=0xFF; dest[2002]=0xFF; @@ -135,15 +141,24 @@ void testEncryptionChecksum(char* filename) { void testIt(char* filename, ulint do_not_cmp_checksum) { int fl = 0; byte* buf = readFile(filename, &fl); + char str[80]; + strcpy (str,"File "); + strcat (str,filename ); byte* dest = (byte *) malloc(16384*sizeof(byte)); ulint out_len; ulint cc1 = 0; + ulint ec = 0; ulint orig_page_type = mach_read_from_2(buf + 24); - byte* snd = fil_encrypt_page(0,buf,dest,fl,255, &out_len, fl==8192 ? fl : 1); + byte* snd = fil_encrypt_page(0,buf,dest,fl,255, &out_len, &ec, fl==8192 ? fl : 1); + if ((orig_page_type ==8) || (orig_page_type==9)) { + cc1 = (ec == 5) && (snd == buf); + ok(cc1, "page type 8 or 9 will not be encrypted! file %s", (char*) str); + return; + } cc1 = (buf!=dest); cc1 = cc1 && (snd==dest); if (!do_not_cmp_checksum) { @@ -163,9 +178,6 @@ void testIt(char* filename, ulint do_not_cmp_checksum) { } ulint i = memcmp((buf + a),(dest +a), (size_t)(write_size - (a+b))); - char str[80]; - strcpy (str,"File "); - strcat (str,filename ); cc1 = (i==0) && (cc1); if (!cc1) { dump_buffer(fl, buf); From df2ec547a0ca6c025875a7fef9ca60d5214202eb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Clemens=20D=C3=B6rrh=C3=B6fer?= Date: Thu, 30 Oct 2014 10:30:35 +0100 Subject: [PATCH 62/70] Removed printf from productive build. --- storage/xtradb/enc/EncKeys.cc | 15 ++++++++++----- storage/xtradb/enc/KeySingleton.cc | 5 +++-- unittest/eperi/pageenc-t.cc | 4 ++-- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/storage/xtradb/enc/EncKeys.cc b/storage/xtradb/enc/EncKeys.cc index ee6060fea9b4b..439e5cef18c85 100644 --- a/storage/xtradb/enc/EncKeys.cc +++ b/storage/xtradb/enc/EncKeys.cc @@ -168,7 +168,9 @@ int EncKeys::initKeysThroughFile(const char *name, const char *path, const char int EncKeys::initKeysThroughServer( const char *name, const char *path, const char *filekey) { //TODO +#ifdef UNIV_DEBUG fprintf(stderr, errorNotImplemented); +#endif //UNIV_DEBUG return ERROR_KEYINITTYPE_SERVER_NOT_IMPLEMENTED; } @@ -216,11 +218,16 @@ void EncKeys::parseSecret( const char *secretfile, char *secret ) { */ keyentry *EncKeys::getKeys(int id) { if (KEY_MIN <= id && KEY_MAX >= id && (oneKey = &keys[id - 1])->iv) + { return oneKey; + } +#ifdef UNIV_DEBUG else { + fprintf(stderr, errorNoKeyId, id); return NULL; } +#endif //UNIV_DEBUG } /** @@ -403,15 +410,13 @@ bool EncKeys::isComment(const char *line) { void EncKeys::printKeyEntry( ulint id) { +#ifdef UNIV_DEBUG keyentry *entry = getKeys(id); if( NULL == entry) { - fprintf(stderr, "No such keyID = %u\n", id); + fprintf(stderr, "No such keyID=%u\n",id); } else { -#ifdef UNIV_DEBUG fprintf(stderr, "Key: id:%3u \tiv:%d bytes\tkey:%d bytes\n", entry->id, strlen(entry->iv)/2, strlen(entry->key)/2); -#else - fprintf(stderr, "Key: id:%3u\n", entry->id); -#endif //UNIV_DEBUG } +#endif //UNIV_DEBUG } diff --git a/storage/xtradb/enc/KeySingleton.cc b/storage/xtradb/enc/KeySingleton.cc index 279a5c5982f9f..fc633b4f63ceb 100644 --- a/storage/xtradb/enc/KeySingleton.cc +++ b/storage/xtradb/enc/KeySingleton.cc @@ -32,11 +32,12 @@ EncKeys KeySingleton::encKeys; KeySingleton & KeySingleton::getInstance() { +#ifdef UNIV_DEBUG if( !instanceInited) { fprintf(stderr, "Encryption / decryption keys were not initialized. " - "You can not read encrypted tables or columns\n\n"); - fflush(stderr); + "You can not read encrypted tables or columns\n"); } +#endif UNIV_DEBUG return theInstance; } diff --git a/unittest/eperi/pageenc-t.cc b/unittest/eperi/pageenc-t.cc index 4166bb78d793b..4b517bbfb6b1f 100644 --- a/unittest/eperi/pageenc-t.cc +++ b/unittest/eperi/pageenc-t.cc @@ -180,9 +180,9 @@ void testIt(char* filename, ulint do_not_cmp_checksum) { cc1 = (i==0) && (cc1); if (!cc1) { - dump_buffer(fl, buf); + //dump_buffer(fl, buf); - dump_buffer(write_size, dest); + //dump_buffer(write_size, dest); } ok (cc1, "%s", (char*) str); From f8b6f19c18137f74e2a2ec229f03306e7bb99f5d Mon Sep 17 00:00:00 2001 From: Ludger Goeckel Date: Thu, 30 Oct 2014 14:55:46 +0100 Subject: [PATCH 63/70] free allocated buffers --- storage/xtradb/fil/fil0pageencryption.cc | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/storage/xtradb/fil/fil0pageencryption.cc b/storage/xtradb/fil/fil0pageencryption.cc index d7bcdff2c0862..a400e4efa1895 100644 --- a/storage/xtradb/fil/fil0pageencryption.cc +++ b/storage/xtradb/fil/fil0pageencryption.cc @@ -124,10 +124,11 @@ fil_encrypt_page( fil_system_exit(); #ifdef UNIV_DEBUG - + ulint pageno = mach_read_from_4(buf + FIL_PAGE_OFFSET); + fprintf(stderr, - "InnoDB: Note: Preparing for encryption for space %lu name %s len %lu\n", - space_id, fil_space_name(space), len); + "InnoDB: Note: Preparing for encryption for space %lu name %s len %lu, page no %lu\n", + space_id, fil_space_name(space), len, pageno); #endif /* UNIV_DEBUG */ } /* read original page type */ @@ -374,10 +375,6 @@ ulint fil_decrypt_page( } data_size = ((len - FIL_PAGE_DATA - FIL_PAGE_DATA_END) / MY_AES_BLOCK_SIZE) * MY_AES_BLOCK_SIZE; - tmp_buf= static_cast(ut_malloc(64)); - tmp_page_buf = static_cast(ut_malloc(len)); - memset(tmp_page_buf,0, len); - const unsigned char rkey[] = {0xbd, 0xe4, 0x72, 0xa2, 0x95, 0x67, 0x5c, 0xa9, 0x2e, 0x04, 0x67, 0xea, 0xdb, 0xc0,0xe0, 0x23, 0x00,0x00,0x00,0x00,0x00,0x00,0x00,0x00, @@ -455,6 +452,9 @@ ulint fil_decrypt_page( if (NULL == page_buf) { ut_free(in_buffer); } + ut_free(tmp_page_buf); + ut_free(tmp_buf); + return err; } @@ -511,6 +511,10 @@ ulint fil_decrypt_page( /* Copy the decrypted page to the buffer pool*/ memcpy(buf, in_buf, len); + if (NULL == page_buf) { + ut_free(in_buffer); + } + /* setting original page type */ mach_write_to_2(buf + FIL_PAGE_TYPE, orig_page_type); From d357cda0c0e13fa1c5b27c4185ee7edd071beccd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludger=20G=C3=B6ckel?= Date: Fri, 31 Oct 2014 10:10:33 +0100 Subject: [PATCH 64/70] changed comments, removed dead code --- storage/xtradb/fil/fil0pageencryption.cc | 2 ++ storage/xtradb/os/os0file.cc | 9 --------- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/storage/xtradb/fil/fil0pageencryption.cc b/storage/xtradb/fil/fil0pageencryption.cc index a400e4efa1895..53c64b7cec4dd 100644 --- a/storage/xtradb/fil/fil0pageencryption.cc +++ b/storage/xtradb/fil/fil0pageencryption.cc @@ -67,6 +67,8 @@ byte fil_page_encryption_calc_checksum(unsigned char* buf, ulint len) { For page encrypted pages encrypt the page before actual write operation. + Note, that FIL_PAGE_TYPE_FSP_HDR and FIL_PAGE_TYPE_XDES type pages are not encrypted! + Pages are encrypted with AES/CBC/NoPadding algorithm. "No padding" is used to ensure, that the encrypted page does not exceed the page size. diff --git a/storage/xtradb/os/os0file.cc b/storage/xtradb/os/os0file.cc index ed08378c6d2a2..c264bd9b96db3 100644 --- a/storage/xtradb/os/os0file.cc +++ b/storage/xtradb/os/os0file.cc @@ -5658,15 +5658,6 @@ os_aio_linux_collect( if (fil_page_is_encrypted(slot->buf)) { fil_decrypt_page(slot->page_buf2, slot->buf, slot->len, slot->write_size, NULL, 0); } - } else { - if (slot->page_encryption_success && - fil_page_is_encrypted(slot->page_buf2)) { - ut_ad(slot->page_encryption_page); - if (srv_use_trim && os_fallocate_failed == FALSE) { - // Deallocate unused blocks from file system ??? - //os_file_trim(slot->file, slot, slot->len); - } - } } } From cda7631b5aca3dceef7671eb60f755e03874eca4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludger=20G=C3=B6ckel?= Date: Fri, 31 Oct 2014 10:48:39 +0100 Subject: [PATCH 65/70] fixed unit test --- unittest/eperi/pageenc-t.cc | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/unittest/eperi/pageenc-t.cc b/unittest/eperi/pageenc-t.cc index 4b517bbfb6b1f..1c712ff7a783a 100644 --- a/unittest/eperi/pageenc-t.cc +++ b/unittest/eperi/pageenc-t.cc @@ -155,7 +155,8 @@ void testIt(char* filename, ulint do_not_cmp_checksum) { ulint orig_page_type = mach_read_from_2(buf + 24); byte* snd = fil_encrypt_page(0,buf,dest,fl,255, &out_len, &ec, fl==8192 ? fl : 1); if ((orig_page_type ==8) || (orig_page_type==9)) { - cc1 = (ec == 5) && (snd == buf); + ulint cc2 = memcmp(buf,snd,fl) == 0; + cc1 = (ec == 5) && cc2; ok(cc1, "page type 8 or 9 will not be encrypted! file %s", (char*) str); return; } @@ -310,7 +311,7 @@ int main() testSecrets(); test_page_enc_dec(); - testEncryptionChecksum((char* )"xaa"); + testEncryptionChecksum((char* )"xab"); return 0; } From 049a6a89d5f931702c56517c334abe4c5511402f Mon Sep 17 00:00:00 2001 From: Ludger Goeckel Date: Mon, 3 Nov 2014 13:17:51 +0100 Subject: [PATCH 66/70] fix win 64 build (only page_encryption files) --- .gitignore | 2 ++ include/my_aes.h | 8 +++--- mysys_ssl/my_aes.cc | 8 +++--- storage/xtradb/enc/EncKeys.cc | 4 +-- storage/xtradb/fil/fil0pageencryption.cc | 8 +++--- unittest/eperi/eperi_aes-t.cc | 36 ++++++++++++------------ 6 files changed, 34 insertions(+), 32 deletions(-) diff --git a/.gitignore b/.gitignore index c743fdc9d4c81..7388644ce9443 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ Debug/* pcre/* pcre3/* +_32/* +_64/* bld/* *-t *.a diff --git a/include/my_aes.h b/include/my_aes.h index 049cad3fd4175..a4868631b7473 100644 --- a/include/my_aes.h +++ b/include/my_aes.h @@ -54,8 +54,8 @@ C_MODE_START != 0 error 0 no error */ -int my_aes_encrypt_cbc(const char* source, unsigned long int source_length, - char* dest, unsigned long int *dest_length, +int my_aes_encrypt_cbc(const char* source, uint32 source_length, + char* dest, uint32 *dest_length, const unsigned char* key, uint8 key_length, const unsigned char* iv, uint8 iv_length, int noPadding); @@ -118,8 +118,8 @@ int my_aes_encrypt(const char *source, int source_length, char *dest, != 0 error 0 no error */ -int my_aes_decrypt_cbc(const char* source, unsigned long int source_length, - char* dest, unsigned long int *dest_length, +int my_aes_decrypt_cbc(const char* source, uint32 source_length, + char* dest, uint32 *dest_length, const unsigned char* key, uint8 key_length, const unsigned char* iv, uint8 iv_length, int noPadding); diff --git a/mysys_ssl/my_aes.cc b/mysys_ssl/my_aes.cc index 02526c5dbd35f..52640a41587d9 100644 --- a/mysys_ssl/my_aes.cc +++ b/mysys_ssl/my_aes.cc @@ -219,8 +219,8 @@ my_bytes_to_key(const unsigned char *salt, const char *secret, unsigned char *ke != 0 error 0 no error */ -int my_aes_encrypt_cbc(const char* source, unsigned long int source_length, - char* dest, unsigned long int *dest_length, +int my_aes_encrypt_cbc(const char* source, uint32 source_length, + char* dest, uint32* dest_length, const unsigned char* key, uint8 key_length, const unsigned char* iv, uint8 iv_length, int noPadding) @@ -338,8 +338,8 @@ int my_aes_encrypt_cbc(const char* source, unsigned long int source_length, != 0 error 0 no error */ -int my_aes_decrypt_cbc(const char* source, unsigned long int source_length, - char* dest, unsigned long int *dest_length, +int my_aes_decrypt_cbc(const char* source, uint32 source_length, + char* dest, uint32 *dest_length, const unsigned char* key, uint8 key_length, const unsigned char* iv, uint8 iv_length, int noPadding) diff --git a/storage/xtradb/enc/EncKeys.cc b/storage/xtradb/enc/EncKeys.cc index 439e5cef18c85..6ef7288cf18b9 100644 --- a/storage/xtradb/enc/EncKeys.cc +++ b/storage/xtradb/enc/EncKeys.cc @@ -200,7 +200,7 @@ void EncKeys::parseSecret( const char *secretfile, char *secret ) { memcpy(_initPwd, initialPwd, strlen(initialPwd)); _initPwd[strlen(initialPwd)]= '\0'; my_bytes_to_key((unsigned char *) salt, _initPwd, key, iv); - unsigned long int d_size = 0; + uint32 d_size = 0; int res = my_aes_decrypt_cbc((const char*)buf + 2 * magicSize, bytes_to_read - 2 * magicSize, secret, &d_size, key, keySize32, iv, ivSize16, 0); if (d_size>EncKeys::MAX_SECRET_SIZE) { @@ -378,7 +378,7 @@ char* EncKeys::decryptFile(const char* filename, const char *secret, int *errorC char *decrypted = new char[file_size]; memcpy(&salt, buffer + magicSize, magicSize); my_bytes_to_key((unsigned char *) salt, secret, key, iv); - unsigned long int d_size = 0; + uint32 d_size = 0; int res = my_aes_decrypt_cbc((const char*)buffer + 2 * magicSize, file_size - 2 * magicSize, decrypted, &d_size, key, keySize32, iv, ivSize16, 0); if(0 != res) { diff --git a/storage/xtradb/fil/fil0pageencryption.cc b/storage/xtradb/fil/fil0pageencryption.cc index 53c64b7cec4dd..f47221bc6f8fa 100644 --- a/storage/xtradb/fil/fil0pageencryption.cc +++ b/storage/xtradb/fil/fil0pageencryption.cc @@ -106,9 +106,9 @@ fil_encrypt_page( int err = AES_OK; int key = 0; - ulint data_size = 0; + uint32 data_size = 0; ulint orig_page_type = 0; - ulint write_size = 0; + uint32 write_size = 0; fil_space_t* space = NULL; byte* tmp_buf = NULL; ulint unit_test = 0; @@ -299,9 +299,9 @@ ulint fil_decrypt_page( ) { int err = AES_OK; ulint page_decryption_key; - ulint data_size = 0; + uint32 data_size = 0; ulint orig_page_type = 0; - ulint tmp_write_size = 0; + uint32 tmp_write_size = 0; ulint offset = 0; byte * in_buffer; byte * in_buf; diff --git a/unittest/eperi/eperi_aes-t.cc b/unittest/eperi/eperi_aes-t.cc index 84850fea886c8..4b4a4e6b761da 100644 --- a/unittest/eperi/eperi_aes-t.cc +++ b/unittest/eperi/eperi_aes-t.cc @@ -17,15 +17,15 @@ typedef unsigned long int ibool; #include #include extern "C" { -extern int my_aes_decrypt_cbc(const char* source, unsigned long int source_length, - char* dest, unsigned long int *dest_length, +extern int my_aes_decrypt_cbc(const char* source, uint32 source_length, + char* dest, uint32* dest_length, const unsigned char* key, uint8 key_length, const unsigned char* iv, uint8 iv_length, int noPadding); -extern int my_aes_encrypt_cbc(const char* source, unsigned long int source_length, - char* dest, unsigned long int *dest_length, +extern int my_aes_encrypt_cbc(const char* source, uint32 source_length, + char* dest, uint32 *dest_length, const unsigned char* key, uint8 key_length, const unsigned char* iv, uint8 iv_length, int noPadding); @@ -100,11 +100,11 @@ test_cbc128noPadding() plan(1); int i; char* source = "int i = memcmp(decbuf,inbuf,16);"; - ulint s_len = strlen(source); + uint32 s_len = strlen(source); char* dest = (char* ) malloc(100); char* result = (char*) malloc(100); - ulint dest_len = 0; + uint32 dest_len = 0; unsigned char key[16] = {0x58, 0x3b, 0xe7, 0xf3, 0x34, 0xf8, 0x5e, 0x7d, 0x9d, 0xdb, 0x36, 0x2e, 0x9a, 0xc3, 0x81, 0x51}; uint8 k_len = 16; @@ -135,11 +135,11 @@ test_cbc128() plan(2); int i; char* source = "int i = memcmp(decbuf,inbuf,16);"; - ulint s_len = strlen(source); + uint32 s_len = strlen(source); char* dest = (char* ) malloc(100); char* result = (char*) malloc(100); - ulint dest_len = 0; + uint32 dest_len = 0; unsigned char key[16] = {0x58, 0x3b, 0xe7, 0xf3, 0x34, 0xf8, 0x5e, 0x7d, 0x9d, 0xdb, 0x36, 0x2e, 0x9a, 0xc3, 0x81, 0x51}; uint8 k_len = 16; @@ -164,11 +164,11 @@ test_cbc192noPadding() plan(3); int i; char* source = "int i = memcmp(decbuf,inbuf,16);"; - ulint s_len = strlen(source); + uint32 s_len = strlen(source); char* dest = (char* ) malloc(100); char* result = (char*) malloc(100); - ulint dest_len = 0; + uint32 dest_len = 0; unsigned char key[24] = {0x58, 0x3b, 0xe7, 0xf3, 0x34, 0xf8, 0x5e, 0x7d, 0x9d, 0xdb, 0x36, 0x2e, 0x9a, 0xc3, 0x81, 0x51, 0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08}; @@ -192,11 +192,11 @@ test_cbc192() plan(4); int i; char* source = "int i = memcmp(decbuf,inbuf,16);"; - ulint s_len = strlen(source); + uint32 s_len = strlen(source); char* dest = (char* ) malloc(100); char* result = (char*) malloc(100); - ulint dest_len = 0; + uint32 dest_len = 0; unsigned char key[24] = {0x58, 0x3b, 0xe7, 0xf3, 0x34, 0xf8, 0x5e, 0x7d, 0x9d, 0xdb, 0x36, 0x2e, 0x9a, 0xc3, 0x81, 0x51, 0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08}; @@ -223,11 +223,11 @@ test_cbc256() plan(5); int i; char* source = "int i = memcmp(decbuf,inbuf,16);"; - ulint s_len = strlen(source); + uint32 s_len = strlen(source); char* dest = (char* ) malloc(100); char* result = (char*) malloc(100); - ulint dest_len = 0; + uint32 dest_len = 0; unsigned char key[32] = {0x58, 0x3b, 0xe7, 0xf3, 0x34, 0xf8, 0x5e, 0x7d, 0x9d, 0xdb, 0x36, 0x2e, 0x9a, 0xc3, 0x81, 0x51, 0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08, @@ -261,11 +261,11 @@ test_cbc256noPadding() plan(6); int i; char* source = "int i = memcmp(decbuf,inbuf,16);"; - ulint s_len = strlen(source); + uint32 s_len = strlen(source); char* dest = (char* ) malloc(100); char* result = (char*) malloc(100); - ulint dest_len = 0; + uint32 dest_len = 0; unsigned char key[32] = {0x58, 0x3b, 0xe7, 0xf3, 0x34, 0xf8, 0x5e, 0x7d, 0x9d, 0xdb, 0x36, 0x2e, 0x9a, 0xc3, 0x81, 0x51, 0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08, @@ -301,11 +301,11 @@ test_cbc256noPaddingWrongInputSize() plan(7); int i; char* source = "int i = memcmp(decbuf,inbuf,16);sdfsd"; - ulint s_len = strlen(source); + uint32 s_len = strlen(source); char* dest = (char* ) malloc(100); char* result = (char*) malloc(100); - ulint dest_len = 0; + uint32 dest_len = 0; unsigned char key[32] = {0x58, 0x3b, 0xe7, 0xf3, 0x34, 0xf8, 0x5e, 0x7d, 0x9d, 0xdb, 0x36, 0x2e, 0x9a, 0xc3, 0x81, 0x51, 0x01,0x02,0x03,0x04,0x05,0x06,0x07,0x08, From e209ec8f6f95840569b37bad72062351ce5436a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludger=20G=C3=B6ckel?= Date: Tue, 18 Nov 2014 13:52:43 +0100 Subject: [PATCH 67/70] fix memory leak --- storage/xtradb/enc/EncKeys.cc | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/storage/xtradb/enc/EncKeys.cc b/storage/xtradb/enc/EncKeys.cc index 6ef7288cf18b9..ec5a8bd174f14 100644 --- a/storage/xtradb/enc/EncKeys.cc +++ b/storage/xtradb/enc/EncKeys.cc @@ -217,9 +217,9 @@ void EncKeys::parseSecret( const char *secretfile, char *secret ) { * Returns a struct keyentry with the asked 'id' or NULL. */ keyentry *EncKeys::getKeys(int id) { - if (KEY_MIN <= id && KEY_MAX >= id && (oneKey = &keys[id - 1])->iv) + if (KEY_MIN <= id && KEY_MAX >= id && (&keys[id - 1])->iv) { - return oneKey; + return &keys[id - 1]; } #ifdef UNIV_DEBUG else { @@ -237,6 +237,7 @@ keyentry *EncKeys::getKeys(int id) { */ int EncKeys::parseFile(const char* filename, const ulint maxKeyId, const char *secret) { int errorCode = 0; + ulint id = 0; char *buffer = decryptFile(filename, secret, &errorCode); if (NO_ERROR_PARSE_OK != errorCode) return errorCode; @@ -247,9 +248,11 @@ int EncKeys::parseFile(const char* filename, const ulint maxKeyId, const char *s keyLineInKeyFile++; switch (parseLine(line, maxKeyId)) { case NO_ERROR_PARSE_OK: + id = oneKey->id; keys[oneKey->id - 1] = *oneKey; + delete(oneKey); countKeys++; - fprintf(stderr, "Line: %u --> ", keyLineInKeyFile); printKeyEntry(oneKey->id); + fprintf(stderr, "Line: %u --> ", keyLineInKeyFile); printKeyEntry(id); break; case ERROR_ID_TOO_BIG: fprintf(stderr, errorExceedKeySize, KEY_MAX, keyLineInKeyFile); @@ -289,7 +292,7 @@ int EncKeys::parseLine(const char *line, const ulint maxKeyId) { else { const char *error_p = NULL; int offset; - static const pcre *pattern = pcre_compile( + pcre *pattern = pcre_compile( "([0-9]+);([0-9,a-f,A-F]{32});([0-9,a-f,A-F]{64}|[0-9,a-f,A-F]{48}|[0-9,a-f,A-F]{32})", 0, &error_p, &offset, NULL); if ( NULL != error_p) @@ -297,6 +300,7 @@ int EncKeys::parseLine(const char *line, const ulint maxKeyId) { int m_len = (int) strlen(line), ovector[MAX_OFFSETS_IN_PCRE_PATTERNS]; int rc = pcre_exec(pattern, NULL, line, m_len, 0, 0, ovector, MAX_OFFSETS_IN_PCRE_PATTERNS); + pcre_free(pattern); if (4 == rc) { char lin[MAX_KEY_LINE_SIZE + 1]; strncpy( lin, line, MAX_KEY_LINE_SIZE); @@ -401,8 +405,9 @@ char* EncKeys::decryptFile(const char* filename, const char *secret, int *errorC bool EncKeys::isComment(const char *line) { const char *error_p; int offset, m_len = (int) strlen(line), ovector[MAX_OFFSETS_IN_PCRE_PATTERNS]; - static const pcre *pattern = pcre_compile("\\s*#.*", 0, &error_p, &offset, NULL); + pcre *pattern = pcre_compile("\\s*#.*", 0, &error_p, &offset, NULL); int rc = pcre_exec( pattern, NULL, line, m_len, 0, 0, ovector, MAX_OFFSETS_IN_PCRE_PATTERNS); + pcre_free(pattern); if (0 > rc) return false; else return true; } From d75dac2c0acb1af8c0334a3cb82c8c9f060a0025 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludger=20G=C3=B6ckel?= Date: Wed, 19 Nov 2014 15:44:16 +0100 Subject: [PATCH 68/70] fix error with page_compressed tables --- storage/xtradb/fil/fil0pageencryption.cc | 55 +++++++++++++-------- storage/xtradb/include/fil0pageencryption.h | 4 +- unittest/eperi/pageenc-t.cc | 37 +++++++++++--- 3 files changed, 66 insertions(+), 30 deletions(-) diff --git a/storage/xtradb/fil/fil0pageencryption.cc b/storage/xtradb/fil/fil0pageencryption.cc index f47221bc6f8fa..5a9e655b94bfc 100644 --- a/storage/xtradb/fil/fil0pageencryption.cc +++ b/storage/xtradb/fil/fil0pageencryption.cc @@ -33,6 +33,7 @@ this program; if not, write to the Free Software Foundation, Inc., #include #include +#include /* @@ -87,7 +88,7 @@ byte fil_page_encryption_calc_checksum(unsigned char* buf, ulint len) { Checksum verification for encrypted pages is disabled. This checksum should be restored after decryption. To be able to verify decryption in a later stage, a 1-byte checksum at position 4 of the FIL_PAGE_SPACE_OR_CHKSUM header is stored. - + For page compressed table pages the log base 2 of the length of the encrypted data is stored. @return encrypted page to be written*/ byte* @@ -115,6 +116,7 @@ fil_encrypt_page( ut_ad(buf);ut_ad(out_buf); key = encryption_key; ulint offset = 0; + ulint page_len = 0; unit_test = mode ? 1 : 0; *errorCode = AES_OK; @@ -143,6 +145,12 @@ fil_encrypt_page( return (out_buf); } + if (FIL_PAGE_PAGE_COMPRESSED == orig_page_type) { + page_len = log10(len)/log10(2); + } + + + byte checksum_byte = fil_page_encryption_calc_checksum(buf + FIL_PAGE_DATA, len - FIL_PAGE_DATA); /* data_size bytes will be encrypted at first. @@ -256,8 +264,13 @@ fil_encrypt_page( /* store original page type. Written to 2nd and 3rd byte of the checksum header field */ mach_write_to_2(out_buf + FIL_PAGE_SPACE_OR_CHKSUM + 1, orig_page_type); - /* set byte 4 of checksum field to checksum byte */ - memset(out_buf + FIL_PAGE_SPACE_OR_CHKSUM + 3, checksum_byte, 1); + if (FIL_PAGE_PAGE_COMPRESSED == orig_page_type) { + /* set byte 4 of checksum field to page length (ln(len)) */ + memset(out_buf + FIL_PAGE_SPACE_OR_CHKSUM + 3, page_len, 1); + } else { + /* set byte 4 of checksum field to checksum byte */ + memset(out_buf + FIL_PAGE_SPACE_OR_CHKSUM + 3, checksum_byte, 1); + } #ifdef UNIV_DEBUG /* Verify */ @@ -283,7 +296,7 @@ fil_encrypt_page( If the decryption can be verified, original page should be completely restored. This includes original page type, 4-byte checksum field at page start. - Decryption is verified against a 1-byte checksum built over the plain data bytes. If this verification fails, an error state is returned.. + If it is not a page compressed table's page, decryption is verified against a 1-byte checksum built over the plain data bytes. If this verification fails, an error state is returned.. @return decrypted page */ @@ -293,7 +306,7 @@ ulint fil_decrypt_page( byte* buf, /*!< in/out: buffer from which to read; in aio this must be appropriately aligned */ ulint len, /*!< in: length buffer, which should be decrypted.*/ - ulint* write_size, /*!< out: size of the decrypted data. If no error occurred equal to len */ + ulint* write_size, /*!< out: size of the decrypted data. If no error occurred equal to len, except for page compressed tables */ ibool* page_compressed, /*!(ut_malloc(2 * (len))); - in_buf = static_cast(ut_align(in_buffer, len)); + in_buffer = static_cast(ut_malloc(2 * (UNIV_PAGE_SIZE))); + in_buf = static_cast(ut_align(in_buffer, UNIV_PAGE_SIZE)); } else { in_buf = page_buf; @@ -526,7 +540,7 @@ ulint fil_decrypt_page( ulint pageno = mach_read_from_4(buf + FIL_PAGE_OFFSET); ulint flags = 0; ulint zip_size = 0; - /* please note, that pane with number 0 is not encrypted */ + /* please note, that page with number 0 is not encrypted */ if (pageno == 0 ) { flags = mach_read_from_4(FSP_HEADER_OFFSET + FSP_SPACE_FLAGS + buf); @@ -544,7 +558,7 @@ ulint fil_decrypt_page( fil_system_exit(); } } - if (!unit_test || pageno==0) { + if (!(page_compression_flag) && (!unit_test || pageno==0)) { zip_size = fsp_flags_get_zip_size(flags); } @@ -553,15 +567,17 @@ ulint fil_decrypt_page( } - byte checksum_byte = fil_page_encryption_calc_checksum(buf + FIL_PAGE_DATA, len - FIL_PAGE_DATA); - if (checksum_byte != stored_checksum_byte) { - err = PAGE_ENCRYPTION_WRONG_KEY; - fprintf(stderr, "InnoDB: Corruption: Page is marked as encrypted\n" - "InnoDB: but decryption verification failed with error %d, encryption key %d.\n", - err, (int)page_decryption_key); - fflush(stderr); - - return err; + if (!(page_compression_flag)) { + byte checksum_byte = fil_page_encryption_calc_checksum(buf + FIL_PAGE_DATA, len - FIL_PAGE_DATA); + if (checksum_byte != stored_checksum_byte) { + err = PAGE_ENCRYPTION_WRONG_KEY; + fprintf(stderr, "InnoDB: Corruption: Page is marked as encrypted\n" + "InnoDB: but decryption verification failed with error %d, encryption key %d.\n", + err, (int)page_decryption_key); + fflush(stderr); + + return err; + } } if (!(page_compression_flag)) { @@ -577,7 +593,6 @@ ulint fil_decrypt_page( - srv_stats.pages_page_decrypted.inc(); return err; } diff --git a/storage/xtradb/include/fil0pageencryption.h b/storage/xtradb/include/fil0pageencryption.h index 469eda76f55d8..2e68e26ca3616 100644 --- a/storage/xtradb/include/fil0pageencryption.h +++ b/storage/xtradb/include/fil0pageencryption.h @@ -85,7 +85,7 @@ fil_encrypt_page( byte* out_buf, /*!< out: compressed buffer */ ulint len, /*!< in: length of input buffer.*/ ulint compression_level, /*!< in: compression level */ - ulint* out_len, /*!< out: actual length of compressed page */ + ulint* out_len, /*!< out: actual length of encrypted page */ ulint* errorCode, /*!< out: an error code. set, if page is intentionally not encrypted */ ulint mode /*!< in: calling mode. Should be 0. */ ); @@ -101,7 +101,7 @@ fil_decrypt_page( byte* buf, /*!< out: buffer from which to read; in aio this must be appropriately aligned */ ulint len, /*!< in: length buffer, which should be decrypted.*/ - ulint* write_size, /*!< out: size of the decrypted data. If no error occurred equal to len */ + ulint* write_size, /*!< out: size of the decrypted data. If no error occurred equal to len, except for page compressed tables */ ibool* page_compressed, /*! #include +#include #include #define FIL_PAGE_TYPE_FSP_HDR 8 /*!< File space header */ @@ -138,7 +139,7 @@ void testEncryptionChecksum(char* filename) { } -void testIt(char* filename, ulint do_not_cmp_checksum) { +void testIt(char* filename, ulint do_not_cmp_checksum, ulint page_compressed, ulint input_size) { int fl = 0; byte* buf = readFile(filename, &fl); char str[80]; @@ -153,14 +154,24 @@ void testIt(char* filename, ulint do_not_cmp_checksum) { ulint ec = 0; ulint orig_page_type = mach_read_from_2(buf + 24); - byte* snd = fil_encrypt_page(0,buf,dest,fl,255, &out_len, &ec, fl==8192 ? fl : 1); + ulint compressed_page =0; + if (orig_page_type == 0x8632) { + compressed_page = 1; + } + + byte* snd = fil_encrypt_page(0,buf,dest,page_compressed? input_size : fl,255, &out_len, &ec, fl==8192 ? fl : 1); if ((orig_page_type ==8) || (orig_page_type==9)) { ulint cc2 = memcmp(buf,snd,fl) == 0; cc1 = (ec == 5) && cc2; ok(cc1, "page type 8 or 9 will not be encrypted! file %s", (char*) str); return; } + cc1 = (buf!=dest); + if (compressed_page) { + ulint write_size = mach_read_from_1(dest+3); + cc1 = cc1 && (pow(2,write_size) ==fl); + } cc1 = cc1 && (snd==dest); if (!do_not_cmp_checksum) { /* verify page type and enryption key*/ @@ -168,9 +179,12 @@ void testIt(char* filename, ulint do_not_cmp_checksum) { /* 255 is the key used for unit test */ cc1 = cc1 && (mach_read_from_1(dest) == 255); } + if (page_compressed) { + memcpy (dest+out_len, buf+out_len, fl-out_len); + } ulint write_size = 0; - ulint result = fil_decrypt_page(NULL, dest, out_len,&write_size,NULL, out_len==8192 ? out_len : 1); - cc1 = (result == 0) && (write_size==fl); + ulint result = fil_decrypt_page(NULL, dest, page_compressed? fl: out_len,&write_size,NULL, out_len==8192 ? out_len : 1); + cc1 = (result == 0) && (page_compressed? (write_size = out_len):(write_size==fl)); ulint a = 0; ulint b = 0; if (do_not_cmp_checksum) { @@ -185,9 +199,14 @@ void testIt(char* filename, ulint do_not_cmp_checksum) { //dump_buffer(write_size, dest); } - + if (page_compressed) { + ok(cc1, "%s %s write size: %lu", str, "page_compressed", out_len ); + } ok (cc1, "%s", (char*) str); } +void testIt(char* filename, ulint do_not_cmp_checksum) { + testIt(filename, do_not_cmp_checksum, 0, 0); +} void test_page_enc_dec() { char compressed[] = "compressed"; char compressed_full[] = "compressed_full"; @@ -223,9 +242,11 @@ void test_page_enc_dec() { testIt("row_format_compactad", 0); - testIt(compressed,0); - testIt(compressed_full,0); - testIt(compressed_6bytes_av,0); + testIt(compressed,0, 1, 16384); + testIt(compressed_full, 0, 1, 16384); + testIt(compressed_6bytes_av, 0, 1, 16384); + + testIt(compressed,0, 1, 4096); testIt(xaa,0); From 0a762593c0d0ca9471a4627ad8e1ece0c08a590a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ludger=20G=C3=B6ckel?= Date: Mon, 24 Nov 2014 14:43:15 +0100 Subject: [PATCH 69/70] use slot structure for temporary memory. fixed memory leak in unit test. --- storage/xtradb/enc/EncKeys.cc | 2 + storage/xtradb/fil/fil0pageencryption.cc | 42 +++++++----- storage/xtradb/include/fil0pageencryption.h | 6 +- storage/xtradb/os/os0file.cc | 62 ++++++++++++++---- unittest/eperi/CMakeLists.txt | 3 + unittest/eperi/pageenc-t.cc | 38 +++++++++-- unittest/eperi/row_format_compactad_encrypted | Bin 0 -> 16384 bytes 7 files changed, 113 insertions(+), 40 deletions(-) create mode 100644 unittest/eperi/row_format_compactad_encrypted diff --git a/storage/xtradb/enc/EncKeys.cc b/storage/xtradb/enc/EncKeys.cc index ec5a8bd174f14..f0934c14b0657 100644 --- a/storage/xtradb/enc/EncKeys.cc +++ b/storage/xtradb/enc/EncKeys.cc @@ -206,6 +206,8 @@ void EncKeys::parseSecret( const char *secretfile, char *secret ) { if (d_size>EncKeys::MAX_SECRET_SIZE) { d_size = EncKeys::MAX_SECRET_SIZE; } + delete[] key; + delete[] iv; secret[d_size] = '\0'; } free(buf); diff --git a/storage/xtradb/fil/fil0pageencryption.cc b/storage/xtradb/fil/fil0pageencryption.cc index 5a9e655b94bfc..6e2aa09459e29 100644 --- a/storage/xtradb/fil/fil0pageencryption.cc +++ b/storage/xtradb/fil/fil0pageencryption.cc @@ -102,6 +102,7 @@ fil_encrypt_page( ulint encryption_key,/*!< in: encryption key */ ulint* out_len, /*!< out: actual length of encrypted page */ ulint* errorCode, /*!< out: an error code. set, if page is intentionally not encrypted */ + byte* tmp_encryption_buf, /*!< in: temorary buffer or NULL */ ulint mode /*!< in: calling mode. Should be 0. Can be used for unit tests */ ) { @@ -207,9 +208,12 @@ fil_encrypt_page( * Old-style checksum and the "Low 32 bits of LSN */ memcpy(out_buf + FIL_PAGE_DATA + data_size , buf + FIL_PAGE_DATA + data_size , len - FIL_PAGE_DATA -data_size); - - //create temporary buffer for 2nd encryption - tmp_buf = static_cast(ut_malloc(64)); + if (tmp_encryption_buf == NULL) { + //create temporary buffer for 2nd encryption + tmp_buf = static_cast(ut_malloc(64)); + } else { + tmp_buf = tmp_encryption_buf; + } /* 2nd encryption: 63 bytes from out_buf, result length is 64 bytes */ err = my_aes_encrypt_cbc((char*)out_buf + len -offset -64, 64, @@ -237,7 +241,7 @@ fil_encrypt_page( *out_len = len; /* free temporary buffer */ - if (tmp_buf!=NULL) { + if (tmp_buf!=NULL && tmp_encryption_buf == NULL) { ut_free(tmp_buf); } *errorCode = err; @@ -282,7 +286,7 @@ fil_encrypt_page( *out_len = len; /* free temporary buffer */ - if (tmp_buf!=NULL) { + if (tmp_buf!=NULL && tmp_encryption_buf == NULL) { ut_free(tmp_buf); } return (out_buf); @@ -308,6 +312,7 @@ ulint fil_decrypt_page( ulint len, /*!< in: length buffer, which should be decrypted.*/ ulint* write_size, /*!< out: size of the decrypted data. If no error occurred equal to len, except for page compressed tables */ ibool* page_compressed, /*!(ut_malloc(len)); - tmp_buf= static_cast(ut_malloc(64)); - memset(tmp_page_buf,0, len); + if (tmp_encryption_buf == NULL) { + tmp_page_buf = static_cast(ut_malloc(len)); + tmp_buf= static_cast(ut_malloc(64)); + } else { + tmp_page_buf = tmp_encryption_buf; + tmp_buf = tmp_encryption_buf + UNIV_PAGE_SIZE; + } /* 1st decryption: 64 bytes */ @@ -468,9 +476,10 @@ ulint fil_decrypt_page( if (NULL == page_buf) { ut_free(in_buffer); } - ut_free(tmp_page_buf); - ut_free(tmp_buf); - + if (NULL == tmp_encryption_buf) { + ut_free(tmp_page_buf); + ut_free(tmp_buf); + } return err; } @@ -481,8 +490,6 @@ ulint fil_decrypt_page( memcpy(tmp_page_buf + FIL_PAGE_DATA, buf + FIL_PAGE_DATA, len -offset -64 - FIL_PAGE_DATA); - /* fill target buffer with zeros */ - memset(in_buf, 0, len); @@ -509,9 +516,10 @@ ulint fil_decrypt_page( - - ut_free(tmp_page_buf); - ut_free(tmp_buf); + if (NULL == tmp_encryption_buf) { + ut_free(tmp_page_buf); + ut_free(tmp_buf); + } diff --git a/storage/xtradb/include/fil0pageencryption.h b/storage/xtradb/include/fil0pageencryption.h index 2e68e26ca3616..b43843571f4e3 100644 --- a/storage/xtradb/include/fil0pageencryption.h +++ b/storage/xtradb/include/fil0pageencryption.h @@ -87,7 +87,8 @@ fil_encrypt_page( ulint compression_level, /*!< in: compression level */ ulint* out_len, /*!< out: actual length of encrypted page */ ulint* errorCode, /*!< out: an error code. set, if page is intentionally not encrypted */ - ulint mode /*!< in: calling mode. Should be 0. */ + byte* tmp_encryption_buf, /*!< in: temorary buffer or NULL */ + ulint mode /*!< in: calling mode. Should be 0. */ ); /****************************************************************//** @@ -103,7 +104,8 @@ fil_decrypt_page( ulint len, /*!< in: length buffer, which should be decrypted.*/ ulint* write_size, /*!< out: size of the decrypted data. If no error occurred equal to len, except for page compressed tables */ ibool* page_compressed, /*!page_encryption_page); slot->page_encryption_page = NULL; } + if (slot->tmp_encryption_buf) { + ut_free(slot->tmp_encryption_buf); + slot->tmp_encryption_buf = NULL; + } } @@ -4866,10 +4877,13 @@ os_aio_array_reserve_slot( if (slot->page_buf2 == NULL) { os_slot_alloc_page_buf2(slot); } + os_slot_alloc_tmp_encryption_buf(slot); + + ut_ad(slot->page_buf2); - tmp = fil_encrypt_page(fil_node_get_space_id(slot->message1), (byte *)buf, slot->page_buf2, len, page_encryption_key, &real_len, &ec, 0); + tmp = fil_encrypt_page(fil_node_get_space_id(slot->message1), (byte *)buf, slot->page_buf2, len, page_encryption_key, &real_len, &ec, slot->tmp_encryption_buf, 0); /* If encryption succeeded, set up the length and buffer */ if (tmp != buf || (ec == PAGE_ENCRYPTION_WILL_NOT_ENCRYPT)) { @@ -5318,6 +5332,7 @@ os_aio_func( if (page_encryption) { if (!slot->page_encryption_success) goto err_exit; buffer = slot->page_buf2; + n = slot->len; } else { buffer = buf; } @@ -5479,14 +5494,18 @@ os_aio_windows_handle( switch (slot->type) { case OS_FILE_WRITE: - if (slot->message1 && slot->page_compression && slot->page_buf) { - ret_val = os_file_write(slot->name, slot->file, slot->page_buf, - slot->offset, slot->len); - } else { + if (slot->message1 && slot->page_encryption && slot->page_buf2) { + ret_val = os_file_write(slot->name, slot->file, slot->page_buf2, + slot->offset, slot->len); + } else + if (slot->message1 && slot->page_compression && slot->page_buf) { + ret_val = os_file_write(slot->name, slot->file, slot->page_buf, + slot->offset, slot->len); + } else { - ret_val = os_file_write(slot->name, slot->file, slot->buf, - slot->offset, slot->len); - } + ret_val = os_file_write(slot->name, slot->file, slot->buf, + slot->offset, slot->len); + } break; case OS_FILE_READ: ret_val = os_file_read(slot->file, slot->buf, @@ -5520,12 +5539,13 @@ os_aio_windows_handle( if (slot->page_buf2==NULL) { os_slot_alloc_page_buf2(slot); } + os_slot_alloc_tmp_encryption_buf(slot); ut_ad(slot->page_buf2); if (slot->type == OS_FILE_READ) { if (fil_page_is_encrypted(slot->buf)) { - fil_decrypt_page(slot->page_buf2, slot->buf, slot->len, slot->write_size, NULL, 0); + fil_decrypt_page(slot->page_buf2, slot->buf, slot->len, slot->write_size, NULL, slot->tmp_encryption_buf, 0); } } @@ -5651,12 +5671,13 @@ os_aio_linux_collect( if (slot->page_buf2==NULL) { os_slot_alloc_page_buf2(slot); } + os_slot_alloc_tmp_encryption_buf(slot); ut_ad(slot->page_buf2); if (slot->type == OS_FILE_READ) { if (fil_page_is_encrypted(slot->buf)) { - fil_decrypt_page(slot->page_buf2, slot->buf, slot->len, slot->write_size, NULL, 0); + fil_decrypt_page(slot->page_buf2, slot->buf, slot->len, slot->write_size, NULL, slot->tmp_encryption_buf, 0); } } } @@ -6758,6 +6779,19 @@ os_slot_alloc_page_buf2( slot->page_buf2 = static_cast(cbuf); } +/**********************************************************************//** +Allocate memory for temporal buffer used for page encryption. */ +UNIV_INTERN +void +os_slot_alloc_tmp_encryption_buf( +/*===================*/ +os_aio_slot_t* slot) /*!< in: slot structure */ +{ + if (slot->tmp_encryption_buf == NULL) { + slot->tmp_encryption_buf = static_cast(ut_malloc(UNIV_PAGE_SIZE + 64)); + } +} + /**********************************************************************//** Allocate memory for temporal buffer used for page compression. This buffer is freed later. */ diff --git a/unittest/eperi/CMakeLists.txt b/unittest/eperi/CMakeLists.txt index d3fd944aa9eff..a833f17f5bccc 100644 --- a/unittest/eperi/CMakeLists.txt +++ b/unittest/eperi/CMakeLists.txt @@ -149,4 +149,7 @@ file(COPY file(COPY ${CMAKE_CURRENT_LIST_DIR}/row_format_compactad DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) +file(COPY + ${CMAKE_CURRENT_LIST_DIR}/row_format_compactad_encrypted + DESTINATION ${CMAKE_CURRENT_BINARY_DIR}/) \ No newline at end of file diff --git a/unittest/eperi/pageenc-t.cc b/unittest/eperi/pageenc-t.cc index 190d3eb62104e..ec7498200ce20 100644 --- a/unittest/eperi/pageenc-t.cc +++ b/unittest/eperi/pageenc-t.cc @@ -71,6 +71,7 @@ fil_encrypt_page( ulint compression_level, /*!< in: compression level */ ulint* out_len, /*!< out: actual length of compressed page */ ulint* errorCode, /*!< out: an error code. set, if page is intentionally not encrypted */ + byte* tmp_encryption_buf, /*!< in: temorary buffer or NULL */ ulint mode /*!< in: calling mode */ ); @@ -87,6 +88,7 @@ fil_decrypt_page( ulint len, /*!< in: length of output buffer.*/ ulint* write_size, /*!< in/out: Actual payload size of the decrypted data. */ ibool* page_compressed, + byte* tmp_encryption_buf, /*!< in: temorary buffer or NULL */ ulint mode /*!xr0MZdHF000C3|NsC0|Ns9000000JN@d0GXMYp00000000sI-wuY(Bml*X z1_t0XxI9dRvJd8yT1;k!E)_Ni(fnDy;MY@Ok$UI)>3NqMXw2tm*+J)JT>A_Dd-qYJ zx{RVcaQH$%7G~E)vqV*=QNs7DD>M`*lH`IM0I5r5RcPni>Nn-NB4b@994_9Wuk!Kd zRIz|L^_GI>1=KS@d(Oi9k@7*Cbjun*@UUo838B{hpVOhP?PTL|7!S!*H*tv2+Wt79 zu<^c*ca7@a!gQF&=zJL`QYDOe4^Fe5w!wG0axn($=v{LO35pRl##XOU4EXfJz_Ta+yXXcNr(BD zy?#mEA>AW58K0x~;%2$SJ9rL}RyNd^V5t*Hh@v0@aMvVHazZ^ZllVTrHHHiwc}ksiinszRoH{(ro*VpxM``PRTmL|ogDdEuS~P$k6sIF38Q8?Zv`7O@CKbrItiJ zS5%rEFW3=oLdL%CF3Y@{0}{x|8E=1GH5U=K+B^N@wv}!M>Qv?!+*$RX0vyt>75J@+ zPZGPJ70@>@(ZpD#9^AC3O4RQ-AJ}}yYI10yt<6d{wF}v-p&y^1UkSEvCb<*Uuwjlm1nf@lh~HwGp!)jLzhl@wpy>F6++*0?6xI%SYl=U2|?s zD$n14Wx<`ZR4LtB(?r6QTh<8j=lkS!eiZ!KU@%q@i{UQTW1`rk=^d#jkV6a>JMswdIK-BUdm`#u6zTQFQ~vnq(GyZN zINTqHW%|*&WQ4_JC(S-Equ4KKf+r7|y692*LDv;t$FArxm&X7NuCM5|yj5I%mXjFr z5QOSG$OhPVj|y^bXx;2bRU2F{gThtT(3lPw*fNqO`B03YJ5g*-GpJ&}YVzV62x4_o z(`Qc$iZd5&pm@=X%4YQc7l8ddp}4!zHpOz+HrWS!^X5=yJ<21Z+-JXzkMSv}jv5RK zDy*l`u4;EK0_b9+C|5ZWh3TCB>5}9sKI#?8&*)sb>0yb*t|IT=ASDsql7U{h27{5DC5kIDxz{PpcJswc+9E- zBP4u24hraWrUm}vxJH5NKji{u=HzYuAyz~63$TB1kg-S}hj*deOX$T}iH`dIj4kk3 zGTQAk*sis&i7}g5J|Ey*M0)obU?%i%!l)+7@HE_K+gS zC|PnzS;eGU9vhgsPZoHfQR)QFH-&s`kC}(qR^QGTWij?pMpB#i&oN6N$g@5%&R`&n zl3uX#K|kXpS0AN>pd==-i#%5p5N9PQDGl%!a8?L}E?dZ>1t(bxcWp~OdHP)R8cqy) z`B%nS(Io)b6h~Gq0>m+`$&?YC(r;Zr*bKUPiJ-$9N_Epmxq$+ZYpn@l7+d%dvC|ai zJDE-8f;vNq?-+H4FJFVIKG6<~9CYW{<}g2Iiddv1-HXY~3^1#}X-;H|c%JPQP3LkI z^X@ZTc0e#~7=QPn|4E~2-Cu-VP@;VfG7KK?;B8dI-+01VB>;kxJzVt}Cokbm4-(N|oMM%?lS3#aji6HwFT%CVt+ z^dB^vW!0h<#{7p`Y%DLo>!q>YUAr-H7GU@^;aQ;Nlh(xIe-?45Ne5}rJQ6LvZIl*B z`qqqo<(%sR2Bs&x54|amKvXek{UDag3*TD}Z?@<1fkjyV>z0X4HaRt{Ci7J(X3Ch3 z0K2uhY+aF#sh7>E`phMG%*^riY@DWS-9}{PLhZLArE8dR_BZ0-RvpLYWkYjz%z5CJ-0DMKL?bh=U8`XhhU-E-;gW`jWBzmPIi00yNOxEvFL!gDzhTcS z$zf&mQAnJ*x@B$}bZ3oBq1&Ny8aods{?fY2tL*BBfsyOn2g z^kvAT8eQT%%Jau@Ief%K@^PFfS@MUQF9igC9Q)2`Hc^tkSLm)h5qGy`-a}#ne6Xdp z4-UuylEsT2Q@OjYiJ0^V*P=f^rrIb5&FLTCJd-~ZAm9FF znc*P$aU|CzfHjr;5M%`v7Gt_i|CM5AvA>mdfYQ86l0W;;aKJb)H+J^Y{=gE_roged z{cX4*^Ob9FQ7jreS0idB&icD=~D zE*_pK5$v4UY=K8vdK^3PGCR_;3(=( z{U$Te~~6G z%f9K};ovT;rjG0dCYL7ROzq;fRfQ0F?^|Pr2UXtl!|Q4?B8R}!;1=5WWeZG2U(_z4 zvb-+;zg?1A84?dU+(U7*Y+|mmUkdlzGb6|9Oz#w;XDe8-LwEQF94lWotVs<{mK^%g z-qj+Z@7sBZmdEqz^LvS;M7y3jSV0YPxHoyNm-HBZts?A>r5c&R{Xl#k5G&zSfDl{s z?OMAhji<)HSiHDI#3;cwtReRugg;$M9$~@r2FB9bB|6x_-vMG=A;9e0+_$YOR_mP? z9)!*IFP*!f#C5pF3H4hF{h7B1sZY@V`V6M_*ZLN8zBv`AQK`TjAmwZ)XAr^)MWk8j z^6=)ZMNjmRmx2&_VeF`y_*+ciy44!&&>UN@ql{&uMwTMjsX^z3YLfv|t^$)<*$BgJ zR~Tp(SIW3xlHiBf5fu88p*7vURPa7?CjWWlSM%w8rBl5WHoT0ZX5Y;*hx_#i0)PJT zvop8+oc?Ne7Toojk%FXfW2 z?r96tpSB?FrFLD$;|8Bwu~kVA6uI4-bEXNcD%z=l8_z>O6=Q8G!Qqbi^zEb z==v`aXaEd0asN(b&jS4jC=J;iibY~YBWX4eCN2HV64_m&t?=0Upj&3H;oOe!p^q$q zz(I*S2+GWxrb@~Cw6Qhsl7=Z@{Y*5=YyP0u8^YTpQ-=o#b-T{66--Ms`bp5YDcu28 zGrt#6$(t(Ghir!8D(!j1QKa;U15>#m*@M@7NoG-Yn>M9fKN`W`D%tz+dn>8aB|y@F zktz8`krcL<-!+AV8@8$!w@`79ngcsnNNju{FH_CVgM3p6+lAv!UOQ_TJRNS1r+U0A z{M5@Ac1E5RG=s*hLgUgSfV8TyfrCAa;w6tAyGxBgIJG$KuA%N%dN0=aj4k~?X&q!2 z)oCom?70uyp7!-35E{uWQm{vM{Jx{5id4IL7ZZZiLbU%CG(5~HiikYY&s^!KRx4Yh zRmuIaSn1aW!RI>cqVmffq!%1uBR4eLR?5|r>^Gqlbh|K=eG#RyXhUO*h#kN;XcJqOpZ;W=y9$y~q<@nvYMZ*J;|Hm0Bf6^msMy1MFl-1~JWv&&`j6lN zkr9$&@E4ZP?KppNFE9qcV))!_FxP4(Ehir7O|xs+gd{-_XEXmLsoeo4}rC-qj}LQ4@}`jXrM!Kk9h%?JLuVW@9U)K$TVoy`N|` zJwWs@F^gvKhTe0nHCd`5l%pHR&53I{oL$LeP=aKm#6b-NN?UUj1at!M`3y0Z3;-%_ z%lD@+FQ`wmFj3RL6tHX6{+FAg7w+8mCimlrdzR2{pZ1CzKEn z9WHmG3N8eGMLQ)oWx{-FrOhRRsQr9|wK$5{!aeZkj*}mxP zJT}IVUf)(X5Caz9qB29VYaFsDoL{uzq2UPdF{e~f0m;fa@zoR6K-@0Ww@k- zZ@3`TKYGwT!}3ykC>~`V%M3dEdBv{m3x2-GScq|5vu~2?uY@3RI1pMzDp6d7tslYU z_qU_+A~%bhNpQ^`_(_17hObN|9LLE|pxgY0ToDddpLfC&?7ds=%vye6vqwYk7K=a!4HJr+n^|ZZvd|6 zsh0`j2ZOe^PVHg8i+~D5mPZSZRA-2BLB!HgHG!I-06>)(sxHAE-)Er*kJ3NaJKq=7qm1F^MU?uFVk>=uz0RPY-8Nx|p& z;gR#2inB*Ti=;m*$=lrY1*D=mmQOq~k|nt6-y;~{GVAM8o_e>N%q^^<=(35aH52PH z6KOjDlG0bG3|?H zKa8iFU{%t+pVzAg{GIF$=gA^d%s2KQpnP|}71QFvHb`zOjH-6^z5mtAX9`5{Z=yZx z<-{_k((Iy1y2nuq?Y{!?%syDC-{FFrN8%9Nb0lEb8UGVzIvY^XjNRJw_zl?3mtjfQ z=U#dcnokS+ILhyJ3dpwNs0+f$4M^qfL9(kSY1I$_pUvPoAOPnDH84DH#<`<2od?SL zmPos|9cWef9k!9v`t~NfaCMB@&PBO+WOp(;5zbmw$qlm8{ z5y#~l=7zD;7qszF{-N8J95BFMNz$+&xlFzI=JNVS%A}m<`bAqBpRKFSIXj)ezi27_ z7baIIt4+r}mC&uQmK(WBo8F=)$Jd$+nXK;El;5<7X&MFWV-mIhO{JrO+IA`Erdl+B z&|5E84qgL^+o3=sFg^G__gZ1_22cMnJ#vxio0-14ZFXB+IhUdIJ3b@59r* ztdXhHzp5mIG@9CQk2W2GCLFsNd``Htv{>tk zz&m0%w93X<$<@+Fm_#YDN}fb33r^ocWJPTPTh!@3x1k5+Y`Cw2a4;L&wfP}`(#_5B zzNUT3s>@gaJ0m9^tc8IoS#gt9yr(5QE{r6!Q<+a8?WG(bKn38OAby$Ek>m9~ee`mO z#OesmFVVQ$WH)c>j@wvtu`u_K95!j@Gw++(Bcm=3!uKB`h#W@MB`u z#+!7G|Ltl)JrNxPW}9$4$W0Z{>9ZbuB^xeUQ)O&i)%-hduyXsp$n zHJ>y!B8l&Q#Y?9|B5W%uqyA+%4NZ2@dcR=yGH;u-{P@s@ElEPVkI@e@7IujSQlZAD z&PHoO6tbNDy^-}tQ6#BRN^KiQ-uDx@=YK5={1pMWI50+RCJVvS20uw<7p;L=?9$F&fQaj06!$ac5-frYcnr=l%5xzlH3vK6ELPr+Vr{&eAF&gy;hkE=9m?N<3eR zWz#T+p4EsA@bUp%5&Z;Q5#4#wf=L;}e%6ZW+>(obIfVktDX<&&d)mqdY$}Hp?}^!* z)!y7gM5FUt0F6O3;wc%|W1Sr9_A9WJS1AkLQ=^AD*~dUaB)fJCmu*eZ$6{~Yu7VK< z(B|&{e>9W4uoxZ$V#^WRA+OkKP?3w^v)5-6XPb&S^|G4KpDo)YUHO2>oG31Am`+{3 zntfyc*RK4n_~oerez3K;N!w{13+un;ND~7u*|RV0IjeBRgbgbA;vDL4>2V5^W~PzU zu9D)oj}${aCN=hD)FNHKgICt9C|ny!cvyMf@GtlJ5wnW_c&W6VzkH;ZX0pY3-(a=$ z1Pazpmf>~uN9__lVBH#-E6_RvlJ+5Z z);-N?aGc>qnSNGa--P<#GdkB%)~vXbd@s71%%h z?XX)MtbViF9~~Ju0{9t#9CDDI0bryhq_XH|q3_Fq`dx~aj3{}rWIk)f#0AVeV73rJ zYs!Rk5*Maa_9huCJVxXH!wBUx(@EVQFMtc5g#x4&Vf43M4=fzjulFBAb}VM5ccs5Z4nsQDC5Eq?+Bk}DQ7bZ|XKN%= zE5x=49KruY6z6xzdllb5iXB0LW#$y_BG^oPD-@H&WYxfUWd;w*vG zWrk3j?sEcJ6w-8ou#4yUx@S{XMnKbH2xry*kd>Z5|JkaHud4b3;kc zt)CJX$P}%@*&#)8%?v_3LSp=l{uC);*v5M#4w66P{wF)2Q|eo=VaV2=NAO4!Ep>~1 zw!F^}p>`y4%pE;){KEJy=~A&yUZB%#!ialof^5eR5b3(9R0^PbfnVGNtNT$j`3QUd zjzI*HAiRai?5zF@iAytWRp_^AAU}rFM`#dz9^iM54%u+Vh=^JLJG!KmpM~N6y zd4w%*pOGo`<6#&OSX8Ny7z=DUEBIWm9ymul99#5xXB;}!5YNB{txDND z?g50@kbm95Ue(sdwjYk1L(rnmR7$)8+dNj+*M^yc<;(Rx2@{>M`sQF<=d83FP|4g(8Vu%?$b@b~&&k}B972R^!d(68U8owFCR|*63#SX$V zYj;ERIzd*jIaTQ3I<{cz&M!da$j}kT7MQ{QZ<8@yS9I4?vjE;! zNvTezW1cjb_h+fEKYtNV@{Ji+Ky@RP|AwYZ1WvTKr9dLk$o+?k0$b;2_d02gt=fH-% zVp2Z_)^C{kMCt%%3ng`INOM+IPZ2>yR(n6sDy_YxtKATmXit5B`XO`(=$Q&NgC`;y zl{la43Qg0W4i%R+tW6M)W2i7!iti!qbb|3swODyTTd2I)h^wA0Y!C>W)^XycJM7qR z-O-*5=+U&n#!>ECSOPDQ&u1M%>1)X&^>;P(X`6(kXG`J9`m#isxyDc;Jb4aUO(Kyn zI9P-xh8L#Ofuv6#U=6f;W>7I$z$Ah(r8(75-#>%h+4u-|#@o55xq8DP`6kVt$r__0 zk5~DiAjXn_=CFX$|EYP|k%%k@#ftT~^;>g(;Z3?aZN__L-4DL_(wntsX8%I#k*SH_ z^==cnUmGs%*I*e#bT5gkl+o~he$06&+Mfe${?bc-7*qQ|Xdy=|i2zHxtv_vcJHL`8 z9t$<=H3np2s2yY^{he`=1y?QYXTLL|RZ&pZQXGxZ8ZSobN+v@2!+oUPVoTd_r&PiJ zmYhdc1Xus9W-Bpe! zOYhJ*Mqw)up(zWdU6?t$btb&U=An%^s-g_o@6eps#}H;aRBgoWi(zov1KkQQAzAi* z7eUT;3?x3|@u8Ol`py9V*)u5}9k<*kW>~n`n=mm~>lRx-8;vLoPm~ZSy>ya9VH7w=H^f4s9mzU(}=K3 zvq3g^d2zg+nR(|Jb~;lIyf%bqI9=&85`5zpIF?eLH&r~fZWr#G3tx!x|5u$4q0%Ga z>gQ#P&&Sir=(np+P=6$#J5^VE+>4B{eC7je{};f4cMxv+V04i4? zk+dfz2;4!wL-@BS=i5h%s8yEa@q$9C>&Qo17hqq8YQ*|m4no-b=^DT5tFXbqAegai z5X7`guhXWFIznHeBnCD;zhmIY%+c+*PT?+@O*7Zt$TYSjGy}vIJgJ3JV=4=IK*9cC z6XNbO7yKFUJ}sCJQqh`MYB{kGUpJ0h@)u)Zw74aneElu7ijQkQ6%^`bW;}+3y0d4 z4~lSoJ%8ur83mTFrlrO3_!YSc)}F<@B%f9KKD1_(It!--_G~_4hb=OQp(|6B;LcYg zJ{D-D)?P3whlum75` z`pR|1p6bu%2)JPYc$CCgsVsuvO1pR+MWIUExLBf;1qt0j5&Iov%|Z$_dcCD_XgHz2 z5n25PbS8ba7gYc#wQriSE^6*9_@jWOPhMH9``sbT1waE%U>e>}@n3@1 zr?CzAtN%WPZg1eT-CGRIMwN~Wk#}ls0AU8e;Gca_d9!|9~o-|C8j`wYB*B+6+i45s5oLMVj4tU-wN&+ zA9{*49a6>r%#iy`_q!(LwdzP^k{)YZ&nE1ya89ku1DmTrSI%JY$cTrQXPgHlL-_$j z(QKZR&C8%w#we>g16Aahn)h3PKlbE<)WS2Rq z-}3v{$;3B=p}i=jk)X~ZkNA!LHvfZInOe48!jUoyNu8@R;uU0VR?V0~(T@CZNtQQJ zcn`u6Zx@@6>C>hJxtPn-Sh~EJAa-{~Dz_UnkU;JRS!wA^1g3QQf<@KC)*-i9fOP=l z;2{C;)Aki|lCmptT$4GIZa837tq=-QdBL+1NNRy>&oW14ZI4vWGZSPjVw zvr-{Muzsls1o89R0O;u?mOZ2%L|Vp<+`w_D3FZH&t4%4IZ%&fXG6FZ&Dcpz0Jq_{T z_*AVppp(Z41y9>Sp{DcTvpcVY%A_81A-b*OU!C9!IeKS%Io0(5JAl`m0LAPTxdX_z z>xKQp+y6-{`L^9GfQiQo7_aRj$M7h7+kjtE{z=PL*TFX)SCbQNBnuI*?ImUY?z&cGZU(bzql8QB}><0~1pLt7Mh{vy{n>E-SEEJ$E?fvVQO z8=|O$L}?n)FAGaSt0emilA&z_;l4wq8asbR7O;Y^@Mpvuxh!woPzPl}Z`OwbudOl~ z%t!_%1QYuS>f#hcSZM9WT0OS`V|$_A!t#j@p{z(6fqYf~ICnzYxGJ|?iO7f~&f9{u zIx?RC#=B;URpP{<87x>Mz(`4u-Lz=eS4YD8h3dc}$(nZf9{0j>E$&&`fm` zcgq&4(s|9#aAZ z@@=k5I#S{QdfpcECCk_dQDjO5En{po^%UGLZGsY&mB}0uQ8bz(fyi+EMVHjI8YDXG z74a$E3MujF6tWfGM>OCG5FT#(_x?mqX+)k2s*H?jf&QGxhlv)fM^p~kEW%1N3+ms~lltan3(bjJ% z4SU|aX2D;~Ep5t5JNQnT-A&6_Pqn=vQQPYn-_NKN_3tYoT2s?B(@NwWAlJS9;6}bb zk5vk2*;yRAlz`!j(RhZ&wmeaR>0u?GO}A8&O9NLjm!D@~d{(F7bv1Or16+wkfBH{2 zlkWvxsr&0OlRme!pTrHu#v=_yL9`ZM8nF`NgqnAaJ)3OsKyJBa8&I11q7CB2yS9Yx zIMLg7|D*6ISD4bCj=GgEM?C*eF(e~Nz|KFHwop@C#_zENr2f1TP!r*h4jZtf?m{mR z!px-m=C_7l+vu@9Fw|)5D>)Q=49q3(_W~e!%|~HngS)4O!K!B;v;xZGlt$|!)j{ex zQaHY^Uac{OidD7L^!z2g>D{3`3h>&fKnG|ir*W?R2a=-85dCZ>ecr@C8C7zDn77+>d%jF3;>urgj$IWSmz1 z2))pF2?JyD6K{j$eAuR~#gQGM-in9fKrF;M*lI?9)sNatw061Kra{StjTNDV6NIBG zlIQJ)(#UvN97lTm)sPDSCK|jY)GVi6v3Hj3PeQmxAr2!^rHusfCrbp;}2z2W4~U`?Pkf863aNX<_!%P@P{G)u{5W(kvNF+ z8F0@Y+RA9{<4izEJs69nquLQjd9?hXi=g0eJgAc`hF1-P-NQVBIkO;g9ac+n15 zW!;sC7Pw`CHu&cW4=wqWDvW-+LQtkW{zO)hX*R%y5wcId#uY+|eco-+kea|2 z_?O+dNN}~?rSg^2S+@wfLP_660uV3wfnRW94SDNVuNV0(e7fEpPmx|L+E@JGzsaiK zyWxbAm^BzF>eR-+9^<*cC2O%?TucRQ?CTQ7dOLPpAJ>O=B_G@JWV`keHP7PUlg<#m zm~IPEIl!7{l=~Q@9OySmKYLT1t%cn7X3;e=UoThhbY7L@9xj5x_-A#Uu_Xn&&MD_?MM?4Jn;qAYZlK6*K zdU@R(PS^R0(yt{jlpN>C=#^)o>Vcr;&QUzkx2Gq4swZ=V-g_W#omkLVRma)iYnn=h zEL$XFSiN-Ifo^MW$~Gx@Svq;8cHli>yX{}qIX8%*U^0-cbgx?DhhUAs!NLcSlXYP^ zxO#gaA2I+1E4(Cm#J!gtvVCTCYuUvm;4Q0^@f zRJ@`?m%#^B@WA+c>)Y2U**(Nq^dFVsa;`IdJ9!K^YB- z%>Y&sXZ0O^j(mB=wKkcV__OTT1#LFtg7^)|p`9$2Jsf)&Xs2m6h7E)4d~1Y|>(ngu z+0|7-Q|#$(x*_X1;D|fLB7W0HcT0$vVi0f)7u0@)G^1HUCG8yMOwdNG>=?ic@iNgl zG|z4DRc@T->x2m4*i`Z*g}US4^i=vXGEqELv{hOb9ejo`Ih z45E8tmhywgie7{I$^cuN{BMNK(SfV(wpw2wg1rFnopA)2yuin?1}bLg_5w0`<^LPD zanwT|c+amr6$yL>!W!ki)8I2c3y2&PT$B-_SIpR$C@CC;1q${_TS&Xv z3BC^B|zN(DFRr_AJ&AgaW1kjXz z5=;G!BFknJ$aqhNf_>SsK*>yatVZ;quFi0Hfm;Oz%3^*GyIlCqz{Y8ID|M^FR?qgX zo*+hFV+uc-lel}jU5FiV(^8If}3ic1HnZ99~)%BvfTTS_1qee_u}h{y!8t_qMGVF`>8@L)x-ir?&N zT1%eoo?LG^4z^Mce`~K%>({^C^q{4e$i?D(sk)84l(AXBvarsxnxq`@Svn3P%_J)m zb{x7~h&3ZrxV(uRM@4HjBNqf1Y#4wO*lF?sRu5<)t~e8AQN3}c1geSatl`qkv92G7 zpsB>hM09FnLoWm|>MT3waVtU1^n#-@7;{A1-4Le^rEmn6BzQ{jHVu8oJ*6$^G0PXE znbGj+aXp!$N9x@v8X120v=36r>yB(zZ^H*s?Mr2s>L2)_81?b!#fIy!!^0Pz&t}S6 zA~5sAJ-tpodbjia4PFRP#pc)(n*3@o{~x~ie9V03RnPfV6EH8cho-$C=~mV?d5d$A zk1=Q0iWkXdj)&{-{^7;|^0YcFZjQsaIRzUjNT^`*f;?q3Z1ivovlU~x&7kTF!qR9~ z(B2kYSGU_&a!&d!q15Ot**896nrK%k5ZSzLq-hZ2d*3}%s;0abqA=8gawZp`#>7sr zjM-G`j(}-)tsHA^)G)?uZ{opO?=6!%KMesdemCaoH@2~+l^Ry#p_9-urW8NNObkzz zys!K-CdAQRLyaa&E<+;zmtGD>07xMsBbliWy>P259Uw!{(&W{4RtPF#RUxNX%Xc3f zYrdKRz!Tep`)$@{o4O|2nI|?azaPRG%EY*loNeLbDfd)--I1h?pGr!84$tKuFeNZ2 ziPWz2%BTFZu?j+e!%VpYCA8p01~B*rJ%~eN3X1#BlVW5-6teI(TY1kfE_!vP1l%G$ zLjsaxJa3P_% z4oLq-1G7Z(7mT7ZqN$&7PdQZBhsJ5Kfq4Y&(F0VKoxM3Ru~{d~j$7v-CaEc^PHt+Q zOXkj|=c)$>3^so`W{NL#VEnaNoO*JeOqHsUX6i^k%)VZJaYt1(widtfA)j7>gbICV znf`++J4Z7Mqq9DCBgaD3psKSua~ZBP>d=ZeIrA+G4nxC;w7BN7+IBNc^h0hXOlND# zlAGLde61g-ib?~E?)r>;`VohqX?ifU8F5Nbt0?Tpf9kGm%TA{d3VE-p(+9qTOCyjH z(=;iAJFn^sCns*&wfYcow7tZ>P>@vO+ zAY7p`zb`grUf)ML5s#7$!aS1QePP^$cUY3KbgqT5#>1{JXGmCN^J#Uxh6^~2KPHUy z7L7UI&af#5iTfV+?0P$o!jh+96wnBWm~mlY2%&xpCvS<)SFwVvtHqyRPk4PZOY|PK z7Z0d&xgJOfSwT2#33rB2iC-txoSd0Z?e$1<2Jjd(^QUg7Oa}F?6))R#UzhgIFhLk3 zzyNny2JC3LxoKD$gKAlHm49eptXw{g=8OBm6wFxR5Kaa&?yc4*MtZa_i zXg|7%p2eXRti57D-ss|PB*jJNQTu|{l+BRKqo%BwwtxUiO{c1=7w0{#PqM`%UHjWd zzK3W@0Amf;dRkV|Yf$is&S0`~F{p4C@59eoc7u_MdiDg+vl4y2|GxbM-4bk1lfF&% zn4{GJ%yRJw`f8s6aJ7L5ZXo;LRK5C*4aC*h_T?2O9X|Z~nZ@=yt#pM@%O(C0B6YDN z%X>A_n>2HMhSWLkSC|6AUxy0$S^X}L{TN;{$@7!RXI=P*ONb9{IPyP(Wces+@C+x^ zk#uRx(7FktcyZv@+qhEmJO`IT-BUt3@(O6Lq0)+goR)5Q6s*i;w>q%_S)kxS)#46c zCAp1eTK7HJKYXh)7e#u&T@Hy1Y;(yh#d7ii`FiR&M`$8+psLqnBoD?Sb3WhDy z?LoP>qbF9Kt10TFN2LLYy?y+F3ui}NTSStY-0C`{!H0y=Sy1fDYqz#&jS|6TkNw%m zv~Bq!{aCA8sFF|W30yM%z`5P78rX+;VOwJp9M(6}R) z9bJXI+&VaC&4qGZw9cVp;*hz6deQH0Nzb%Lvh#e@utb>4=J2-BJ)_1DlkB_s$m5hNKNg^rkg^a`Ej8rN4ml!-p?79)xZ}pXPFv zN#Yp348tBfLMeTMMYrWxCC16WE5?PtA{o_KDo-36ZhTgj!*7tzV3TdnH`if+FI`JU za&r+j?*?xTmp4J%B_$z&R76igFCl>*kKR{2gb!2jkyQdDyxsbDHSF2?bg{a)%{~Tz zSAwE5l;3$q0TQ~^)fxJu0kV}DjZN(H-`^{FWiN>>G8nJv-}7uB8c$Gw&(xe zF;FbMx)c0|V5=YC)vL2cvQWja#aayV#xwS6F58ffkt5~}G*m6ndyOiE?9`^?zcT$O zz^ER%3#|R8S)SVLtImlP3dlfBJ8*b7JJ3YsQA*{HuPj#o@4*|;Mh2uwt1-NLcN&w3 z=2Zs#V7J82>X@-iCA+f;#GfKAME9K}?aw%o1(%p2sla@!pWVox z25|Atr)IlhEZ}}Bf8r`jKo=2_D1Dytej!SoOo^54p+I;10O^mP)xzXHgAmx<#g^NZ zn3+0-OrC#HQH&oa%Pj=OpDDOvD&I~UYncf$wopYj6)wAXJ9V zhcjFUtH7ExO?7Y?gmfWRM0M$UzDGsZYS`lO#Fl2(=P7+;+pWIR*<%AgBv9v7@AZ{R z8y?{h86*F3|KNbN@n4hNd10@p7T5OV88=*(5`iSGrce65W=%<2gmM*FkA+?#Lr!JzZ&|?< zZ*c{WGl!F&IhL_7NS$B$Zvg1rEJ5{~S$Ml4O7=uDgIv)gnwkTnYn78DllK%&bpea%K38jO~v z#3X+%lIB+uVSwkM(W!Aw@{dd$m|Q3GUqy-^FS7$*%LVE2|20X(Gra#$r`#>FDka zg;;44S=-sPl_Lse|M2G#Uy*gUfU`LjOd~!&Ue>r#Lu!)PxJHY}^dh>vIq&mCzdrJ7 zX@&5jFjkG~i{7ApnWnM#$|E0bZ58uN=}Acd&O+`)N(-@+{1&Vfib}4aQJ7~(*sJ@% z$%f=7U`tE&d2Bhf9<1`IDLKjA&Po&frZ9TSdl}&F7?Lyao*x(|*fgC|sdyb2oB%V3 zi2-R(kzomPLoTbANGK+gZJS+_&x%C1vv_fD&jqr*F`^!$3i~avBB4Z(#v>XLUIi4} zp8Gqg7{V_bk2x@Z$*Wrt{?<6diEW)@TZzgJDQ9hqV`s|)kX0N7VkL_O4butjDP_7` ze}P2SQImj8^`Klp<&djhYG#hp46)B-y9Mzdu%Iu@$L1LOwR}S#*bif9eNnpx5VCQC zK)E_~KvWU^@-WCp)7#uX1xd~U1@V8=z2PVyMmC9#T@TPnMjHe|N@8NhHqo-HiBdhl zjJjP>JQL~HS(nR!1FF*-VgH%jwFvprdGQ5)pQ%3DuVZ>(4&~OMK Date: Tue, 25 Nov 2014 16:03:10 +0100 Subject: [PATCH 70/70] reduced usage of buffer variables --- storage/xtradb/fil/fil0pageencryption.cc | 61 +++++---------------- storage/xtradb/include/fil0pageencryption.h | 4 +- storage/xtradb/os/os0file.cc | 2 +- 3 files changed, 18 insertions(+), 49 deletions(-) diff --git a/storage/xtradb/fil/fil0pageencryption.cc b/storage/xtradb/fil/fil0pageencryption.cc index 6e2aa09459e29..503e379fd7e7c 100644 --- a/storage/xtradb/fil/fil0pageencryption.cc +++ b/storage/xtradb/fil/fil0pageencryption.cc @@ -102,7 +102,7 @@ fil_encrypt_page( ulint encryption_key,/*!< in: encryption key */ ulint* out_len, /*!< out: actual length of encrypted page */ ulint* errorCode, /*!< out: an error code. set, if page is intentionally not encrypted */ - byte* tmp_encryption_buf, /*!< in: temorary buffer or NULL */ + byte* tmp_encryption_buf, /*!< in: temporary buffer or NULL */ ulint mode /*!< in: calling mode. Should be 0. Can be used for unit tests */ ) { @@ -214,7 +214,7 @@ fil_encrypt_page( } else { tmp_buf = tmp_encryption_buf; } - /* 2nd encryption: 63 bytes from out_buf, result length is 64 bytes */ + /* 2nd encryption: 64 bytes from out_buf, result length is 64 bytes */ err = my_aes_encrypt_cbc((char*)out_buf + len -offset -64, 64, (char*)tmp_buf, @@ -312,7 +312,7 @@ ulint fil_decrypt_page( ulint len, /*!< in: length buffer, which should be decrypted.*/ ulint* write_size, /*!< out: size of the decrypted data. If no error occurred equal to len, except for page compressed tables */ ibool* page_compressed, /*!(ut_malloc(2 * (UNIV_PAGE_SIZE))); - in_buf = static_cast(ut_align(in_buffer, UNIV_PAGE_SIZE)); - + in_buf = static_cast(ut_malloc(UNIV_PAGE_SIZE)); } else { in_buf = page_buf; } @@ -439,17 +434,15 @@ ulint fil_decrypt_page( err, (int)page_decryption_key); fflush(stderr); if (NULL == page_buf) { - ut_free(in_buffer); + ut_free(in_buf); } return err; } if (tmp_encryption_buf == NULL) { - tmp_page_buf = static_cast(ut_malloc(len)); tmp_buf= static_cast(ut_malloc(64)); } else { - tmp_page_buf = tmp_encryption_buf; - tmp_buf = tmp_encryption_buf + UNIV_PAGE_SIZE; + tmp_buf = tmp_encryption_buf; } @@ -459,7 +452,7 @@ ulint fil_decrypt_page( memcpy(tmp_buf, buf + len - offset - 64, 64); if (err == AES_OK) { err = my_aes_decrypt_cbc((const char*) tmp_buf, 64, - (char *) tmp_page_buf + len - offset - 64, + (char *) in_buf + len - offset - 64, &tmp_write_size, (const unsigned char *) &rkey, key_len, (const unsigned char *) &iv, iv_len, 1); } @@ -474,10 +467,9 @@ ulint fil_decrypt_page( len, (int)page_decryption_key); fflush(stderr); if (NULL == page_buf) { - ut_free(in_buffer); + ut_free(in_buf); } if (NULL == tmp_encryption_buf) { - ut_free(tmp_page_buf); ut_free(tmp_buf); } return err; @@ -485,18 +477,13 @@ ulint fil_decrypt_page( ut_ad(tmp_write_size == 64); - /* copy 1st part of payload from buf to tmp_page_buf */ + /* copy 1st part of payload from buf to in_buf */ /* do not override result of 1st decryption */ - memcpy(tmp_page_buf + FIL_PAGE_DATA, buf + FIL_PAGE_DATA, len -offset -64 - FIL_PAGE_DATA); - - - - + memcpy(in_buf + FIL_PAGE_DATA, buf + FIL_PAGE_DATA, len -offset -64 - FIL_PAGE_DATA); - - err = my_aes_decrypt_cbc((char*) tmp_page_buf + FIL_PAGE_DATA, + err = my_aes_decrypt_cbc((char*) in_buf + FIL_PAGE_DATA, data_size, - (char *) in_buf + FIL_PAGE_DATA, + (char *) buf + FIL_PAGE_DATA, &tmp_write_size, (const unsigned char *)&rkey, key_len, @@ -505,46 +492,28 @@ ulint fil_decrypt_page( 1); ut_ad(tmp_write_size = data_size); - /* copy remaining bytes from tmp_page_buf to in_buf. + /* copy remaining bytes from in_buf to buf. */ ulint bytes_to_copy = len - FIL_PAGE_DATA - data_size - offset; - memcpy(in_buf + FIL_PAGE_DATA + data_size, tmp_page_buf + FIL_PAGE_DATA + data_size, bytes_to_copy); - - /* apart from header data everything is now in in_buf */ - - - - + memcpy(buf + FIL_PAGE_DATA + data_size, in_buf + FIL_PAGE_DATA + data_size, bytes_to_copy); if (NULL == tmp_encryption_buf) { - ut_free(tmp_page_buf); ut_free(tmp_buf); } - - #ifdef UNIV_PAGEENCRIPTION_DEBUG fprintf(stderr, "InnoDB: Note: Decryption succeeded for len %lu\n", len); fflush(stderr); #endif - /* copy header */ - memcpy(in_buf, buf, FIL_PAGE_DATA); - - - /* Copy the decrypted page to the buffer pool*/ - memcpy(buf, in_buf, len); - if (NULL == page_buf) { - ut_free(in_buffer); + ut_free(in_buf); } /* setting original page type */ mach_write_to_2(buf + FIL_PAGE_TYPE, orig_page_type); - - ulint pageno = mach_read_from_4(buf + FIL_PAGE_OFFSET); ulint flags = 0; ulint zip_size = 0; diff --git a/storage/xtradb/include/fil0pageencryption.h b/storage/xtradb/include/fil0pageencryption.h index b43843571f4e3..2164aeaa0f856 100644 --- a/storage/xtradb/include/fil0pageencryption.h +++ b/storage/xtradb/include/fil0pageencryption.h @@ -87,7 +87,7 @@ fil_encrypt_page( ulint compression_level, /*!< in: compression level */ ulint* out_len, /*!< out: actual length of encrypted page */ ulint* errorCode, /*!< out: an error code. set, if page is intentionally not encrypted */ - byte* tmp_encryption_buf, /*!< in: temorary buffer or NULL */ + byte* tmp_encryption_buf, /*!< in: temporary buffer or NULL */ ulint mode /*!< in: calling mode. Should be 0. */ ); @@ -104,7 +104,7 @@ fil_decrypt_page( ulint len, /*!< in: length buffer, which should be decrypted.*/ ulint* write_size, /*!< out: size of the decrypted data. If no error occurred equal to len, except for page compressed tables */ ibool* page_compressed, /*!tmp_encryption_buf == NULL) { - slot->tmp_encryption_buf = static_cast(ut_malloc(UNIV_PAGE_SIZE + 64)); + slot->tmp_encryption_buf = static_cast(ut_malloc(64)); } }