Skip to content

58558916 improve run manager opening speed #673

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
elainethale merged 6 commits into from
Dec 6, 2013
Merged

58558916 improve run manager opening speed #673

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
7b1c794
[#58558916] RunManager load improvements
lefticus Nov 23, 2013
4667823
Merge remote-tracking branch 'origin/develop' into 58558916-ImproveRu…
lefticus Nov 23, 2013
d3bff15
[#58558916] Rearrange runmanager loading code for better performance
lefticus Dec 4, 2013
d67d334
[#58558916] Upgrade Sqlite to current recommended version
lefticus Dec 4, 2013
8c251fd
[#58558916] Parallelize loading of RunManagerDB
lefticus Dec 4, 2013
ebb824a
Merge remote-tracking branch 'origin/develop' into 58558916-ImproveRu…
lefticus Dec 4, 2013
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
78,393 changes: 44,198 additions & 34,195 deletions openstudiocore/src/litesql/library/sqlite3.c
View file
Open in desktop

Large diffs are not rendered by default.

564 changes: 457 additions & 107 deletions openstudiocore/src/litesql/library/sqlite3.h
View file
Open in desktop

Large diffs are not rendered by default.

139 changes: 123 additions & 16 deletions openstudiocore/src/litesql/library/sqlite3ext.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,6 @@
** an SQLite instance. Shared libraries that intend to be loaded
** as extensions by SQLite should #include this file instead of
** sqlite3.h.
**
** @(#) $Id: sqlite3ext.h,v 1.25 2008/10/12 00:27:54 shane Exp $
*/
#ifndef _SQLITE3EXT_H_
#define _SQLITE3EXT_H_
Expand Down Expand Up @@ -51,8 +49,10 @@ struct sqlite3_api_routines {
int (*busy_timeout)(sqlite3*,int ms);
int (*changes)(sqlite3*);
int (*close)(sqlite3*);
int (*collation_needed)(sqlite3*,void*,void(*)(void*,sqlite3*,int eTextRep,const char*));
int (*collation_needed16)(sqlite3*,void*,void(*)(void*,sqlite3*,int eTextRep,const void*));
int (*collation_needed)(sqlite3*,void*,void(*)(void*,sqlite3*,
int eTextRep,const char*));
int (*collation_needed16)(sqlite3*,void*,void(*)(void*,sqlite3*,
int eTextRep,const void*));
const void * (*column_blob)(sqlite3_stmt*,int iCol);
int (*column_bytes)(sqlite3_stmt*,int iCol);
int (*column_bytes16)(sqlite3_stmt*,int iCol);
Expand All @@ -77,10 +77,18 @@ struct sqlite3_api_routines {
void * (*commit_hook)(sqlite3*,int(*)(void*),void*);
int (*complete)(const char*sql);
int (*complete16)(const void*sql);
int (*create_collation)(sqlite3*,const char*,int,void*,int(*)(void*,int,const void*,int,const void*));
int (*create_collation16)(sqlite3*,const void*,int,void*,int(*)(void*,int,const void*,int,const void*));
int (*create_function)(sqlite3*,const char*,int,int,void*,void (*xFunc)(sqlite3_context*,int,sqlite3_value**),void (*xStep)(sqlite3_context*,int,sqlite3_value**),void (*xFinal)(sqlite3_context*));
int (*create_function16)(sqlite3*,const void*,int,int,void*,void (*xFunc)(sqlite3_context*,int,sqlite3_value**),void (*xStep)(sqlite3_context*,int,sqlite3_value**),void (*xFinal)(sqlite3_context*));
int (*create_collation)(sqlite3*,const char*,int,void*,
int(*)(void*,int,const void*,int,const void*));
int (*create_collation16)(sqlite3*,const void*,int,void*,
int(*)(void*,int,const void*,int,const void*));
int (*create_function)(sqlite3*,const char*,int,int,void*,
void (*xFunc)(sqlite3_context*,int,sqlite3_value**),
void (*xStep)(sqlite3_context*,int,sqlite3_value**),
void (*xFinal)(sqlite3_context*));
int (*create_function16)(sqlite3*,const void*,int,int,void*,
void (*xFunc)(sqlite3_context*,int,sqlite3_value**),
void (*xStep)(sqlite3_context*,int,sqlite3_value**),
void (*xFinal)(sqlite3_context*));
int (*create_module)(sqlite3*,const char*,const sqlite3_module*,void*);
int (*data_count)(sqlite3_stmt*pStmt);
sqlite3 * (*db_handle)(sqlite3_stmt*);
Expand Down Expand Up @@ -125,16 +133,19 @@ struct sqlite3_api_routines {
void (*result_text16le)(sqlite3_context*,const void*,int,void(*)(void*));
void (*result_value)(sqlite3_context*,sqlite3_value*);
void * (*rollback_hook)(sqlite3*,void(*)(void*),void*);
int (*set_authorizer)(sqlite3*,int(*)(void*,int,const char*,const char*,const char*,const char*),void*);
int (*set_authorizer)(sqlite3*,int(*)(void*,int,const char*,const char*,
const char*,const char*),void*);
void (*set_auxdata)(sqlite3_context*,int,void*,void (*)(void*));
char * (*snprintf)(int,char*,const char*,...);
int (*step)(sqlite3_stmt*);
int (*table_column_metadata)(sqlite3*,const char*,const char*,const char*,char const**,char const**,int*,int*,int*);
int (*table_column_metadata)(sqlite3*,const char*,const char*,const char*,
char const**,char const**,int*,int*,int*);
void (*thread_cleanup)(void);
int (*total_changes)(sqlite3*);
void * (*trace)(sqlite3*,void(*xTrace)(void*,const char*),void*);
int (*transfer_bindings)(sqlite3_stmt*,sqlite3_stmt*);
void * (*update_hook)(sqlite3*,void(*)(void*,int ,char const*,char const*,sqlite_int64),void*);
void * (*update_hook)(sqlite3*,void(*)(void*,int ,char const*,char const*,
sqlite_int64),void*);
void * (*user_data)(sqlite3_context*);
const void * (*value_blob)(sqlite3_value*);
int (*value_bytes)(sqlite3_value*);
Expand All @@ -156,15 +167,19 @@ struct sqlite3_api_routines {
int (*prepare16_v2)(sqlite3*,const void*,int,sqlite3_stmt**,const void**);
int (*clear_bindings)(sqlite3_stmt*);
/* Added by 3.4.1 */
int (*create_module_v2)(sqlite3*,const char*,const sqlite3_module*,void*,void (*xDestroy)(void *));
int (*create_module_v2)(sqlite3*,const char*,const sqlite3_module*,void*,
void (*xDestroy)(void *));
/* Added by 3.5.0 */
int (*bind_zeroblob)(sqlite3_stmt*,int,int);
int (*blob_bytes)(sqlite3_blob*);
int (*blob_close)(sqlite3_blob*);
int (*blob_open)(sqlite3*,const char*,const char*,const char*,sqlite3_int64,int,sqlite3_blob**);
int (*blob_open)(sqlite3*,const char*,const char*,const char*,sqlite3_int64,
int,sqlite3_blob**);
int (*blob_read)(sqlite3_blob*,void*,int,int);
int (*blob_write)(sqlite3_blob*,const void*,int,int);
int (*create_collation_v2)(sqlite3*,const char*,int,void*,int(*)(void*,int,const void*,int,const void*),void(*)(void*));
int (*create_collation_v2)(sqlite3*,const char*,int,void*,
int(*)(void*,int,const void*,int,const void*),
void(*)(void*));
int (*file_control)(sqlite3*,const char*,int,void*);
sqlite3_int64 (*memory_highwater)(int);
sqlite3_int64 (*memory_used)(void);
Expand Down Expand Up @@ -193,6 +208,48 @@ struct sqlite3_api_routines {
sqlite3_stmt *(*next_stmt)(sqlite3*,sqlite3_stmt*);
const char *(*sql)(sqlite3_stmt*);
int (*status)(int,int*,int*,int);
int (*backup_finish)(sqlite3_backup*);
sqlite3_backup *(*backup_init)(sqlite3*,const char*,sqlite3*,const char*);
int (*backup_pagecount)(sqlite3_backup*);
int (*backup_remaining)(sqlite3_backup*);
int (*backup_step)(sqlite3_backup*,int);
const char *(*compileoption_get)(int);
int (*compileoption_used)(const char*);
int (*create_function_v2)(sqlite3*,const char*,int,int,void*,
void (*xFunc)(sqlite3_context*,int,sqlite3_value**),
void (*xStep)(sqlite3_context*,int,sqlite3_value**),
void (*xFinal)(sqlite3_context*),
void(*xDestroy)(void*));
int (*db_config)(sqlite3*,int,...);
sqlite3_mutex *(*db_mutex)(sqlite3*);
int (*db_status)(sqlite3*,int,int*,int*,int);
int (*extended_errcode)(sqlite3*);
void (*log)(int,const char*,...);
sqlite3_int64 (*soft_heap_limit64)(sqlite3_int64);
const char *(*sourceid)(void);
int (*stmt_status)(sqlite3_stmt*,int,int);
int (*strnicmp)(const char*,const char*,int);
int (*unlock_notify)(sqlite3*,void(*)(void**,int),void*);
int (*wal_autocheckpoint)(sqlite3*,int);
int (*wal_checkpoint)(sqlite3*,const char*);
void *(*wal_hook)(sqlite3*,int(*)(void*,sqlite3*,const char*,int),void*);
int (*blob_reopen)(sqlite3_blob*,sqlite3_int64);
int (*vtab_config)(sqlite3*,int op,...);
int (*vtab_on_conflict)(sqlite3*);
/* Version 3.7.16 and later */
int (*close_v2)(sqlite3*);
const char *(*db_filename)(sqlite3*,const char*);
int (*db_readonly)(sqlite3*,const char*);
int (*db_release_memory)(sqlite3*);
const char *(*errstr)(int);
int (*stmt_busy)(sqlite3_stmt*);
int (*stmt_readonly)(sqlite3_stmt*);
int (*stricmp)(const char*,const char*);
int (*uri_boolean)(const char*,const char*,int);
sqlite3_int64 (*uri_int64)(const char*,const char*,sqlite3_int64);
const char *(*uri_parameter)(const char*,const char*);
char *(*vsnprintf)(int,char*,const char*,va_list);
int (*wal_checkpoint_v2)(sqlite3*,const char*,int,int*,int*);
};

/*
Expand Down Expand Up @@ -372,9 +429,59 @@ struct sqlite3_api_routines {
#define sqlite3_next_stmt sqlite3_api->next_stmt
#define sqlite3_sql sqlite3_api->sql
#define sqlite3_status sqlite3_api->status
#define sqlite3_backup_finish sqlite3_api->backup_finish
#define sqlite3_backup_init sqlite3_api->backup_init
#define sqlite3_backup_pagecount sqlite3_api->backup_pagecount
#define sqlite3_backup_remaining sqlite3_api->backup_remaining
#define sqlite3_backup_step sqlite3_api->backup_step
#define sqlite3_compileoption_get sqlite3_api->compileoption_get
#define sqlite3_compileoption_used sqlite3_api->compileoption_used
#define sqlite3_create_function_v2 sqlite3_api->create_function_v2
#define sqlite3_db_config sqlite3_api->db_config
#define sqlite3_db_mutex sqlite3_api->db_mutex
#define sqlite3_db_status sqlite3_api->db_status
#define sqlite3_extended_errcode sqlite3_api->extended_errcode
#define sqlite3_log sqlite3_api->log
#define sqlite3_soft_heap_limit64 sqlite3_api->soft_heap_limit64
#define sqlite3_sourceid sqlite3_api->sourceid
#define sqlite3_stmt_status sqlite3_api->stmt_status
#define sqlite3_strnicmp sqlite3_api->strnicmp
#define sqlite3_unlock_notify sqlite3_api->unlock_notify
#define sqlite3_wal_autocheckpoint sqlite3_api->wal_autocheckpoint
#define sqlite3_wal_checkpoint sqlite3_api->wal_checkpoint
#define sqlite3_wal_hook sqlite3_api->wal_hook
#define sqlite3_blob_reopen sqlite3_api->blob_reopen
#define sqlite3_vtab_config sqlite3_api->vtab_config
#define sqlite3_vtab_on_conflict sqlite3_api->vtab_on_conflict
/* Version 3.7.16 and later */
#define sqlite3_close_v2 sqlite3_api->close_v2
#define sqlite3_db_filename sqlite3_api->db_filename
#define sqlite3_db_readonly sqlite3_api->db_readonly
#define sqlite3_db_release_memory sqlite3_api->db_release_memory
#define sqlite3_errstr sqlite3_api->errstr
#define sqlite3_stmt_busy sqlite3_api->stmt_busy
#define sqlite3_stmt_readonly sqlite3_api->stmt_readonly
#define sqlite3_stricmp sqlite3_api->stricmp
#define sqlite3_uri_boolean sqlite3_api->uri_boolean
#define sqlite3_uri_int64 sqlite3_api->uri_int64
#define sqlite3_uri_parameter sqlite3_api->uri_parameter
#define sqlite3_uri_vsnprintf sqlite3_api->vsnprintf
#define sqlite3_wal_checkpoint_v2 sqlite3_api->wal_checkpoint_v2
#endif /* SQLITE_CORE */

#define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api = 0;
#define SQLITE_EXTENSION_INIT2(v) sqlite3_api = v;
#ifndef SQLITE_CORE
/* This case when the file really is being compiled as a loadable
** extension */
# define SQLITE_EXTENSION_INIT1 const sqlite3_api_routines *sqlite3_api=0;
# define SQLITE_EXTENSION_INIT2(v) sqlite3_api=v;
# define SQLITE_EXTENSION_INIT3 \
extern const sqlite3_api_routines *sqlite3_api;
#else
/* This case when the file is being statically linked into the
** application */
# define SQLITE_EXTENSION_INIT1 /*no-op*/
# define SQLITE_EXTENSION_INIT2(v) (void)v; /* unused parameter */
# define SQLITE_EXTENSION_INIT3 /*no-op*/
#endif

#endif /* _SQLITE3EXT_H_ */
1 change: 1 addition & 0 deletions openstudiocore/src/openstudio_lib/OSDocument.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,7 @@ OSDocument::OSDocument( openstudio::model::Model library,

openstudio::analysisdriver::SimpleProjectOptions options;
options.setPauseRunManagerQueue(true); // do not start running when opening
options.setInitializeRunManagerUI(true);
options.setLogLevel(Debug);

// initialize project object
Expand Down
2 changes: 2 additions & 0 deletions openstudiocore/src/pat_app/PatApp.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -542,6 +542,7 @@ void PatApp::create()
openstudio::path projectDir = openstudio::toPath(fileName);
openstudio::analysisdriver::SimpleProjectOptions options;
options.setPauseRunManagerQueue(true); // do not start running when creating
options.setInitializeRunManagerUI(false);
options.setLogLevel(Debug);
boost::optional<openstudio::analysisdriver::SimpleProject> project = analysisdriver::createPATProject(projectDir, options);
if(project.is_initialized()){
Expand Down Expand Up @@ -1195,6 +1196,7 @@ bool PatApp::openFile(const QString& fileName)
openstudio::path projectDir = openstudio::toPath(dirAbsolutePath);
openstudio::analysisdriver::SimpleProjectOptions options;
options.setPauseRunManagerQueue(true); // do not start running when opening
options.setInitializeRunManagerUI(false);
options.setLogLevel(Debug);
boost::optional<openstudio::analysisdriver::SimpleProject> project = analysisdriver::openPATProject(projectDir, options);
if(project.is_initialized()){
Expand Down
8 changes: 4 additions & 4 deletions openstudiocore/src/runmanager/lib/RubyJob.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -45,11 +45,11 @@ namespace detail {
const JobState &t_restoreData)
: ToolBasedJob(t_uuid, JobType::Ruby, t_tools, "ruby", t_params, t_files, false, t_restoreData)
{
LOG(Debug, "Creating a RubyJob");
LOG(Trace, "Creating a RubyJob");

m_description = buildDescription("rb");

LOG(Debug, "RubyJob Created");
LOG(Trace, "RubyJob Created");
}

RubyJob::RubyJob(const UUID &t_uuid,
Expand All @@ -60,11 +60,11 @@ namespace detail {
const JobState &t_restoreData)
: ToolBasedJob(t_uuid, t_jobType, t_tools, "ruby", t_params, t_files, false, t_restoreData)
{
LOG(Debug, "Creating a RubyJob");
LOG(Trace, "Creating a RubyJob");

m_description = buildDescription("rb");

LOG(Debug, "RubyJob Created");
LOG(Trace, "RubyJob Created");
}


Expand Down
12 changes: 6 additions & 6 deletions openstudiocore/src/runmanager/lib/RunManager.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ namespace runmanager {
}
}

boost::shared_ptr<detail::RunManager_Impl> get_impl(const openstudio::path &DB, bool t_new, bool t_paused, bool t_initui, bool t_tempdb)
boost::shared_ptr<detail::RunManager_Impl> get_impl(const openstudio::path &DB, bool t_new, bool t_paused, bool t_initui, bool t_tempdb, bool t_useStatusGUI)
{
// use complete path
openstudio::path wDB = completeAndNormalize(DB);
Expand Down Expand Up @@ -128,7 +128,7 @@ namespace runmanager {

// We were unable to construct a valid shared_ptr from what we
// did or did not have, so make a new one
boost::shared_ptr<detail::RunManager_Impl> impl(new detail::RunManager_Impl(wDB, t_paused, t_initui, t_tempdb));
boost::shared_ptr<detail::RunManager_Impl> impl(new detail::RunManager_Impl(wDB, t_paused, t_initui, t_tempdb, t_useStatusGUI));
m_dbs.insert(std::make_pair(wDB, boost::weak_ptr<detail::RunManager_Impl>(impl)));
return impl;

Expand All @@ -151,13 +151,13 @@ namespace runmanager {
#endif
}

RunManager::RunManager(const openstudio::path &DB, bool t_new, bool t_paused, bool t_initui)
: m_impl(get_db_handler().get_impl(DB, t_new, t_paused, t_initui, false))
RunManager::RunManager(const openstudio::path &DB, bool t_new, bool t_paused, bool t_initui, bool t_useStatusGUI)
: m_impl(get_db_handler().get_impl(DB, t_new, t_paused, t_initui, false, t_useStatusGUI))
{
}

RunManager::RunManager(bool t_paused, bool t_initui)
: m_impl(get_db_handler().get_impl(generateTempPathName(), true, t_paused, t_initui, true))
RunManager::RunManager(bool t_paused, bool t_initui, bool t_useStatusGUI)
: m_impl(get_db_handler().get_impl(generateTempPathName(), true, t_paused, t_initui, true, t_useStatusGUI))
{
}

Expand Down
6 changes: 4 additions & 2 deletions openstudiocore/src/runmanager/lib/RunManager.hpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,15 +80,17 @@ namespace detail {
/// \param[in] t_paused If true the processor is started in paused mode.
/// \param[in] t_initializeui If true the application's UI subsystem is initialized. Used in a standalone
/// application that utilized RunManager
RunManager(bool t_paused = false, bool t_initializeui = true);
/// \param[in] t_useStatusGUI Enable the use of Job status GUI elements
RunManager(bool t_paused = false, bool t_initializeui = true, bool t_useStatusGUI = true);

/// Construct a RunManager from a given database file path
/// \param[in] DB The file location for the database storing prefs and job queue
/// \param[in] t_new If true, the database file is truncated and started from scratch
/// \param[in] t_paused If true the processor is started in paused mode.
/// \param[in] t_initializeui If true the application's UI subsystem is initialized. Used in a standalone
/// application that utilized RunManager
RunManager(const openstudio::path &DB, bool t_new = false, bool t_paused = false, bool t_initializeui=true);
/// \param[in] t_useStatusGUI Enable the use of Job status GUI elements
RunManager(const openstudio::path &DB, bool t_new = false, bool t_paused = false, bool t_initializeui=true, bool t_useStatusGUI = true);
~RunManager();

/// Return tue if the given job is out of date
Expand Down
Loading