[OE-core] [PATCH 1/3] Use in-memory database for files
Peter Seebach
peter.seebach at windriver.com
Sun Feb 17 02:23:52 UTC 2013
It turns out that file databases don't get very large, and that
sqlite3 can be quite fast with an in-memory database. It also turns
out that dumping the database to disk on exit (or during idle times)
is pretty cheap compared to constant updates.
So: We add "--enable-memory-db", which defaults to on if you have
sqlite 3.7 or later, and off for 3.6 (because 3.6 has horrible
performance with in-memory db on some hosts we tried).
---
ChangeLog.txt | 5 +++
Makefile.in | 5 ++-
configure | 32 +++++++++++++++
pseudo_db.c | 119 +++++++++++++++++++++++++++++++++++++++++++++++++++++--
pseudo_db.h | 1 +
pseudo_server.c | 2 +
6 files changed, 159 insertions(+), 5 deletions(-)
diff --git a/ChangeLog.txt b/ChangeLog.txt
index 8f52cc9..ed9e5fa 100644
--- a/ChangeLog.txt
+++ b/ChangeLog.txt
@@ -1,3 +1,8 @@
+2013-02-15:
+ * (seebs) Add support for in-memory DB. This, plus upcoming
+ fsync-related changes, are expected to be big enough to justify
+ calling this 1.5.
+
2013-02-13:
* (seebs) calling link while chrooted could in some cases result
in the root path not being prepended at all. One more try!
diff --git a/Makefile.in b/Makefile.in
index e0cd7a9..094deba 100644
--- a/Makefile.in
+++ b/Makefile.in
@@ -23,11 +23,12 @@ LIBDIR=@LIBDIR@
SUFFIX=@SUFFIX@
SQLITE=@SQLITE@
SQLITE_LIB=@SQLITE_LIB@
+SQLITE_MEMORY=@SQLITE_MEMORY@
BITS=@BITS@
ARCH_FLAGS=@ARCH_FLAGS@
MARK64=@MARK64@
RPATH=@RPATH@
-VERSION=1.4.5
+VERSION=1.5
LIB=@LIB@
BIN=bin
@@ -37,7 +38,7 @@ LOCALSTATEDIR=$(PREFIX)/$(LOCALSTATE)
CFLAGS_BASE=-pipe -std=gnu99 -Wall -W -Wextra
CFLAGS_CODE=-fPIC -D_LARGEFILE64_SOURCE -D_ATFILE_SOURCE $(ARCH_FLAGS)
-CFLAGS_DEFS=-DPSEUDO_PREFIX='"$(PREFIX)"' -DPSEUDO_SUFFIX='"$(SUFFIX)"' -DPSEUDO_BINDIR='"$(BIN)"' -DPSEUDO_LIBDIR='"$(LIB)"' -DPSEUDO_LOCALSTATEDIR='"$(LOCALSTATE)"' -DPSEUDO_VERSION='"$(VERSION)"'
+CFLAGS_DEFS=-DPSEUDO_PREFIX='"$(PREFIX)"' -DPSEUDO_SUFFIX='"$(SUFFIX)"' -DPSEUDO_BINDIR='"$(BIN)"' -DPSEUDO_LIBDIR='"$(LIB)"' -DPSEUDO_LOCALSTATEDIR='"$(LOCALSTATE)"' -DPSEUDO_VERSION='"$(VERSION)"' $(SQLITE_MEMORY)
CFLAGS_DEBUG=-O2 -g
CFLAGS_SQL=-L$(SQLITE)/$(SQLITE_LIB) -I$(SQLITE)/include $(RPATH)
CFLAGS_PSEUDO=$(CFLAGS_BASE) $(CFLAGS_CODE) $(CFLAGS_DEFS) \
diff --git a/configure b/configure
index 9cb7804..44bee74 100755
--- a/configure
+++ b/configure
@@ -25,6 +25,7 @@ opt_arch=x86
opt_bits=
opt_sqlite=/usr
opt_rpath=
+opt_memory=
compile_x86_32=-m32
compile_x86_64=-m64
@@ -35,6 +36,7 @@ usage()
echo >&2 " configure --prefix=..."
echo >&2 " [--libdir=...]"
echo >&2 " [--suffix=...]"
+ echo >&2 " [--enable-memory-db]"
echo >&2 " [--with-sqlite=...]"
echo >&2 " [--with-sqlite-lib=...]"
echo >&2 " [--enable-static-sqlite]"
@@ -69,6 +71,12 @@ do
sqlite_ldarg='$(SQLITE)/$(SQLITE_LIB)/libsqlite3.a'
use_maybe_rpath=false
;;
+ --enable-memory-db=no)
+ opt_memory=false
+ ;;
+ --enable-memory-db=yes | --enable-memory-db)
+ opt_memory=true
+ ;;
--with-sqlite=*)
opt_sqlite=${arg#--with-sqlite=}
# assign new value if unset
@@ -170,6 +178,29 @@ if [ "${SQLITE3_VERSION}" -lt "03006000" ]; then
exit 1
fi
+if [ -z "$opt_memory" ]; then
+ if [ "${SQLITE3_VERSION}" -lt "03007000" ]; then
+ echo "Disabling in-memory database by default (sqlite too old)."
+ opt_memory=false
+ else
+ echo "Enabling in-memory database by default."
+ opt_memory=true
+ fi
+fi
+
+if $opt_memory; then
+ if [ "${SQLITE3_VERSION}" -lt "03007000" ]; then
+ cat >&2 <<EOF
+WARNING: sqlite prior to 3.7 has been known to perform exceedingly poorly
+with the in-memory database option. You asked for it, you get it, but if
+you get horrible performance, try turning it off.
+EOF
+ fi
+ SQLITE_MEMORY="-DUSE_MEMORY_DB"
+else
+ SQLITE_MEMORY=""
+fi
+
sed -e '
s, at PREFIX@,'"$opt_prefix"',g
s, at LIBDIR@,'"$opt_libdir"',g
@@ -179,6 +210,7 @@ sed -e '
s, at ARCH_FLAGS@,'"$arch_flags"',g
s, at SQLITE_LDARG@,'"$sqlite_ldarg"',g
s, at SQLITE_LIB@,'"$opt_sqlite_lib"',g
+ s, at SQLITE_MEMORY@,'"$SQLITE_MEMORY"',g
s!@RPATH@!'"$opt_rpath"'!g
s, at MARK64@,'"$opt_mark64"',g
s, at ARCH@,'"$opt_arch"',g
diff --git a/pseudo_db.c b/pseudo_db.c
index 540a3c2..21f2f53 100644
--- a/pseudo_db.c
+++ b/pseudo_db.c
@@ -1,7 +1,7 @@
/*
* pseudo_db.c, sqlite3 interface
*
- * Copyright (c) 2008-2010 Wind River Systems, Inc.
+ * Copyright (c) 2008-2010,2013 Wind River Systems, Inc.
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the Lesser GNU General Public License version 2.1 as
@@ -50,6 +50,10 @@ struct pdb_file_list {
sqlite3_stmt *stmt;
};
+static int file_db_dirty = 0;
+#ifdef USE_MEMORY_DB
+static sqlite3 *real_file_db = 0;
+#endif
static sqlite3 *file_db = 0;
static sqlite3 *log_db = 0;
@@ -118,6 +122,10 @@ static struct sql_index {
static char *file_pragmas[] = {
"PRAGMA legacy_file_format = OFF;",
"PRAGMA journal_mode = OFF;",
+ /* the default page size produces painfully bad behavior
+ * for memory databases with some versions of sqlite.
+ */
+ "PRAGMA page_size = 8192;",
"PRAGMA locking_mode = EXCLUSIVE;",
/* Setting this to NORMAL makes pseudo noticably slower
* than fakeroot, but is perhaps more secure. However,
@@ -139,7 +147,6 @@ static char *log_pragmas[] = {
NULL
};
-
/* table migrations: */
/* If there is no migration table, we assume "version -1" -- the
* version shipped with wrlinux 3.0, which had no version
@@ -239,6 +246,17 @@ static struct database_info db_infos[] = {
file_migrations,
file_pragmas,
file_setups,
+#ifdef USE_MEMORY_DB
+ &real_file_db
+ },
+ {
+ ":memory:",
+ file_indexes,
+ file_tables,
+ file_migrations,
+ file_pragmas,
+ file_setups,
+#endif
&file_db
},
{ 0, 0, 0, 0, 0, 0, 0 }
@@ -263,6 +281,67 @@ dberr(sqlite3 *db, char *fmt, ...) {
}
}
+#ifdef USE_MEMORY_DB
+
+static void
+pdb_backup() {
+ sqlite3_backup *pBackup;
+ /* no point in doing this if we don't have a database to back up,
+ * or nothing's changed.
+ */
+ if (!file_db || !real_file_db || !file_db_dirty)
+ return;
+
+ pBackup = sqlite3_backup_init(real_file_db, "main", file_db, "main");
+ if (pBackup) {
+ int rc;
+ (void)sqlite3_backup_step(pBackup, -1);
+ rc = sqlite3_backup_finish(pBackup);
+ if (rc != SQLITE_OK) {
+ dberr(real_file_db, "error during flush to disk");
+ }
+ }
+ file_db_dirty = 0;
+}
+
+static void
+pdb_restore() {
+ sqlite3_backup *pBackup;
+ /* no point in doing this if we don't have a database to back up */
+ if (!file_db || !real_file_db)
+ return;
+
+ pBackup = sqlite3_backup_init(file_db, "main", real_file_db, "main");
+ if (pBackup) {
+ int rc;
+ (void)sqlite3_backup_step(pBackup, -1);
+ rc = sqlite3_backup_finish(pBackup);
+ if (rc != SQLITE_OK) {
+ dberr(file_db, "error during load from disk");
+ }
+ }
+ file_db_dirty = 0;
+}
+
+int
+pdb_maybe_backup(void) {
+ static int occasional = 0;
+ if (file_db && real_file_db) {
+ occasional = (occasional + 1) % 10;
+ if (occasional == 0) {
+ pdb_backup();
+ return 1;
+ }
+ }
+ return 0;
+}
+#else /* USE_MEMORY_DB */
+int
+pdb_maybe_backup(void) {
+ return 0;
+}
+#endif
+
/* those who enjoy children, sausages, and databases, should not watch
* them being made.
*/
@@ -448,6 +527,12 @@ make_tables(sqlite3 *db,
static void
cleanup_db(void) {
pseudo_debug(1, "server exiting\n");
+#ifdef USE_MEMORY_DB
+ if (real_file_db) {
+ pdb_backup();
+ sqlite3_close(real_file_db);
+ }
+#endif
if (file_db)
sqlite3_close(file_db);
if (log_db)
@@ -478,7 +563,12 @@ get_db(struct database_info *dbinfo) {
return 0;
dbfile = pseudo_localstatedir_path(dbinfo->pathname);
- rc = sqlite3_open(dbfile, &db);
+#ifdef USE_MEMORY_DB
+ if (!strcmp(dbinfo->pathname, ":memory:")) {
+ rc = sqlite3_open(dbinfo->pathname, &db);
+ } else
+#endif
+ rc = sqlite3_open(dbfile, &db);
free(dbfile);
if (rc) {
pseudo_diag("Failed: %s\n", sqlite3_errmsg(db));
@@ -528,6 +618,11 @@ static int
get_dbs(void) {
int err = 0;
int i;
+#ifdef USE_MEMORY_DB
+ int already_loaded = 0;
+ if (file_db)
+ already_loaded = 1;
+#endif
for (i = 0; db_infos[i].db; ++i) {
if (get_db(&db_infos[i])) {
pseudo_diag("Error getting '%s' database.\n",
@@ -535,6 +630,10 @@ get_dbs(void) {
err = 1;
}
}
+#ifdef USE_MEMORY_DB
+ if (!already_loaded && file_db)
+ pdb_restore();
+#endif
return err;
}
@@ -1082,6 +1181,7 @@ pdb_delete(pseudo_query_t *traits, unsigned long fields) {
/* no need to return it, so... */
if (stmt) {
+ file_db_dirty = 1;
int rc = sqlite3_step(stmt);
if (rc != SQLITE_DONE) {
dberr(log_db, "deletion failed");
@@ -1292,6 +1392,7 @@ pdb_link_file(pseudo_msg_t *msg) {
(msg->pathlen ? msg->path : "<nil> (as NAMELESS FILE)"),
(unsigned long long) msg->dev, (unsigned long long) msg->ino,
(int) msg->mode, msg->uid);
+ file_db_dirty = 1;
rc = sqlite3_step(insert);
if (rc != SQLITE_DONE) {
dberr(file_db, "insert may have failed (rc %d)", rc);
@@ -1324,6 +1425,7 @@ pdb_unlink_file_dev(pseudo_msg_t *msg) {
}
sqlite3_bind_int(sql_delete, 1, msg->dev);
sqlite3_bind_int(sql_delete, 2, msg->ino);
+ file_db_dirty = 1;
rc = sqlite3_step(sql_delete);
if (rc != SQLITE_DONE) {
dberr(file_db, "delete by inode may have failed");
@@ -1359,6 +1461,7 @@ pdb_update_file_path(pseudo_msg_t *msg) {
sqlite3_bind_text(update, 1, msg->path, -1, SQLITE_STATIC);
sqlite3_bind_int(update, 2, msg->dev);
sqlite3_bind_int(update, 3, msg->ino);
+ file_db_dirty = 1;
rc = sqlite3_step(update);
if (rc != SQLITE_DONE) {
dberr(file_db, "update path by inode may have failed");
@@ -1396,6 +1499,7 @@ pdb_may_unlink_file(pseudo_msg_t *msg, int deleting) {
pseudo_debug(1, "cannot mark a file for pending deletion without a path.");
return 1;
}
+ file_db_dirty = 1;
rc = sqlite3_step(mark_file);
if (rc != SQLITE_DONE) {
dberr(file_db, "mark for deletion may have failed");
@@ -1439,6 +1543,7 @@ pdb_cancel_unlink_file(pseudo_msg_t *msg) {
pseudo_debug(1, "cannot unmark a file for pending deletion without a path.");
return 1;
}
+ file_db_dirty = 1;
rc = sqlite3_step(mark_file);
if (rc != SQLITE_DONE) {
dberr(file_db, "unmark for deletion may have failed");
@@ -1475,6 +1580,7 @@ pdb_did_unlink_files(int deleting) {
return 0;
}
sqlite3_bind_int(delete_exact, 1, deleting);
+ file_db_dirty = 1;
rc = sqlite3_step(delete_exact);
if (rc != SQLITE_DONE) {
dberr(file_db, "cleanup of files marked for deletion may have failed");
@@ -1510,6 +1616,7 @@ pdb_did_unlink_file(char *path, int deleting) {
}
sqlite3_bind_text(delete_exact, 1, path, -1, SQLITE_STATIC);
sqlite3_bind_int(delete_exact, 2, deleting);
+ file_db_dirty = 1;
rc = sqlite3_step(delete_exact);
if (rc != SQLITE_DONE) {
dberr(file_db, "cleanup of file marked for deletion may have failed");
@@ -1548,6 +1655,7 @@ pdb_unlink_file(pseudo_msg_t *msg) {
pseudo_debug(1, "cannot unlink a file without a path.");
return 1;
}
+ file_db_dirty = 1;
rc = sqlite3_step(delete_exact);
if (rc != SQLITE_DONE) {
dberr(file_db, "delete exact by path may have failed");
@@ -1595,6 +1703,7 @@ pdb_unlink_contents(pseudo_msg_t *msg) {
pseudo_debug(1, "cannot unlink a file without a path.");
return 1;
}
+ file_db_dirty = 1;
rc = sqlite3_step(delete_sub);
if (rc != SQLITE_DONE) {
dberr(file_db, "delete sub by path may have failed");
@@ -1662,6 +1771,7 @@ pdb_rename_file(const char *oldpath, pseudo_msg_t *msg) {
rc = sqlite3_exec(file_db, "BEGIN;", NULL, NULL, NULL);
+ file_db_dirty = 1;
rc = sqlite3_step(update_exact);
if (rc != SQLITE_DONE) {
dberr(file_db, "update exact may have failed: rc %d", rc);
@@ -1712,6 +1822,7 @@ pdb_renumber_all(dev_t from, dev_t to) {
dberr(file_db, "error binding device numbers to update");
}
+ file_db_dirty = 1;
rc = sqlite3_step(update);
if (rc != SQLITE_DONE) {
dberr(file_db, "update may have failed: rc %d", rc);
@@ -1762,6 +1873,7 @@ pdb_update_inode(pseudo_msg_t *msg) {
dberr(file_db, "error binding %s to select", msg->path);
}
+ file_db_dirty = 1;
rc = sqlite3_step(update);
if (rc != SQLITE_DONE) {
dberr(file_db, "update may have failed: rc %d", rc);
@@ -1806,6 +1918,7 @@ pdb_update_file(pseudo_msg_t *msg) {
sqlite3_bind_int(update, 5, msg->dev);
sqlite3_bind_int(update, 6, msg->ino);
+ file_db_dirty = 1;
rc = sqlite3_step(update);
if (rc != SQLITE_DONE) {
dberr(file_db, "update may have failed: rc %d", rc);
diff --git a/pseudo_db.h b/pseudo_db.h
index fe2fb12..9e0382a 100644
--- a/pseudo_db.h
+++ b/pseudo_db.h
@@ -37,6 +37,7 @@ typedef struct {
char *program;
} log_entry;
+extern int pdb_maybe_backup(void);
extern int pdb_cancel_unlink_file(pseudo_msg_t *msg);
extern int pdb_did_unlink_file(char *path, int deleting);
extern int pdb_did_unlink_files(int deleting);
diff --git a/pseudo_server.c b/pseudo_server.c
index f241242..4af5265 100644
--- a/pseudo_server.c
+++ b/pseudo_server.c
@@ -416,6 +416,8 @@ pseudo_server_loop(void) {
*/
if (active_clients == 1) {
loop_timeout -= LOOP_DELAY;
+ /* maybe flush database to disk */
+ pdb_maybe_backup();
if (loop_timeout <= 0) {
pseudo_debug(1, "no more clients, got bored.\n");
die_peacefully = 1;
--
1.7.9.5
More information about the Openembedded-core
mailing list