From c6fe73f2cf5d4e71962f24bb32a4c9fdd8e4a889 Mon Sep 17 00:00:00 2001 From: Siegfried-Angel Gevatter Pujals Date: Mon, 30 Jul 2012 17:13:23 +0200 Subject: [PATCH] insert_events: group data insertion from separate events https://bugs.freedesktop.org/show_bug.cgi?id=52971 --- src/engine.vala | 95 ++++++++++++++++++++++++++++------------ src/errors.vala | 1 + src/sql.vala | 4 +- tools/scalability_benchmark.py | 6 +-- 4 files changed, 75 insertions(+), 31 deletions(-) diff --git a/src/engine.vala b/src/engine.vala index 77e711c..a902c4e 100644 --- a/src/engine.vala +++ b/src/engine.vala @@ -76,6 +76,7 @@ public class Engine : DbReader database.begin_transaction (); try { + insert_event_data (events); for (int i = 0; i < events.length; ++i) { if (events[i] != null) @@ -141,31 +142,75 @@ public class Engine : DbReader } } - private uint32 insert_event (Event event) throws EngineError - requires (event.id == 0) - requires (event.num_subjects () > 0) + private class DataInserter { - event.id = ++last_id; + // This is the maximum number of "unions" SQLite supports + private static const int MAX_PARAMETERS = 500; + + private Database database; + private string type; + private GenericArray data; + + public DataInserter (Database db, string data_type) + { + database = db; + type = data_type; + data = new GenericArray (); + } + + public ~DataInserter () + { + if (data.length > 0) + warning ("DataInserter: destroyed with unflushed data"); + } + + public void add (string val) throws EngineError + { + if (data.length == MAX_PARAMETERS) + flush (); + data.add (val); + } + + public void flush () throws EngineError + { + if (data.length > 0) + { + database.insert_or_ignore_into_table (type, data); + data = new GenericArray (); + } + } + } - // Make sure all the URIs, texts and storage are inserted + /** + * Makes sure all the URIs, texts and storage values used + * by the given events are in the database. + */ + private void insert_event_data (GenericArray events) + throws EngineError + { + var uris = new DataInserter (database, "uri"); + var texts = new DataInserter (database, "text"); + var storages = new DataInserter (database, "storage"); + + for (int j = 0; j < events.length; ++j) { - var uris = new GenericArray (); - var texts = new GenericArray (); - var storages = new GenericArray (); + if (events[j] == null) continue; + + Event event = events[j]; var subj_uris = new SList (); if (!is_empty_string (event.origin)) uris.add (event.origin); // Iterate through subjects and check for validity - for (int i = 0; i < event.num_subjects(); ++i) + for (int i = 0; i < event.num_subjects (); ++i) { unowned Subject subject = event.subjects[i]; - if (subj_uris.find_custom(subject.uri, strcmp) != null) + if (subj_uris.find_custom (subject.uri, strcmp) != null) { // Events with two subjects with the same URI are not supported. - warning ("Events with two subjects with the same URI are not supported"); - return 0; + throw new EngineError.INVALID_EVENT ( + "Events with two subjects with the same URI are not supported"); } subj_uris.append (subject.uri); @@ -180,23 +225,19 @@ public class Engine : DbReader if (!is_empty_string(subject.storage)) storages.add (subject.storage); } - - try - { - if (uris.length > 0) - database.insert_or_ignore_into_table ("uri", uris); - if (texts.length > 0) - database.insert_or_ignore_into_table ("text", texts); - if (storages.length > 0) - database.insert_or_ignore_into_table ("storage", storages); - } - catch (EngineError e) - { - warning ("Can't insert data for event: " + e.message); - return 0; - } } + uris.flush (); + texts.flush (); + storages.flush (); + } + + private uint32 insert_event (Event event) throws EngineError + requires (event.id == 0) + requires (event.num_subjects () > 0) + { + event.id = ++last_id; + var payload_id = store_payload (event); // FIXME: Should we add something just like TableLookup but with LRU diff --git a/src/errors.vala b/src/errors.vala index b0c23a7..4651508 100644 --- a/src/errors.vala +++ b/src/errors.vala @@ -31,6 +31,7 @@ namespace Zeitgeist DATABASE_RETIRE_FAILED, EXISTING_INSTANCE, INVALID_ARGUMENT, + INVALID_EVENT, INVALID_KEY, INVALID_SIGNATURE, // FIXME: change from EngineError to sth. + public TOO_MANY_RESULTS, diff --git a/src/sql.vala b/src/sql.vala index eab9289..dd48ed7 100644 --- a/src/sql.vala +++ b/src/sql.vala @@ -236,9 +236,11 @@ namespace Zeitgeist.SQLite public void insert_or_ignore_into_table (string table_name, GenericArray values) throws EngineError { + if (values.length == 0) + return; + int rc; - assert (values.length > 0); var sql = new StringBuilder (); sql.append ("INSERT OR IGNORE INTO "); sql.append (table_name); diff --git a/tools/scalability_benchmark.py b/tools/scalability_benchmark.py index 642155a..b7b2f03 100755 --- a/tools/scalability_benchmark.py +++ b/tools/scalability_benchmark.py @@ -96,8 +96,8 @@ class ScalabilityBenchmark: return (insertion_times,) + query_times -def main(events_per_iteration=5000): - data = ScalabilityBenchmark().run(50, events_per_iteration, 5) +def main(iterations=50, events_per_iteration=5000): + data = ScalabilityBenchmark().run(iterations, events_per_iteration, 5) chart = google_chart_api.LineChart() chart.AddLine(data[0], label="Insertion time") chart.AddLine(data[1], label="Query time (50 most recent events)") @@ -111,6 +111,6 @@ def main(events_per_iteration=5000): if __name__ == '__main__': try: - main() + main(10, 5000) except KeyboardInterrupt: pass -- 1.7.9.5