X-Git-Url: http://rtime.felk.cvut.cz/gitweb/notmuch.git/blobdiff_plain/e0635bd003244ae8b88a885e2e5f23c9104ed164..58a4277d3b2ad88bc285f2dae998626b0aa73299:/lib/database.cc diff --git a/lib/database.cc b/lib/database.cc index 6a7ce299..92a92d9a 100644 --- a/lib/database.cc +++ b/lib/database.cc @@ -50,8 +50,8 @@ typedef struct { /* Here's the current schema for our database (for NOTMUCH_DATABASE_VERSION): * - * We currently have two different types of documents (mail and - * directory) and also some metadata. + * We currently have three different types of documents (mail, ghost, + * and directory) and also some metadata. * * Mail document * ------------- @@ -109,6 +109,15 @@ typedef struct { * * The data portion of a mail document is empty. * + * Ghost mail document [if NOTMUCH_FEATURE_GHOSTS] + * ----------------------------------------------- + * A ghost mail document is like a mail document, but where we don't + * have the message content. These are used to track thread reference + * information for messages we haven't received. + * + * A ghost mail document has type: ghost; id and thread fields that + * are identical to the mail document fields; and a MESSAGE_ID value. + * * Directory document * ------------------ * A directory document is used by a client of the notmuch library to @@ -172,6 +181,13 @@ typedef struct { * generated is 1 and the value will be * incremented for each thread ID. * + * Obsolete metadata + * ----------------- + * + * If ! NOTMUCH_FEATURE_GHOSTS, there are no ghost mail documents. + * Instead, the database has the following additional database + * metadata: + * * thread_id_* A pre-allocated thread ID for a particular * message. This is actually an arbitrarily large * family of metadata name. Any particular name is @@ -286,6 +302,8 @@ static const struct { "from/subject/message-ID in database", "w" }, { NOTMUCH_FEATURE_BOOL_FOLDER, "exact folder:/path: search", "rw" }, + { NOTMUCH_FEATURE_GHOSTS, + "mail documents for missing messages", "w"}, }; const char * @@ -316,6 +334,8 @@ notmuch_status_to_string (notmuch_status_t status) return "Unbalanced number of calls to notmuch_database_begin_atomic/end_atomic"; case NOTMUCH_STATUS_UNSUPPORTED_OPERATION: return "Unsupported operation"; + case NOTMUCH_STATUS_UPGRADE_REQUIRED: + return "Operation requires a database upgrade"; default: case NOTMUCH_STATUS_LAST_STATUS: return "Unknown error status value"; @@ -388,8 +408,8 @@ find_document_for_doc_id (notmuch_database_t *notmuch, unsigned doc_id) * * notmuch-sha1- */ -static char * -_message_id_compressed (void *ctx, const char *message_id) +char * +_notmuch_message_id_compressed (void *ctx, const char *message_id) { char *sha1, *compressed; @@ -413,7 +433,7 @@ notmuch_database_find_message (notmuch_database_t *notmuch, return NOTMUCH_STATUS_NULL_POINTER; if (strlen (message_id) > NOTMUCH_MESSAGE_ID_MAX) - message_id = _message_id_compressed (notmuch, message_id); + message_id = _notmuch_message_id_compressed (notmuch, message_id); try { status = _notmuch_database_find_unique_doc_id (notmuch, "id", @@ -901,28 +921,30 @@ notmuch_database_close (notmuch_database_t *notmuch) { notmuch_status_t status = NOTMUCH_STATUS_SUCCESS; - try { - if (notmuch->xapian_db != NULL && - notmuch->mode == NOTMUCH_DATABASE_MODE_READ_WRITE) - (static_cast (notmuch->xapian_db))->flush (); - } catch (const Xapian::Error &error) { - status = NOTMUCH_STATUS_XAPIAN_EXCEPTION; - if (! notmuch->exception_reported) { - fprintf (stderr, "Error: A Xapian exception occurred flushing database: %s\n", - error.get_msg().c_str()); - } - } - /* Many Xapian objects (and thus notmuch objects) hold references to * the database, so merely deleting the database may not suffice to * close it. Thus, we explicitly close it here. */ if (notmuch->xapian_db != NULL) { try { + /* If there's an outstanding transaction, it's unclear if + * closing the Xapian database commits everything up to + * that transaction, or may discard committed (but + * unflushed) transactions. To be certain, explicitly + * cancel any outstanding transaction before closing. */ + if (notmuch->mode == NOTMUCH_DATABASE_MODE_READ_WRITE && + notmuch->atomic_nesting) + (static_cast (notmuch->xapian_db)) + ->cancel_transaction (); + + /* Close the database. This implicitly flushes + * outstanding changes. */ notmuch->xapian_db->close(); } catch (const Xapian::Error &error) { - /* don't clobber previous error status */ - if (status == NOTMUCH_STATUS_SUCCESS) - status = NOTMUCH_STATUS_XAPIAN_EXCEPTION; + status = NOTMUCH_STATUS_XAPIAN_EXCEPTION; + if (! notmuch->exception_reported) { + fprintf (stderr, "Error: A Xapian exception occurred closing database: %s\n", + error.get_msg().c_str()); + } } } @@ -1220,7 +1242,7 @@ notmuch_database_upgrade (notmuch_database_t *notmuch, target_features = notmuch->features | NOTMUCH_FEATURES_CURRENT; new_features = NOTMUCH_FEATURES_CURRENT & ~notmuch->features; - if (! new_features) + if (! notmuch_database_needs_upgrade (notmuch)) return NOTMUCH_STATUS_SUCCESS; if (progress_notify) { @@ -1241,6 +1263,19 @@ notmuch_database_upgrade (notmuch_database_t *notmuch, timer_is_active = TRUE; } + /* Figure out how much total work we need to do. */ + if (new_features & + (NOTMUCH_FEATURE_FILE_TERMS | NOTMUCH_FEATURE_BOOL_FOLDER)) { + notmuch_query_t *query = notmuch_query_create (notmuch, ""); + total += notmuch_query_count_messages (query); + notmuch_query_destroy (query); + } + if (new_features & NOTMUCH_FEATURE_DIRECTORY_DOCS) { + t_end = db->allterms_end ("XTIMESTAMP"); + for (t = db->allterms_begin ("XTIMESTAMP"); t != t_end; t++) + ++total; + } + /* Perform the upgrade in a transaction. */ db->begin_transaction (true); @@ -1256,8 +1291,6 @@ notmuch_database_upgrade (notmuch_database_t *notmuch, notmuch_message_t *message; char *filename; - total = notmuch_query_count_messages (query); - for (messages = notmuch_query_search_messages (query); notmuch_messages_valid (messages); notmuch_messages_move_to_next (messages)) @@ -1340,6 +1373,8 @@ notmuch_database_upgrade (notmuch_database_t *notmuch, db->delete_document (*p); } + + ++count; } } @@ -1711,12 +1746,18 @@ static char * _get_metadata_thread_id_key (void *ctx, const char *message_id) { if (strlen (message_id) > NOTMUCH_MESSAGE_ID_MAX) - message_id = _message_id_compressed (ctx, message_id); + message_id = _notmuch_message_id_compressed (ctx, message_id); return talloc_asprintf (ctx, NOTMUCH_METADATA_THREAD_ID_PREFIX "%s", message_id); } +static notmuch_status_t +_resolve_message_id_to_thread_id_old (notmuch_database_t *notmuch, + void *ctx, + const char *message_id, + const char **thread_id_ret); + /* Find the thread ID to which the message with 'message_id' belongs. * * Note: 'thread_id_ret' must not be NULL! @@ -1725,15 +1766,58 @@ _get_metadata_thread_id_key (void *ctx, const char *message_id) * * Note: If there is no message in the database with the given * 'message_id' then a new thread_id will be allocated for this - * message and stored in the database metadata, (where this same + * message ID and stored in the database metadata so that the * thread ID can be looked up if the message is added to the database - * later). + * later. */ static notmuch_status_t _resolve_message_id_to_thread_id (notmuch_database_t *notmuch, void *ctx, const char *message_id, const char **thread_id_ret) +{ + notmuch_private_status_t status; + notmuch_message_t *message; + + if (! (notmuch->features & NOTMUCH_FEATURE_GHOSTS)) + return _resolve_message_id_to_thread_id_old (notmuch, ctx, message_id, + thread_id_ret); + + /* Look for this message (regular or ghost) */ + message = _notmuch_message_create_for_message_id ( + notmuch, message_id, &status); + if (status == NOTMUCH_PRIVATE_STATUS_SUCCESS) { + /* Message exists */ + *thread_id_ret = talloc_steal ( + ctx, notmuch_message_get_thread_id (message)); + } else if (status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) { + /* Message did not exist. Give it a fresh thread ID and + * populate this message as a ghost message. */ + *thread_id_ret = talloc_strdup ( + ctx, _notmuch_database_generate_thread_id (notmuch)); + if (! *thread_id_ret) { + status = NOTMUCH_PRIVATE_STATUS_OUT_OF_MEMORY; + } else { + status = _notmuch_message_initialize_ghost (message, *thread_id_ret); + if (status == 0) + /* Commit the new ghost message */ + _notmuch_message_sync (message); + } + } else { + /* Create failed. Fall through. */ + } + + notmuch_message_destroy (message); + + return COERCE_STATUS (status, "Error creating ghost message"); +} + +/* Pre-ghost messages _resolve_message_id_to_thread_id */ +static notmuch_status_t +_resolve_message_id_to_thread_id_old (notmuch_database_t *notmuch, + void *ctx, + const char *message_id, + const char **thread_id_ret) { notmuch_status_t status; notmuch_message_t *message; @@ -1941,13 +2025,47 @@ _notmuch_database_link_message_to_children (notmuch_database_t *notmuch, return ret; } -/* Given a (mostly empty) 'message' and its corresponding +/* Fetch and clear the stored thread_id for message, or NULL if none. */ +static char * +_consume_metadata_thread_id (void *ctx, notmuch_database_t *notmuch, + notmuch_message_t *message) +{ + const char *message_id; + string stored_id; + char *metadata_key; + + message_id = notmuch_message_get_message_id (message); + metadata_key = _get_metadata_thread_id_key (ctx, message_id); + + /* Check if we have already seen related messages to this one. + * If we have then use the thread_id that we stored at that time. + */ + stored_id = notmuch->xapian_db->get_metadata (metadata_key); + if (stored_id.empty ()) { + return NULL; + } else { + Xapian::WritableDatabase *db; + + db = static_cast (notmuch->xapian_db); + + /* Clear the metadata for this message ID. We don't need it + * anymore. */ + db->set_metadata (metadata_key, ""); + + return talloc_strdup (ctx, stored_id.c_str ()); + } +} + +/* Given a blank or ghost 'message' and its corresponding * 'message_file' link it to existing threads in the database. * - * The first check is in the metadata of the database to see if we - * have pre-allocated a thread_id in advance for this message, (which - * would have happened if a message was previously added that - * referenced this one). + * First, if is_ghost, this retrieves the thread ID already stored in + * the message (which will be the case if a message was previously + * added that referenced this one). If the message is blank + * (!is_ghost), it doesn't have a thread ID yet (we'll generate one + * later in this function). If the database does not support ghost + * messages, this checks for a thread ID stored in database metadata + * for this message ID. * * Second, we look at 'message_file' and its link-relevant headers * (References and In-Reply-To) for message IDs. @@ -1955,7 +2073,7 @@ _notmuch_database_link_message_to_children (notmuch_database_t *notmuch, * Finally, we look in the database for existing message that * reference 'message'. * - * In all cases, we assign to the current message the first thread_id + * In all cases, we assign to the current message the first thread ID * found (through either parent or child). We will also merge any * existing, distinct threads where this message belongs to both, * (which is not uncommon when messages are processed out of order). @@ -1969,44 +2087,33 @@ _notmuch_database_link_message_to_children (notmuch_database_t *notmuch, static notmuch_status_t _notmuch_database_link_message (notmuch_database_t *notmuch, notmuch_message_t *message, - notmuch_message_file_t *message_file) + notmuch_message_file_t *message_file, + notmuch_bool_t is_ghost) { + void *local = talloc_new (NULL); notmuch_status_t status; - const char *message_id, *thread_id = NULL; - char *metadata_key; - string stored_id; - - message_id = notmuch_message_get_message_id (message); - metadata_key = _get_metadata_thread_id_key (message, message_id); + const char *thread_id = NULL; - /* Check if we have already seen related messages to this one. - * If we have then use the thread_id that we stored at that time. - */ - stored_id = notmuch->xapian_db->get_metadata (metadata_key); - if (! stored_id.empty()) { - Xapian::WritableDatabase *db; - - db = static_cast (notmuch->xapian_db); - - /* Clear the metadata for this message ID. We don't need it - * anymore. */ - db->set_metadata (metadata_key, ""); - thread_id = stored_id.c_str(); - - _notmuch_message_add_term (message, "thread", thread_id); + /* Check if the message already had a thread ID */ + if (notmuch->features & NOTMUCH_FEATURE_GHOSTS) { + if (is_ghost) + thread_id = notmuch_message_get_thread_id (message); + } else { + thread_id = _consume_metadata_thread_id (local, notmuch, message); + if (thread_id) + _notmuch_message_add_term (message, "thread", thread_id); } - talloc_free (metadata_key); status = _notmuch_database_link_message_to_parents (notmuch, message, message_file, &thread_id); if (status) - return status; + goto DONE; status = _notmuch_database_link_message_to_children (notmuch, message, &thread_id); if (status) - return status; + goto DONE; /* If not part of any existing thread, generate a new thread ID. */ if (thread_id == NULL) { @@ -2015,7 +2122,10 @@ _notmuch_database_link_message (notmuch_database_t *notmuch, _notmuch_message_add_term (message, "thread", thread_id); } - return NOTMUCH_STATUS_SUCCESS; + DONE: + talloc_free (local); + + return status; } notmuch_status_t @@ -2027,6 +2137,7 @@ notmuch_database_add_message (notmuch_database_t *notmuch, notmuch_message_t *message = NULL; notmuch_status_t ret = NOTMUCH_STATUS_SUCCESS, ret2; notmuch_private_status_t private_status; + notmuch_bool_t is_ghost = false; const char *date, *header; const char *from, *to, *subject; @@ -2083,14 +2194,6 @@ notmuch_database_add_message (notmuch_database_t *notmuch, * better than no message-id at all. */ if (message_id == NULL) message_id = talloc_strdup (message_file, header); - - /* If a message ID is too long, substitute its sha1 instead. */ - if (message_id && strlen (message_id) > NOTMUCH_MESSAGE_ID_MAX) { - char *compressed = _message_id_compressed (message_file, - message_id); - talloc_free (message_id); - message_id = compressed; - } } if (message_id == NULL ) { @@ -2127,12 +2230,20 @@ notmuch_database_add_message (notmuch_database_t *notmuch, _notmuch_message_add_filename (message, filename); - /* Is this a newly created message object? */ - if (private_status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND) { + /* Is this a newly created message object or a ghost + * message? We have to be slightly careful: if this is a + * blank message, it's not safe to call + * notmuch_message_get_flag yet. */ + if (private_status == NOTMUCH_PRIVATE_STATUS_NO_DOCUMENT_FOUND || + (is_ghost = notmuch_message_get_flag ( + message, NOTMUCH_MESSAGE_FLAG_GHOST))) { _notmuch_message_add_term (message, "type", "mail"); + if (is_ghost) + /* Convert ghost message to a regular message */ + _notmuch_message_remove_term (message, "type", "ghost"); ret = _notmuch_database_link_message (notmuch, message, - message_file); + message_file, is_ghost); if (ret) goto DONE; @@ -2213,6 +2324,9 @@ notmuch_database_find_message_by_filename (notmuch_database_t *notmuch, if (message_ret == NULL) return NOTMUCH_STATUS_NULL_POINTER; + if (! (notmuch->features & NOTMUCH_FEATURE_FILE_TERMS)) + return NOTMUCH_STATUS_UPGRADE_REQUIRED; + /* return NULL on any failure */ *message_ret = NULL;