CLI: add optional config data to dump output.

Note that it changes the default dump output format, but doesn't break
existing notmuch-restore. It might break user scripts though.

diff --git a/doc/man1/notmuch-dump.rst b/doc/man1/notmuch-dump.rst
index a37c337c72e6a9e3189704006ca332546d3772b1..eda9e07f6cdd00a725fa88521912ed7845b24acc 100644 (file)
--- a/doc/man1/notmuch-dump.rst
+++ b/doc/man1/notmuch-dump.rst
@@ -71,6 +71,23 @@ Supported options for **dump** include
             characters. Note also that tags with spaces will not be
             correctly restored with this format.
 
             characters. Note also that tags with spaces will not be
             correctly restored with this format.
 

+    ``--include=(config|tags)``
+
+    Control what kind of metadata is included in the output.
+
+      **config**
+
+       Output configuration data stored in the database. Each line
+       starts with "#@ ", followed by a space seperated key-value
+       pair.  Both key and value are hex encoded if needed.
+
+      **tags**
+
+       Output per-message metadata, namely tags. See *format* above
+       for description of the output.
+
+      The default is to include both tags and configuration information
+

     ``--output=``\ <filename>
         Write output to given file instead of stdout.
 
     ``--output=``\ <filename>
         Write output to given file instead of stdout.
 

diff --git a/notmuch-client.h b/notmuch-client.h
index b3d0b668bc4d142726881c6eb8e471f181de3583..c130b937790abe9b6e6957b1f70e5c64b31ec60b 100644 (file)
--- a/notmuch-client.h
+++ b/notmuch-client.h
@@ -446,11 +446,19 @@ typedef enum dump_formats {
     DUMP_FORMAT_SUP
 } dump_format_t;
 
     DUMP_FORMAT_SUP
 } dump_format_t;
 

+typedef enum dump_includes {
+    DUMP_INCLUDE_TAGS = 1,
+    DUMP_INCLUDE_CONFIG = 2,
+} dump_include_t;
+
+#define NOTMUCH_DUMP_VERSION 2
+

 int
 notmuch_database_dump (notmuch_database_t *notmuch,
                       const char *output_file_name,
                       const char *query_str,
                       dump_format_t output_format,
 int
 notmuch_database_dump (notmuch_database_t *notmuch,
                       const char *output_file_name,
                       const char *query_str,
                       dump_format_t output_format,

+                      dump_include_t include,

                       notmuch_bool_t gzip_output);
 
 /* If status is non-zero (i.e. error) print appropriate
                       notmuch_bool_t gzip_output);
 
 /* If status is non-zero (i.e. error) print appropriate

diff --git a/notmuch-dump.c b/notmuch-dump.c
index 829781f8b44feba2f76fb4e39427c5f1029b7fd1..8e79511574a8e984c337c0516ddd15ea86541231 100644 (file)
--- a/notmuch-dump.c
+++ b/notmuch-dump.c
@@ -23,16 +23,80 @@
 #include "string-util.h"
 #include <zlib.h>
 
 #include "string-util.h"
 #include <zlib.h>
 

+static int
+database_dump_config (notmuch_database_t *notmuch, gzFile output)
+{
+    notmuch_config_list_t *list;
+    int ret = EXIT_FAILURE;
+    char *buffer = NULL;
+    size_t buffer_size = 0;
+
+    if (print_status_database ("notmuch dump", notmuch,
+                              notmuch_database_get_config_list (notmuch, NULL, &list)))
+       goto DONE;
+
+    for (; notmuch_config_list_valid (list); notmuch_config_list_move_to_next (list)) {
+       if (hex_encode (notmuch, notmuch_config_list_key (list),
+                       &buffer, &buffer_size) != HEX_SUCCESS) {
+           fprintf (stderr, "Error: failed to hex-encode config key %s\n",
+                    notmuch_config_list_key (list));
+           goto DONE;
+       }
+       gzprintf (output, "#@ %s", buffer);
+
+       if (hex_encode (notmuch, notmuch_config_list_value (list),
+                       &buffer, &buffer_size) != HEX_SUCCESS) {
+           fprintf (stderr, "Error: failed to hex-encode config value %s\n",
+                    notmuch_config_list_value (list) );
+           goto DONE;
+       }
+
+       gzprintf (output, " %s\n", buffer);
+    }
+
+    ret = EXIT_SUCCESS;
+
+ DONE:
+    if (list)
+       notmuch_config_list_destroy (list);
+
+    if (buffer)
+       talloc_free (buffer);
+
+    return ret;
+}
+
+static void
+print_dump_header (gzFile output, int output_format, int include)
+{
+    gzprintf (output, "#notmuch-dump %s:%d %s%s%s\n",
+             (output_format == DUMP_FORMAT_SUP) ? "sup" : "batch-tag",
+             NOTMUCH_DUMP_VERSION,
+             (include & DUMP_INCLUDE_CONFIG) ? "config" : "",
+             (include & DUMP_INCLUDE_TAGS) && (include & DUMP_INCLUDE_CONFIG) ? "," : "",
+             (include & DUMP_INCLUDE_TAGS) ? "tags" : "");
+}

 
 static int
 database_dump_file (notmuch_database_t *notmuch, gzFile output,
 
 static int
 database_dump_file (notmuch_database_t *notmuch, gzFile output,

-                   const char *query_str, int output_format)
+                   const char *query_str, int output_format, int include)

 {
     notmuch_query_t *query;
     notmuch_messages_t *messages;
     notmuch_message_t *message;
     notmuch_tags_t *tags;
 
 {
     notmuch_query_t *query;
     notmuch_messages_t *messages;
     notmuch_message_t *message;
     notmuch_tags_t *tags;
 

+    print_dump_header (output, output_format, include);
+
+    if (include & DUMP_INCLUDE_CONFIG) {
+       if (print_status_database ("notmuch dump", notmuch,
+                                  database_dump_config(notmuch,output)))
+           return EXIT_FAILURE;
+    }
+
+    if (! (include & DUMP_INCLUDE_TAGS))
+       return EXIT_SUCCESS;
+

     if (! query_str)
        query_str = "";
 
     if (! query_str)
        query_str = "";
 


@@ -130,6 +194,7 @@ notmuch_database_dump (notmuch_database_t *notmuch,
                       const char *output_file_name,
                       const char *query_str,
                       dump_format_t output_format,
                       const char *output_file_name,
                       const char *query_str,
                       dump_format_t output_format,

+                      dump_include_t include,

                       notmuch_bool_t gzip_output)
 {
     gzFile output = NULL;
                       notmuch_bool_t gzip_output)
 {
     gzFile output = NULL;


@@ -164,7 +229,7 @@ notmuch_database_dump (notmuch_database_t *notmuch,
        goto DONE;
     }
 
        goto DONE;
     }
 

-    ret = database_dump_file (notmuch, output, query_str, output_format);
+    ret = database_dump_file (notmuch, output, query_str, output_format, include);

     if (ret) goto DONE;
 
     ret = gzflush (output, Z_FINISH);
     if (ret) goto DONE;
 
     ret = gzflush (output, Z_FINISH);


@@ -226,6 +291,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[])
     int opt_index;
 
     int output_format = DUMP_FORMAT_BATCH_TAG;
     int opt_index;
 
     int output_format = DUMP_FORMAT_BATCH_TAG;

+    int include = 0;

     notmuch_bool_t gzip_output = 0;
 
     notmuch_opt_desc_t options[] = {
     notmuch_bool_t gzip_output = 0;
 
     notmuch_opt_desc_t options[] = {


@@ -233,6 +299,9 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[])
          (notmuch_keyword_t []){ { "sup", DUMP_FORMAT_SUP },
                                  { "batch-tag", DUMP_FORMAT_BATCH_TAG },
                                  { 0, 0 } } },
          (notmuch_keyword_t []){ { "sup", DUMP_FORMAT_SUP },
                                  { "batch-tag", DUMP_FORMAT_BATCH_TAG },
                                  { 0, 0 } } },

+       { NOTMUCH_OPT_KEYWORD_FLAGS, &include, "include", 'I',
+         (notmuch_keyword_t []){ { "config", DUMP_INCLUDE_CONFIG },
+                                 { "tags", DUMP_INCLUDE_TAGS} } },

        { NOTMUCH_OPT_STRING, &output_file_name, "output", 'o', 0  },
        { NOTMUCH_OPT_BOOLEAN, &gzip_output, "gzip", 'z', 0 },
        { NOTMUCH_OPT_INHERIT, (void *) &notmuch_shared_options, NULL, 0, 0 },
        { NOTMUCH_OPT_STRING, &output_file_name, "output", 'o', 0  },
        { NOTMUCH_OPT_BOOLEAN, &gzip_output, "gzip", 'z', 0 },
        { NOTMUCH_OPT_INHERIT, (void *) &notmuch_shared_options, NULL, 0, 0 },


@@ -245,6 +314,9 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[])
 
     notmuch_process_shared_options (argv[0]);
 
 
     notmuch_process_shared_options (argv[0]);
 

+    if (include == 0)
+       include = DUMP_INCLUDE_CONFIG | DUMP_INCLUDE_TAGS;
+

     if (opt_index < argc) {
        query_str = query_string_from_args (notmuch, argc - opt_index, argv + opt_index);
        if (query_str == NULL) {
     if (opt_index < argc) {
        query_str = query_string_from_args (notmuch, argc - opt_index, argv + opt_index);
        if (query_str == NULL) {


@@ -254,7 +326,7 @@ notmuch_dump_command (notmuch_config_t *config, int argc, char *argv[])
     }
 
     ret = notmuch_database_dump (notmuch, output_file_name, query_str,
     }
 
     ret = notmuch_database_dump (notmuch, output_file_name, query_str,

-                                output_format, gzip_output);
+                                output_format, include, gzip_output);

 
     notmuch_database_destroy (notmuch);
 
 
     notmuch_database_destroy (notmuch);
 

diff --git a/notmuch-new.c b/notmuch-new.c
index 2d975eb5b640f4561864372f64612f0e7d4fe94b..8d1545e1d97337b777f1b34d94ca923b7a3d5bec 100644 (file)
--- a/notmuch-new.c
+++ b/notmuch-new.c
@@ -1042,7 +1042,7 @@ notmuch_new_command (notmuch_config_t *config, int argc, char *argv[])
            }
 
            if (notmuch_database_dump (notmuch, backup_name, "",
            }
 
            if (notmuch_database_dump (notmuch, backup_name, "",

-                                      DUMP_FORMAT_BATCH_TAG, TRUE)) {
+                                      DUMP_FORMAT_BATCH_TAG, DUMP_INCLUDE_CONFIG | DUMP_INCLUDE_TAGS, TRUE)) {

                fprintf (stderr, "Backup failed. Aborting upgrade.");
                return EXIT_FAILURE;
            }
                fprintf (stderr, "Backup failed. Aborting upgrade.");
                return EXIT_FAILURE;
            }

diff --git a/test/T150-tagging.sh b/test/T150-tagging.sh
index a451ffaeb39e3db7529332564b214c8d00f90b8a..61d1311608a755c5815bf545ff46170036339fa5 100755 (executable)
--- a/test/T150-tagging.sh
+++ b/test/T150-tagging.sh
@@ -188,7 +188,7 @@ cat <<EOF > EXPECTED
 +%22%27%22%27%22%22%27%27 +inbox +tag4 +tag5 +unread -- id:msg-002@notmuch-test-suite
 EOF
 
 +%22%27%22%27%22%22%27%27 +inbox +tag4 +tag5 +unread -- id:msg-002@notmuch-test-suite
 EOF
 

-notmuch dump --format=batch-tag | sort > OUTPUT
+NOTMUCH_DUMP_TAGS > OUTPUT

 notmuch restore --format=batch-tag < BACKUP
 test_expect_equal_file EXPECTED OUTPUT
 
 notmuch restore --format=batch-tag < BACKUP
 test_expect_equal_file EXPECTED OUTPUT
 


@@ -209,7 +209,7 @@ cat <<EOF > EXPECTED
 +%21@%23%20%24%25%5e%26%2a%29-_=+%5b%7b%5c%20%7c%3b%3a%27%20%22,.%3c%60%7e +inbox +tag5 +unread -- id:msg-001@notmuch-test-suite
 EOF
 
 +%21@%23%20%24%25%5e%26%2a%29-_=+%5b%7b%5c%20%7c%3b%3a%27%20%22,.%3c%60%7e +inbox +tag5 +unread -- id:msg-001@notmuch-test-suite
 EOF
 

-notmuch dump --format=batch-tag | sort > OUTPUT
+NOTMUCH_DUMP_TAGS > OUTPUT

 notmuch restore --format=batch-tag < BACKUP
 test_expect_equal_file EXPECTED OUTPUT
 
 notmuch restore --format=batch-tag < BACKUP
 test_expect_equal_file EXPECTED OUTPUT
 


@@ -235,7 +235,7 @@ cat <<EOF > EXPECTED
 +%2a@%7d%cf%b5%f4%85%80%adO3%da%a7 +=%e0%ac%95%c8%b3+%ef%aa%95%c8%a64w%c7%9d%c9%a2%cf%b3%d6%82%24B%c4%a9%c5%a1UX%ee%99%b0%27E7%ca%a4%d0%8b%5d +A%e1%a0%bc%de%8b%d5%b2V%d9%9b%f3%b5%a2%a3M%d8%a1u@%f0%a0%ac%948%7e%f0%ab%86%af%27 +L%df%85%ef%a1%a5m@%d3%96%c2%ab%d4%9f%ca%b8%f3%b3%a2%bf%c7%b1_u%d7%b4%c7%b1 +P%c4%98%2f +R +inbox +tag5 +unread +%7e%d1%8b%25%ec%a0%ae%d1%a0M%3b%e3%b6%b7%e9%a4%87%3c%db%9a%cc%a8%e1%96%9d +%c4%bf7%c7%ab9H%c4%99k%ea%91%bd%c3%8ck%e2%b3%8dk%c5%952V%e4%99%b2%d9%b3%e4%8b%bda%5b%24%c7%9b +%da%88=f%cc%b9I%ce%af%7b%c9%97%e3%b9%8bH%cb%92X%d2%8c6 +%dc%9crh%d2%86B%e5%97%a2%22t%ed%99%82d -- id:msg-001@notmuch-test-suite
 EOF
 
 +%2a@%7d%cf%b5%f4%85%80%adO3%da%a7 +=%e0%ac%95%c8%b3+%ef%aa%95%c8%a64w%c7%9d%c9%a2%cf%b3%d6%82%24B%c4%a9%c5%a1UX%ee%99%b0%27E7%ca%a4%d0%8b%5d +A%e1%a0%bc%de%8b%d5%b2V%d9%9b%f3%b5%a2%a3M%d8%a1u@%f0%a0%ac%948%7e%f0%ab%86%af%27 +L%df%85%ef%a1%a5m@%d3%96%c2%ab%d4%9f%ca%b8%f3%b3%a2%bf%c7%b1_u%d7%b4%c7%b1 +P%c4%98%2f +R +inbox +tag5 +unread +%7e%d1%8b%25%ec%a0%ae%d1%a0M%3b%e3%b6%b7%e9%a4%87%3c%db%9a%cc%a8%e1%96%9d +%c4%bf7%c7%ab9H%c4%99k%ea%91%bd%c3%8ck%e2%b3%8dk%c5%952V%e4%99%b2%d9%b3%e4%8b%bda%5b%24%c7%9b +%da%88=f%cc%b9I%ce%af%7b%c9%97%e3%b9%8bH%cb%92X%d2%8c6 +%dc%9crh%d2%86B%e5%97%a2%22t%ed%99%82d -- id:msg-001@notmuch-test-suite
 EOF
 

-notmuch dump --format=batch-tag | sort > OUTPUT
+NOTMUCH_DUMP_TAGS > OUTPUT

 notmuch restore --format=batch-tag < BACKUP
 test_expect_equal_file EXPECTED OUTPUT
 
 notmuch restore --format=batch-tag < BACKUP
 test_expect_equal_file EXPECTED OUTPUT
 


@@ -260,7 +260,7 @@ cat <<EOF > EXPECTED
 +foo%3a%3abar%25 +found%3a%3ait +inbox +tag5 +unread +winner -- id:msg-001@notmuch-test-suite
 EOF
 
 +foo%3a%3abar%25 +found%3a%3ait +inbox +tag5 +unread +winner -- id:msg-001@notmuch-test-suite
 EOF
 

-notmuch dump --format=batch-tag | sort > OUTPUT
+NOTMUCH_DUMP_TAGS > OUTPUT

 notmuch restore --format=batch-tag < BACKUP
 test_expect_equal_file EXPECTED OUTPUT
 
 notmuch restore --format=batch-tag < BACKUP
 test_expect_equal_file EXPECTED OUTPUT
 

diff --git a/test/T240-dump-restore.sh b/test/T240-dump-restore.sh
index e6976ff8f2788634d202ed63e0960b0fa0cf6227..bbfb09b928b0d322bd71cb57736596880299efce 100755 (executable)
--- a/test/T240-dump-restore.sh
+++ b/test/T240-dump-restore.sh
@@ -97,7 +97,7 @@ test_expect_equal_file dump.expected dump.actual
 # Note, we assume all messages from cworth have a message-id
 # containing cworth.org
 
 # Note, we assume all messages from cworth have a message-id
 # containing cworth.org
 

-grep 'cworth[.]org' dump.expected > dump-cworth.expected
+{ head -1 dump.expected ; grep 'cworth[.]org' dump.expected; } > dump-cworth.expected

 
 test_begin_subtest "dump -- from:cworth"
 notmuch dump -- from:cworth > dump-dash-cworth.actual
 
 test_begin_subtest "dump -- from:cworth"
 notmuch dump -- from:cworth > dump-dash-cworth.actual


@@ -118,16 +118,16 @@ notmuch search --output=messages from:cworth | sed s/^id:// |\
 test_expect_equal_file OUTPUT EXPECTED
 
 test_begin_subtest "format=batch-tag, dump sanity check."
 test_expect_equal_file OUTPUT EXPECTED
 
 test_begin_subtest "format=batch-tag, dump sanity check."

-notmuch dump --format=sup from:cworth | cut -f1 -d' ' | \
+NOTMUCH_DUMP_TAGS --format=sup from:cworth | cut -f1 -d' ' | \

     sort > EXPECTED.$test_count
     sort > EXPECTED.$test_count

-notmuch dump --format=batch-tag from:cworth | sed 's/^.*-- id://' | \
+NOTMUCH_DUMP_TAGS --format=batch-tag from:cworth | sed 's/^.*-- id://' | \

     sort > OUTPUT.$test_count
 test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
 
 test_begin_subtest "format=batch-tag, missing newline"
 printf "+a_tag_without_newline -- id:20091117232137.GA7669@griffis1.net" > IN
 notmuch restore --accumulate < IN
     sort > OUTPUT.$test_count
 test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
 
 test_begin_subtest "format=batch-tag, missing newline"
 printf "+a_tag_without_newline -- id:20091117232137.GA7669@griffis1.net" > IN
 notmuch restore --accumulate < IN

-notmuch dump id:20091117232137.GA7669@griffis1.net > OUT
+NOTMUCH_DUMP_TAGS id:20091117232137.GA7669@griffis1.net > OUT

 cat <<EOF > EXPECTED
 +a_tag_without_newline +inbox +unread -- id:20091117232137.GA7669@griffis1.net
 EOF
 cat <<EOF > EXPECTED
 +a_tag_without_newline +inbox +unread -- id:20091117232137.GA7669@griffis1.net
 EOF


@@ -155,7 +155,7 @@ cat <<EOF >EXPECTED.$test_count
 + -- id:20091117232137.GA7669@griffis1.net
 EOF
 notmuch restore --format=batch-tag < EXPECTED.$test_count
 + -- id:20091117232137.GA7669@griffis1.net
 EOF
 notmuch restore --format=batch-tag < EXPECTED.$test_count

-notmuch dump --format=batch-tag id:20091117232137.GA7669@griffis1.net > OUTPUT.$test_count
+NOTMUCH_DUMP_TAGS --format=batch-tag id:20091117232137.GA7669@griffis1.net > OUTPUT.$test_count

 test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
 
 tag1='comic_swear=$&^%$^%\\//-+$^%$'
 test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
 
 tag1='comic_swear=$&^%$^%\\//-+$^%$'


@@ -217,9 +217,9 @@ notmuch dump --format=batch-tag > OUTPUT.$test_count
 test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
 
 test_begin_subtest 'format=batch-tag, checking encoded output'
 test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
 
 test_begin_subtest 'format=batch-tag, checking encoded output'

-notmuch dump --format=batch-tag -- from:cworth |\
+NOTMUCH_DUMP_TAGS --format=batch-tag -- from:cworth |\

         awk "{ print \"+$enc1 +$enc2 +$enc3 -- \" \$5 }" > EXPECTED.$test_count
         awk "{ print \"+$enc1 +$enc2 +$enc3 -- \" \$5 }" > EXPECTED.$test_count

-notmuch dump --format=batch-tag -- from:cworth  > OUTPUT.$test_count
+NOTMUCH_DUMP_TAGS --format=batch-tag -- from:cworth  > OUTPUT.$test_count

 test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
 
 test_begin_subtest 'restoring sane tags'
 test_expect_equal_file EXPECTED.$test_count OUTPUT.$test_count
 
 test_begin_subtest 'restoring sane tags'

diff --git a/test/T590-libconfig.sh b/test/T590-libconfig.sh
index 8ca688334079662507e697246c3616de730e8a49..5ea53003b91c727795e56635b1cff388e734726a 100755 (executable)
--- a/test/T590-libconfig.sh
+++ b/test/T590-libconfig.sh
@@ -115,4 +115,21 @@ testkey2 testvalue2
 EOF
 test_expect_equal_file EXPECTED OUTPUT
 
 EOF
 test_expect_equal_file EXPECTED OUTPUT
 

+test_begin_subtest "dump config"
+cat c_head - c_tail <<'EOF' | test_C ${MAIL_DIR}
+{
+    RUN(notmuch_database_set_config (db, "key with spaces", "value, with, spaces!"));
+}
+EOF
+notmuch dump --include=config >OUTPUT
+cat <<'EOF' >EXPECTED
+#notmuch-dump batch-tag:2 config
+#@ aaabefore beforeval
+#@ key%20with%20spaces value,%20with,%20spaces%21
+#@ testkey1 testvalue1
+#@ testkey2 testvalue2
+#@ zzzafter afterval
+EOF
+test_expect_equal_file EXPECTED OUTPUT
+

 test_done
 test_done

diff --git a/test/test-lib.sh b/test/test-lib.sh
index 62e123d65aba8deae1f6e461ac8c4442562f608a..e43986206ced0ae1df88e1be3be82138cecc7e1c 100644 (file)
--- a/test/test-lib.sh
+++ b/test/test-lib.sh
@@ -674,6 +674,12 @@ NOTMUCH_NEW ()
     notmuch new "${@}" | grep -v -E -e '^Processed [0-9]*( total)? file|Found [0-9]* total file'
 }
 
     notmuch new "${@}" | grep -v -E -e '^Processed [0-9]*( total)? file|Found [0-9]* total file'
 }
 

+NOTMUCH_DUMP_TAGS ()
+{
+    # this relies on the default format being batch-tag, otherwise some tests will break
+    notmuch dump --include=tags "${@}" | sed '/^#/d' | sort
+}
+

 notmuch_search_sanitize ()
 {
     perl -pe 's/("?thread"?: ?)("?)................("?)/\1\2XXX\3/'
 notmuch_search_sanitize ()
 {
     perl -pe 's/("?thread"?: ?)("?)................("?)/\1\2XXX\3/'