cli: search: Add configurable way to filter out duplicate addresses

This adds an algorithm to filter out duplicate addresses from address
outputs (sender, receivers). The algorithm can be configured with
--filter-by command line option.

The code here is an extended version of a patch from Jani Nikula.

diff --git a/completion/notmuch-completion.bash b/completion/notmuch-completion.bash
index cfbd3890e09179ae8a1966c24c21dd46a114b31f..6b6d43a3f8a268d06d80b5a30bff19e9c8c329a2 100644 (file)
--- a/completion/notmuch-completion.bash
+++ b/completion/notmuch-completion.bash
@@ -305,12 +305,16 @@ _notmuch_search()
            COMPREPLY=( $( compgen -W "true false flag all" -- "${cur}" ) )
            return
            ;;
            COMPREPLY=( $( compgen -W "true false flag all" -- "${cur}" ) )
            return
            ;;

+       --filter-by)
+           COMPREPLY=( $( compgen -W "nameaddr name addr addrfold nameaddrfold" -- "${cur}" ) )
+           return
+           ;;

     esac
 
     ! $split &&
     case "${cur}" in
        -*)
     esac
 
     ! $split &&
     case "${cur}" in
        -*)

-           local options="--format= --output= --sort= --offset= --limit= --exclude= --duplicate="
+           local options="--format= --output= --sort= --offset= --limit= --exclude= --duplicate= --filter-by="

            compopt -o nospace
            COMPREPLY=( $(compgen -W "$options" -- ${cur}) )
            ;;
            compopt -o nospace
            COMPREPLY=( $(compgen -W "$options" -- ${cur}) )
            ;;

diff --git a/completion/notmuch-completion.zsh b/completion/notmuch-completion.zsh
index 3e52a004d8ee9530b2082ad72cd329513f6657d2..3e535df31997b6d569406099d69ce1acbbd1a1c1 100644 (file)
--- a/completion/notmuch-completion.zsh
+++ b/completion/notmuch-completion.zsh
@@ -53,7 +53,8 @@ _notmuch_search()
     '--max-threads=[display only the first x threads from the search results]:number of threads to show: ' \
     '--first=[omit the first x threads from the search results]:number of threads to omit: ' \
     '--sort=[sort results]:sorting:((newest-first\:"reverse chronological order" oldest-first\:"chronological order"))' \
     '--max-threads=[display only the first x threads from the search results]:number of threads to show: ' \
     '--first=[omit the first x threads from the search results]:number of threads to omit: ' \
     '--sort=[sort results]:sorting:((newest-first\:"reverse chronological order" oldest-first\:"chronological order"))' \

-    '--output=[select what to output]:output:((summary threads messages files tags sender recipients))'
+    '--output=[select what to output]:output:((summary threads messages files tags sender recipients))' \
+    '--filter-by=[filter out duplicate addresses]:filter-by:((nameaddr\:"both name and address part" name\:"name part" addr\:"address part" addrfold\:"case-insensitive address part" nameaddrfold\:"name and case-insensitive address part"))'

 }
 
 _notmuch()
 }
 
 _notmuch()

diff --git a/doc/man1/notmuch-search.rst b/doc/man1/notmuch-search.rst
index b6607c922cc083c35b9add629c771d6d3927da1b..84af2da428ec3feffcd9f3b6b3e12a467661684f 100644 (file)
--- a/doc/man1/notmuch-search.rst
+++ b/doc/man1/notmuch-search.rst
@@ -85,6 +85,9 @@ Supported options for **search** include
             (--format=text0), as a JSON array (--format=json), or as
             an S-Expression list (--format=sexp).
 
             (--format=text0), as a JSON array (--format=json), or as
             an S-Expression list (--format=sexp).
 

+            Duplicate addresses are filtered out. Filtering can be
+            configured with the --filter-by option.
+

            Note: Searching for **sender** should be much faster than
            searching for **recipients**, because sender addresses are
            cached directly in the database whereas other addresses
            Note: Searching for **sender** should be much faster than
            searching for **recipients**, because sender addresses are
            cached directly in the database whereas other addresses


@@ -151,6 +154,41 @@ Supported options for **search** include
         prefix. The prefix matches messages based on filenames. This
         option filters filenames of the matching messages.
 
         prefix. The prefix matches messages based on filenames. This
         option filters filenames of the matching messages.
 

+    ``--filter-by=``\ (**nameaddr**\ \|\ **name** \|\ **addr**\ \|\ **addrfold**\ \|\ **nameaddrfold**\)
+
+       Can be used with ``--output=sender`` or
+       ``--output=recipients`` to filter out duplicate addresses. The
+       filtering algorithm receives a sequence of email addresses and
+       outputs the same sequence without the addresses that are
+       considered a duplicate of a previously output address. What is
+       considered a duplicate depends on how the two addresses are
+       compared and this can be controlled with the follwing flags:
+
+       **nameaddr** means that both name and address parts are
+       compared in case-sensitive manner. Therefore, all same looking
+       addresses strings are considered duplicate. This is the
+       default.
+
+       **name** means that only the name part is compared (in
+       case-sensitive manner). For example, the addresses "John Doe
+       <me@example.com>" and "John Doe <john@doe.name>" will be
+       considered duplicate.
+
+       **addr** means that only the address part is compared (in
+       case-sensitive manner). For example, the addresses "John Doe
+       <john@example.com>" and "Dr. John Doe <john@example.com>" will
+       be considered duplicate.
+
+       **addrfold** is like **addr**, but comparison is done in
+       canse-insensitive manner. For example, the addresses "John Doe
+       <john@example.com>" and "Dr. John Doe <JOHN@EXAMPLE.COM>" will
+       be considered duplicate.
+
+       **nameaddrfold** is like **nameaddr**, but address comparison
+       is done in canse-insensitive manner. For example, the
+       addresses "John Doe <john@example.com>" and "John Doe
+       <JOHN@EXAMPLE.COM>" will be considered duplicate.
+

 EXIT STATUS
 ===========
 
 EXIT STATUS
 ===========
 

diff --git a/notmuch-search.c b/notmuch-search.c
index ce3bfb229655a877e6eaf5431fc825f461588998..47aa97902e69e4aec23773a020a742cab948f858 100644 (file)
--- a/notmuch-search.c
+++ b/notmuch-search.c
@@ -34,6 +34,14 @@ typedef enum {
 
 #define OUTPUT_ADDRESS_FLAGS (OUTPUT_SENDER | OUTPUT_RECIPIENTS)
 
 
 #define OUTPUT_ADDRESS_FLAGS (OUTPUT_SENDER | OUTPUT_RECIPIENTS)
 

+typedef enum {
+    FILTER_BY_NAMEADDR = 0,
+    FILTER_BY_NAME,
+    FILTER_BY_ADDR,
+    FILTER_BY_ADDRFOLD,
+    FILTER_BY_NAMEADDRFOLD,
+} filter_by_t;
+

 typedef struct {
     sprinter_t *format;
     notmuch_query_t *query;
 typedef struct {
     sprinter_t *format;
     notmuch_query_t *query;


@@ -42,6 +50,7 @@ typedef struct {
     int offset;
     int limit;
     int dupe;
     int offset;
     int limit;
     int dupe;

+    filter_by_t filter_by;

 } search_options_t;
 
 typedef struct {
 } search_options_t;
 
 typedef struct {


@@ -229,6 +238,52 @@ do_search_threads (search_options_t *opt)
     return 0;
 }
 
     return 0;
 }
 

+/* Returns TRUE iff name and/or addr is considered duplicite. */
+static notmuch_bool_t
+check_duplicite (const search_options_t *opt, GHashTable *addrs, const char *name, const char *addr)
+{
+    notmuch_bool_t duplicite;
+    char *key;
+
+    if (opt->filter_by == FILTER_BY_ADDRFOLD ||
+       opt->filter_by == FILTER_BY_NAMEADDRFOLD) {
+       gchar *folded = g_utf8_casefold (addr, -1);
+       addr = talloc_strdup (opt->format, folded);
+       g_free (folded);
+    }
+    switch (opt->filter_by) {
+    case FILTER_BY_NAMEADDR:
+    case FILTER_BY_NAMEADDRFOLD:
+       key = talloc_asprintf (opt->format, "%s <%s>", name, addr);
+       break;
+    case FILTER_BY_NAME:
+       key = talloc_strdup (opt->format, name); /* !name results in !key */
+       break;
+    case FILTER_BY_ADDR:
+    case FILTER_BY_ADDRFOLD:
+       key = talloc_strdup (opt->format, addr);
+       break;
+    default:
+       INTERNAL_ERROR("invalid --filter-by flags");
+    }
+
+    if (opt->filter_by == FILTER_BY_ADDRFOLD ||
+       opt->filter_by == FILTER_BY_NAMEADDRFOLD)
+       talloc_free ((char*)addr);
+
+    if (! key)
+       return FALSE;
+
+    duplicite = g_hash_table_lookup_extended (addrs, key, NULL, NULL);
+
+    if (! duplicite)
+       g_hash_table_insert (addrs, key, NULL);
+    else
+       talloc_free (key);
+
+    return duplicite;
+}
+

 static void
 print_mailbox (const search_options_t *opt, const mailbox_t *mailbox)
 {
 static void
 print_mailbox (const search_options_t *opt, const mailbox_t *mailbox)
 {


@@ -263,7 +318,8 @@ print_mailbox (const search_options_t *opt, const mailbox_t *mailbox)
 }
 
 static void
 }
 
 static void

-process_address_list (const search_options_t *opt, InternetAddressList *list)
+process_address_list (const search_options_t *opt, GHashTable *addrs,
+                     InternetAddressList *list)

 {
     InternetAddress *address;
     int i;
 {
     InternetAddress *address;
     int i;


@@ -279,7 +335,7 @@ process_address_list (const search_options_t *opt, InternetAddressList *list)
            if (group_list == NULL)
                continue;
 
            if (group_list == NULL)
                continue;
 

-           process_address_list (opt, group_list);
+           process_address_list (opt, addrs, group_list);

        } else {
            InternetAddressMailbox *mailbox = INTERNET_ADDRESS_MAILBOX (address);
            mailbox_t mbx = {
        } else {
            InternetAddressMailbox *mailbox = INTERNET_ADDRESS_MAILBOX (address);
            mailbox_t mbx = {


@@ -287,13 +343,16 @@ process_address_list (const search_options_t *opt, InternetAddressList *list)
                .addr = internet_address_mailbox_get_addr (mailbox),
            };
 
                .addr = internet_address_mailbox_get_addr (mailbox),
            };
 

+           if (check_duplicite (opt, addrs, mbx.name, mbx.addr))
+               continue;
+

            print_mailbox (opt, &mbx);
        }
     }
 }
 
 static void
            print_mailbox (opt, &mbx);
        }
     }
 }
 
 static void

-process_address_header (const search_options_t *opt, const char *value)
+process_address_header (const search_options_t *opt, GHashTable *addrs, const char *value)

 {
     InternetAddressList *list;
 
 {
     InternetAddressList *list;
 


@@ -304,7 +363,13 @@ process_address_header (const search_options_t *opt, const char *value)
     if (list == NULL)
        return;
 
     if (list == NULL)
        return;
 

-    process_address_list (opt, list);
+    process_address_list (opt, addrs, list);
+}
+
+static void
+_my_talloc_free_for_g_hash (void *ptr)
+{
+    talloc_free (ptr);

 }
 
 static int
 }
 
 static int


@@ -314,8 +379,13 @@ do_search_messages (search_options_t *opt)
     notmuch_messages_t *messages;
     notmuch_filenames_t *filenames;
     sprinter_t *format = opt->format;
     notmuch_messages_t *messages;
     notmuch_filenames_t *filenames;
     sprinter_t *format = opt->format;

+    GHashTable *addresses = NULL;

     int i;
 
     int i;
 

+    if (opt->output & OUTPUT_ADDRESS_FLAGS)
+       addresses = g_hash_table_new_full (g_str_hash, g_str_equal,
+                                          _my_talloc_free_for_g_hash, NULL);
+

     if (opt->offset < 0) {
        opt->offset += notmuch_query_count_messages (opt->query);
        if (opt->offset < 0)
     if (opt->offset < 0) {
        opt->offset += notmuch_query_count_messages (opt->query);
        if (opt->offset < 0)


@@ -363,7 +433,7 @@ do_search_messages (search_options_t *opt)
                const char *addrs;
 
                addrs = notmuch_message_get_header (message, "from");
                const char *addrs;
 
                addrs = notmuch_message_get_header (message, "from");

-               process_address_header (opt, addrs);
+               process_address_header (opt, addresses, addrs);

            }
 
            if (opt->output & OUTPUT_RECIPIENTS) {
            }
 
            if (opt->output & OUTPUT_RECIPIENTS) {


@@ -373,7 +443,7 @@ do_search_messages (search_options_t *opt)
 
                for (j = 0; j < ARRAY_SIZE (hdrs); j++) {
                    addrs = notmuch_message_get_header (message, hdrs[j]);
 
                for (j = 0; j < ARRAY_SIZE (hdrs); j++) {
                    addrs = notmuch_message_get_header (message, hdrs[j]);

-                   process_address_header (opt, addrs);
+                   process_address_header (opt, addresses, addrs);

                }
            }
        }
                }
            }
        }


@@ -381,6 +451,9 @@ do_search_messages (search_options_t *opt)
        notmuch_message_destroy (message);
     }
 
        notmuch_message_destroy (message);
     }
 

+    if (addresses)
+       g_hash_table_unref (addresses);
+

     notmuch_messages_destroy (messages);
 
     format->end (format);
     notmuch_messages_destroy (messages);
 
     format->end (format);


@@ -447,6 +520,7 @@ notmuch_search_command (notmuch_config_t *config, int argc, char *argv[])
        .offset = 0,
        .limit = -1, /* unlimited */
        .dupe = -1,
        .offset = 0,
        .limit = -1, /* unlimited */
        .dupe = -1,

+       .filter_by = FILTER_BY_NAMEADDR,

     };
     char *query_str;
     int opt_index, ret;
     };
     char *query_str;
     int opt_index, ret;


@@ -490,6 +564,13 @@ notmuch_search_command (notmuch_config_t *config, int argc, char *argv[])
        { NOTMUCH_OPT_INT, &opt.offset, "offset", 'O', 0 },
        { NOTMUCH_OPT_INT, &opt.limit, "limit", 'L', 0  },
        { NOTMUCH_OPT_INT, &opt.dupe, "duplicate", 'D', 0  },
        { NOTMUCH_OPT_INT, &opt.offset, "offset", 'O', 0 },
        { NOTMUCH_OPT_INT, &opt.limit, "limit", 'L', 0  },
        { NOTMUCH_OPT_INT, &opt.dupe, "duplicate", 'D', 0  },

+       { NOTMUCH_OPT_KEYWORD, &opt.filter_by, "filter-by", 'b',
+         (notmuch_keyword_t []){ { "nameaddr", FILTER_BY_NAMEADDR },
+                                 { "name", FILTER_BY_NAME },
+                                 { "addr", FILTER_BY_ADDR },
+                                 { "addrfold", FILTER_BY_ADDRFOLD },
+                                 { "nameaddrfold", FILTER_BY_NAMEADDRFOLD },
+                                 { 0, 0 } } },

        { 0, 0, 0, 0, 0 }
     };
 
        { 0, 0, 0, 0, 0 }
     };
 


@@ -500,6 +581,11 @@ notmuch_search_command (notmuch_config_t *config, int argc, char *argv[])
     if (! opt.output)
        opt.output = OUTPUT_SUMMARY;
 
     if (! opt.output)
        opt.output = OUTPUT_SUMMARY;
 

+    if (opt.filter_by && !(opt.output & OUTPUT_ADDRESS_FLAGS)) {
+       fprintf (stderr, "Error: --filter-by can only be used with address output.\n");
+       return EXIT_FAILURE;
+    }
+

     switch (format_sel) {
     case NOTMUCH_FORMAT_TEXT:
        opt.format = sprinter_text_create (config, stdout);
     switch (format_sel) {
     case NOTMUCH_FORMAT_TEXT:
        opt.format = sprinter_text_create (config, stdout);

diff --git a/test/T090-search-output.sh b/test/T090-search-output.sh
index 947d572ebeff71418ae282868500f6b50f195f7b..841a7219272a450edd105ac0142dadaa9b33e102 100755 (executable)
--- a/test/T090-search-output.sh
+++ b/test/T090-search-output.sh
@@ -387,6 +387,93 @@ cat <<EOF >EXPECTED
 EOF
 test_expect_equal_file OUTPUT EXPECTED
 
 EOF
 test_expect_equal_file OUTPUT EXPECTED
 

+test_begin_subtest "--output=sender"
+notmuch search --output=sender '*' >OUTPUT
+cat <<EOF >EXPECTED
+François Boulogne <boulogne.f@gmail.com>
+Olivier Berger <olivier.berger@it-sudparis.eu>
+Chris Wilson <chris@chris-wilson.co.uk>
+Carl Worth <cworth@cworth.org>
+Alexander Botero-Lowry <alex.boterolowry@gmail.com>
+Keith Packard <keithp@keithp.com>
+Jjgod Jiang <gzjjgod@gmail.com>
+Rolland Santimano <rollandsantimano@yahoo.com>
+Jan Janak <jan@ryngle.com>
+Stewart Smith <stewart@flamingspork.com>
+Lars Kellogg-Stedman <lars@seas.harvard.edu>
+Alex Botero-Lowry <alex.boterolowry@gmail.com>
+Ingmar Vanhassel <ingmar@exherbo.org>
+Aron Griffis <agriffis@n01se.net>
+Adrian Perez de Castro <aperez@igalia.com>
+Israel Herraiz <isra@herraiz.org>
+Mikhail Gusarov <dottedmag@dottedmag.net>
+EOF
+test_expect_equal_file OUTPUT EXPECTED
+
+test_begin_subtest "--output=sender --format=json"
+notmuch search --output=sender --format=json '*' >OUTPUT
+cat <<EOF >EXPECTED
+[{"name": "François Boulogne", "address": "boulogne.f@gmail.com"},
+{"name": "Olivier Berger", "address": "olivier.berger@it-sudparis.eu"},
+{"name": "Chris Wilson", "address": "chris@chris-wilson.co.uk"},
+{"name": "Carl Worth", "address": "cworth@cworth.org"},
+{"name": "Alexander Botero-Lowry", "address": "alex.boterolowry@gmail.com"},
+{"name": "Keith Packard", "address": "keithp@keithp.com"},
+{"name": "Jjgod Jiang", "address": "gzjjgod@gmail.com"},
+{"name": "Rolland Santimano", "address": "rollandsantimano@yahoo.com"},
+{"name": "Jan Janak", "address": "jan@ryngle.com"},
+{"name": "Stewart Smith", "address": "stewart@flamingspork.com"},
+{"name": "Lars Kellogg-Stedman", "address": "lars@seas.harvard.edu"},
+{"name": "Alex Botero-Lowry", "address": "alex.boterolowry@gmail.com"},
+{"name": "Ingmar Vanhassel", "address": "ingmar@exherbo.org"},
+{"name": "Aron Griffis", "address": "agriffis@n01se.net"},
+{"name": "Adrian Perez de Castro", "address": "aperez@igalia.com"},
+{"name": "Israel Herraiz", "address": "isra@herraiz.org"},
+{"name": "Mikhail Gusarov", "address": "dottedmag@dottedmag.net"}]
+EOF
+test_expect_equal_file OUTPUT EXPECTED
+
+test_begin_subtest "--output=recipients"
+notmuch search --output=recipients '*' >OUTPUT
+cat <<EOF >EXPECTED
+Allan McRae <allan@archlinux.org>
+Discussion about the Arch User Repository (AUR) <aur-general@archlinux.org>
+olivier.berger@it-sudparis.eu
+notmuch@notmuchmail.org
+notmuch <notmuch@notmuchmail.org>
+Keith Packard <keithp@keithp.com>
+Mikhail Gusarov <dottedmag@dottedmag.net>
+EOF
+test_expect_equal_file OUTPUT EXPECTED
+
+test_begin_subtest "--output=sender --output=recipients"
+notmuch search --output=sender --output=recipients '*' >OUTPUT
+cat <<EOF >EXPECTED
+François Boulogne <boulogne.f@gmail.com>
+Allan McRae <allan@archlinux.org>
+Discussion about the Arch User Repository (AUR) <aur-general@archlinux.org>
+Olivier Berger <olivier.berger@it-sudparis.eu>
+olivier.berger@it-sudparis.eu
+Chris Wilson <chris@chris-wilson.co.uk>
+notmuch@notmuchmail.org
+Carl Worth <cworth@cworth.org>
+Alexander Botero-Lowry <alex.boterolowry@gmail.com>
+Keith Packard <keithp@keithp.com>
+Jjgod Jiang <gzjjgod@gmail.com>
+Rolland Santimano <rollandsantimano@yahoo.com>
+Jan Janak <jan@ryngle.com>
+Stewart Smith <stewart@flamingspork.com>
+Lars Kellogg-Stedman <lars@seas.harvard.edu>
+notmuch <notmuch@notmuchmail.org>
+Alex Botero-Lowry <alex.boterolowry@gmail.com>
+Ingmar Vanhassel <ingmar@exherbo.org>
+Aron Griffis <agriffis@n01se.net>
+Adrian Perez de Castro <aperez@igalia.com>
+Israel Herraiz <isra@herraiz.org>
+Mikhail Gusarov <dottedmag@dottedmag.net>
+EOF
+test_expect_equal_file OUTPUT EXPECTED
+

 test_begin_subtest "sanitize output for quoted-printable line-breaks in author and subject"
 add_message "[subject]='two =?ISO-8859-1?Q?line=0A_subject?=
        headers'"
 test_begin_subtest "sanitize output for quoted-printable line-breaks in author and subject"
 add_message "[subject]='two =?ISO-8859-1?Q?line=0A_subject?=
        headers'"

diff --git a/test/T095-search-filter-by.sh b/test/T095-search-filter-by.sh
new file mode 100755 (executable)
index 0000000..97d9a9b
--- /dev/null
+++ b/test/T095-search-filter-by.sh
@@ -0,0 +1,64 @@
+#!/usr/bin/env bash
+test_description='duplicite address filtering in "notmuch search --output=recipients"'
+. ./test-lib.sh
+
+add_message '[to]="Real Name <foo@example.com>, Real Name <bar@example.com>"'
+add_message '[to]="Nickname <foo@example.com>"' '[cc]="Real Name <Bar@Example.COM>"'
+add_message '[to]="Nickname <foo@example.com>"' '[bcc]="Real Name <Bar@Example.COM>"'
+
+test_begin_subtest "--output=recipients"
+notmuch search --output=recipients "*" >OUTPUT
+cat <<EOF >EXPECTED
+Real Name <foo@example.com>
+Real Name <bar@example.com>
+Nickname <foo@example.com>
+Real Name <Bar@Example.COM>
+EOF
+test_expect_equal_file OUTPUT EXPECTED
+
+test_begin_subtest "--output=recipients --filter-by=nameaddr"
+notmuch search --output=recipients --filter-by=nameaddr "*" >OUTPUT
+# The same as above
+cat <<EOF >EXPECTED
+Real Name <foo@example.com>
+Real Name <bar@example.com>
+Nickname <foo@example.com>
+Real Name <Bar@Example.COM>
+EOF
+test_expect_equal_file OUTPUT EXPECTED
+
+test_begin_subtest "--output=recipients --filter-by=name"
+notmuch search --output=recipients --filter-by=name "*" >OUTPUT
+cat <<EOF >EXPECTED
+Real Name <foo@example.com>
+Nickname <foo@example.com>
+EOF
+test_expect_equal_file OUTPUT EXPECTED
+
+test_begin_subtest "--output=recipients --filter-by=addr"
+notmuch search --output=recipients --filter-by=addr "*" >OUTPUT
+cat <<EOF >EXPECTED
+Real Name <foo@example.com>
+Real Name <bar@example.com>
+Real Name <Bar@Example.COM>
+EOF
+test_expect_equal_file OUTPUT EXPECTED
+
+test_begin_subtest "--output=recipients --filter-by=addrfold"
+notmuch search --output=recipients --filter-by=addrfold "*" >OUTPUT
+cat <<EOF >EXPECTED
+Real Name <foo@example.com>
+Real Name <bar@example.com>
+EOF
+test_expect_equal_file OUTPUT EXPECTED
+
+test_begin_subtest "--output=recipients --filter-by=nameaddrfold"
+notmuch search --output=recipients --filter-by=nameaddrfold "*" >OUTPUT
+cat <<EOF >EXPECTED
+Real Name <foo@example.com>
+Real Name <bar@example.com>
+Nickname <foo@example.com>
+EOF
+test_expect_equal_file OUTPUT EXPECTED
+
+test_done