3 # Copyright (c) 2011-2014 David Bremner <david@tethera.net>
4 # W. Trevor King <wking@tremily.us>
6 # This program is free software: you can redistribute it and/or modify
7 # it under the terms of the GNU General Public License as published by
8 # the Free Software Foundation, either version 3 of the License, or
9 # (at your option) any later version.
11 # This program is distributed in the hope that it will be useful,
12 # but WITHOUT ANY WARRANTY; without even the implied warranty of
13 # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 # GNU General Public License for more details.
16 # You should have received a copy of the GNU General Public License
17 # along with this program. If not, see http://www.gnu.org/licenses/ .
20 Manage notmuch tags with Git
22 Environment variables:
24 * NMBGIT specifies the location of the git repository used by nmbug.
25 If not specified $HOME/.nmbug is used.
26 * NMBPREFIX specifies the prefix in the notmuch database for tags of
27 interest to nmbug. If not specified 'notmuch::' is used.
30 from __future__ import print_function
31 from __future__ import unicode_literals
33 import codecs as _codecs
34 import collections as _collections
35 import inspect as _inspect
36 import locale as _locale
37 import logging as _logging
40 import shutil as _shutil
41 import subprocess as _subprocess
43 import tempfile as _tempfile
44 import textwrap as _textwrap
46 from urllib.parse import quote as _quote
47 from urllib.parse import unquote as _unquote
48 except ImportError: # Python 2
49 from urllib import quote as _quote
50 from urllib import unquote as _unquote
55 _LOG = _logging.getLogger('nmbug')
56 _LOG.setLevel(_logging.ERROR)
57 _LOG.addHandler(_logging.StreamHandler())
59 NMBGIT = _os.path.expanduser(
60 _os.getenv('NMBGIT', _os.path.join('~', '.nmbug')))
61 _NMBGIT = _os.path.join(NMBGIT, '.git')
62 if _os.path.isdir(_NMBGIT):
65 TAG_PREFIX = _os.getenv('NMBPREFIX', 'notmuch::')
66 _HEX_ESCAPE_REGEX = _re.compile('%[0-9A-F]{2}')
67 _TAG_FILE_REGEX = _re.compile('tags/(?P<id>[^/]*)/(?P<tag>[^/]*)')
69 # magic hash for Git (git hash-object -t blob /dev/null)
70 _EMPTYBLOB = 'e69de29bb2d1d6434b8b29ae775ad8c2e48c5391'
74 getattr(_tempfile, 'TemporaryDirectory')
75 except AttributeError: # Python < 3.2
76 class _TemporaryDirectory(object):
78 Fallback context manager for Python < 3.2
80 See PEP 343 for details on context managers [1].
82 [1]: http://legacy.python.org/dev/peps/pep-0343/
84 def __init__(self, **kwargs):
85 self.name = _tempfile.mkdtemp(**kwargs)
90 def __exit__(self, type, value, traceback):
91 _shutil.rmtree(self.name)
94 _tempfile.TemporaryDirectory = _TemporaryDirectory
97 def _hex_quote(string, safe='+@=:,'):
99 quote('abc def') -> 'abc%20def'.
101 Wrap urllib.parse.quote with additional safe characters (in
102 addition to letters, digits, and '_.-') and lowercase hex digits
103 (e.g. '%3a' instead of '%3A').
105 uppercase_escapes = _quote(string, safe)
106 return _HEX_ESCAPE_REGEX.sub(
107 lambda match: match.group(0).lower(),
111 _ENCODED_TAG_PREFIX = _hex_quote(TAG_PREFIX, safe='+@=,') # quote ':'
114 def _xapian_quote(string):
116 Quote a string for Xapian's QueryParser.
118 Xapian uses double-quotes for quoting strings. You can escape
119 internal quotes by repeating them [1,2,3].
121 [1]: http://trac.xapian.org/ticket/128#comment:2
122 [2]: http://trac.xapian.org/ticket/128#comment:17
123 [3]: http://trac.xapian.org/changeset/13823/svn
125 return '"{0}"'.format(string.replace('"', '""'))
128 def _xapian_unquote(string):
130 Unquote a Xapian-quoted string.
132 if string.startswith('"') and string.endswith('"'):
133 return string[1:-1].replace('""', '"')
137 class SubprocessError(RuntimeError):
138 "A subprocess exited with a nonzero status"
139 def __init__(self, args, status, stdout=None, stderr=None):
143 msg = '{args} exited with {status}'.format(args=args, status=status)
145 msg = '{msg}: {stderr}'.format(msg=msg, stderr=stderr)
146 super(SubprocessError, self).__init__(msg)
149 class _SubprocessContextManager(object):
151 PEP 343 context manager for subprocesses.
153 'expect' holds a tuple of acceptable exit codes, otherwise we'll
154 raise a SubprocessError in __exit__.
156 def __init__(self, process, args, expect=(0,)):
157 self._process = process
159 self._expect = expect
164 def __exit__(self, type, value, traceback):
165 for name in ['stdin', 'stdout', 'stderr']:
166 stream = getattr(self._process, name)
169 setattr(self._process, name, None)
170 status = self._process.wait()
171 _LOG.debug('collect {args} with status {status}'.format(
172 args=self._args, status=status))
173 if status not in self._expect:
174 raise SubprocessError(args=self._args, status=status)
177 return self._process.wait()
180 def _spawn(args, input=None, additional_env=None, wait=False, stdin=None,
181 stdout=None, stderr=None, encoding=_locale.getpreferredencoding(),
182 expect=(0,), **kwargs):
183 """Spawn a subprocess, and optionally wait for it to finish.
185 This wrapper around subprocess.Popen has two modes, depending on
186 the truthiness of 'wait'. If 'wait' is true, we use p.communicate
187 internally to write 'input' to the subprocess's stdin and read
188 from it's stdout/stderr. If 'wait' is False, we return a
189 _SubprocessContextManager instance for fancier handling
190 (e.g. piping between processes).
192 For 'wait' calls when you want to write to the subprocess's stdin,
193 you only need to set 'input' to your content. When 'input' is not
194 None but 'stdin' is, we'll automatically set 'stdin' to PIPE
195 before calling Popen. This avoids having the subprocess
196 accidentally inherit the launching process's stdin.
198 _LOG.debug('spawn {args} (additional env. var.: {env})'.format(
199 args=args, env=additional_env))
200 if not stdin and input is not None:
201 stdin = _subprocess.PIPE
203 if not kwargs.get('env'):
204 kwargs['env'] = dict(_os.environ)
205 kwargs['env'].update(additional_env)
206 p = _subprocess.Popen(
207 args, stdin=stdin, stdout=stdout, stderr=stderr, **kwargs)
209 if hasattr(input, 'encode'):
210 input = input.encode(encoding)
211 (stdout, stderr) = p.communicate(input=input)
213 _LOG.debug('collect {args} with status {status}'.format(
214 args=args, status=status))
215 if stdout is not None:
216 stdout = stdout.decode(encoding)
217 if stderr is not None:
218 stderr = stderr.decode(encoding)
220 raise SubprocessError(
221 args=args, status=status, stdout=stdout, stderr=stderr)
222 return (status, stdout, stderr)
223 if p.stdin and not stdin:
227 p.stdin = _codecs.getwriter(encoding=encoding)(stream=p.stdin)
228 stream_reader = _codecs.getreader(encoding=encoding)
230 p.stdout = stream_reader(stream=p.stdout)
232 p.stderr = stream_reader(stream=p.stderr)
233 return _SubprocessContextManager(args=args, process=p, expect=expect)
236 def _git(args, **kwargs):
237 args = ['git', '--git-dir', NMBGIT] + list(args)
238 return _spawn(args=args, **kwargs)
241 def _get_current_branch():
242 """Get the name of the current branch.
244 Return 'None' if we're not on a branch.
247 (status, branch, stderr) = _git(
248 args=['symbolic-ref', '--short', 'HEAD'],
249 stdout=_subprocess.PIPE, stderr=_subprocess.PIPE, wait=True)
250 except SubprocessError as e:
251 if 'not a symbolic ref' in e:
254 return branch.strip()
258 "Get the default remote for the current branch."
259 local_branch = _get_current_branch()
260 (status, remote, stderr) = _git(
261 args=['config', 'branch.{0}.remote'.format(local_branch)],
262 stdout=_subprocess.PIPE, wait=True)
263 return remote.strip()
266 def get_tags(prefix=None):
267 "Get a list of tags with a given prefix."
270 (status, stdout, stderr) = _spawn(
271 args=['notmuch', 'search', '--output=tags', '*'],
272 stdout=_subprocess.PIPE, wait=True)
273 return [tag for tag in stdout.splitlines() if tag.startswith(prefix)]
276 def archive(treeish='HEAD', args=()):
278 Dump a tar archive of the current nmbug tag set.
282 Each tag $tag for message with Message-Id $id is written to
285 tags/encode($id)/encode($tag)
287 The encoding preserves alphanumerics, and the characters
288 "+-_@=.:," (not the quotes). All other octets are replaced with
289 '%' followed by a two digit hex number.
291 _git(args=['archive', treeish] + list(args), wait=True)
294 def clone(repository):
296 Create a local nmbug repository from a remote source.
298 This wraps 'git clone', adding some options to avoid creating a
299 working tree while preserving remote-tracking branches and
302 with _tempfile.TemporaryDirectory(prefix='nmbug-clone.') as workdir:
305 'git', 'clone', '--no-checkout', '--separate-git-dir', NMBGIT,
306 repository, workdir],
308 _git(args=['config', '--unset', 'core.worktree'], wait=True)
309 _git(args=['config', 'core.bare', 'true'], wait=True)
312 def _is_committed(status):
313 return len(status['added']) + len(status['deleted']) == 0
316 def commit(treeish='HEAD', message=None):
318 Commit prefix-matching tags from the notmuch database to Git.
320 status = get_status()
322 if _is_committed(status=status):
323 _LOG.warning('Nothing to commit')
326 _git(args=['read-tree', '--empty'], wait=True)
327 _git(args=['read-tree', treeish], wait=True)
329 _update_index(status=status)
332 stdout=_subprocess.PIPE,
334 (_, parent, _) = _git(
335 args=['rev-parse', treeish],
336 stdout=_subprocess.PIPE,
338 (_, commit, _) = _git(
339 args=['commit-tree', tree.strip(), '-p', parent.strip()],
341 stdout=_subprocess.PIPE,
344 args=['update-ref', treeish, commit.strip()],
345 stdout=_subprocess.PIPE,
347 except Exception as e:
348 _git(args=['read-tree', '--empty'], wait=True)
349 _git(args=['read-tree', treeish], wait=True)
352 def _update_index(status):
354 args=['update-index', '--index-info'],
355 stdin=_subprocess.PIPE) as p:
356 for id, tags in status['deleted'].items():
357 for line in _index_tags_for_message(id=id, status='D', tags=tags):
359 for id, tags in status['added'].items():
360 for line in _index_tags_for_message(id=id, status='A', tags=tags):
364 def fetch(remote=None):
366 Fetch changes from the remote repository.
368 See 'merge' to bring those changes into notmuch.
373 _git(args=args, wait=True)
378 Update the notmuch database from Git.
380 This is mainly useful to discard your changes in notmuch relative
383 status = get_status()
385 args=['notmuch', 'tag', '--batch'], stdin=_subprocess.PIPE) as p:
386 for id, tags in status['added'].items():
387 p.stdin.write(_batch_line(action='-', id=id, tags=tags))
388 for id, tags in status['deleted'].items():
389 p.stdin.write(_batch_line(action='+', id=id, tags=tags))
392 def _batch_line(action, id, tags):
394 'notmuch tag --batch' line for adding/removing tags.
396 Set 'action' to '-' to remove a tag or '+' to add the tags to a
399 tag_string = ' '.join(
400 '{action}{prefix}{tag}'.format(
401 action=action, prefix=_ENCODED_TAG_PREFIX, tag=_hex_quote(tag))
403 line = '{tags} -- id:{id}\n'.format(
404 tags=tag_string, id=_xapian_quote(string=id))
408 def _insist_committed():
409 "Die if the the notmuch tags don't match the current HEAD."
410 status = get_status()
411 if not _is_committed(status=status):
412 _LOG.error('\n'.join([
413 'Uncommitted changes to {prefix}* tags in notmuch',
415 "For a summary of changes, run 'nmbug status'",
416 "To save your changes, run 'nmbug commit' before merging/pull",
417 "To discard your changes, run 'nmbug checkout'",
418 ]).format(prefix=TAG_PREFIX))
422 def pull(repository=None, refspecs=None):
424 Pull (merge) remote repository changes to notmuch.
426 'pull' is equivalent to 'fetch' followed by 'merge'. We use the
427 Git-configured repository for your current branch
428 (branch.<name>.repository, likely 'origin', and
429 branch.<name>.merge, likely 'master').
432 if refspecs and not repository:
433 repository = _get_remote()
436 args.append(repository)
438 args.extend(refspecs)
439 with _tempfile.TemporaryDirectory(prefix='nmbug-pull.') as workdir:
445 additional_env={'GIT_WORK_TREE': workdir},
450 def merge(reference='@{upstream}'):
452 Merge changes from 'reference' into HEAD and load the result into notmuch.
454 The default reference is '@{upstream}'.
457 with _tempfile.TemporaryDirectory(prefix='nmbug-merge.') as workdir:
460 ['merge', reference]]:
463 additional_env={'GIT_WORK_TREE': workdir},
470 A simple wrapper for 'git log'.
472 After running 'nmbug fetch', you can inspect the changes with
473 'nmbug log HEAD..@{upstream}'.
475 # we don't want output trapping here, because we want the pager.
476 args = ['log', '--name-status'] + list(args)
477 with _git(args=args, expect=(0, 1, -13)) as p:
481 def push(repository=None, refspecs=None):
482 "Push the local nmbug Git state to a remote repository."
483 if refspecs and not repository:
484 repository = _get_remote()
487 args.append(repository)
489 args.extend(refspecs)
490 _git(args=args, wait=True)
495 Show pending updates in notmuch or git repo.
497 Prints lines of the form
501 where n is a single character representing notmuch database status
505 Tag is present in notmuch database, but not committed to nmbug
506 (equivalently, tag has been deleted in nmbug repo, e.g. by a
507 pull, but not restored to notmuch database).
511 Tag is present in nmbug repo, but not restored to notmuch
512 database (equivalently, tag has been deleted in notmuch).
516 Message is unknown (missing from local notmuch database).
518 The second character (if present) represents a difference between
519 local and upstream branches. Typically 'nmbug fetch' needs to be
524 Tag is present in upstream, but not in the local Git branch.
528 Tag is present in local Git branch, but not upstream.
530 status = get_status()
531 # 'output' is a nested defaultdict for message status:
532 # * The outer dict is keyed by message id.
533 # * The inner dict is keyed by tag name.
534 # * The inner dict values are status strings (' a', 'Dd', ...).
535 output = _collections.defaultdict(
536 lambda : _collections.defaultdict(lambda : ' '))
537 for id, tags in status['added'].items():
539 output[id][tag] = 'A'
540 for id, tags in status['deleted'].items():
542 output[id][tag] = 'D'
543 for id, tags in status['missing'].items():
545 output[id][tag] = 'U'
547 for id, tag in _diff_refs(filter='A'):
548 output[id][tag] += 'a'
549 for id, tag in _diff_refs(filter='D'):
550 output[id][tag] += 'd'
551 for id, tag_status in sorted(output.items()):
552 for tag, status in sorted(tag_status.items()):
553 print('{status}\t{id}\t{tag}'.format(
554 status=status, id=id, tag=tag))
557 def _is_unmerged(ref='@{upstream}'):
559 (status, fetch_head, stderr) = _git(
560 args=['rev-parse', ref],
561 stdout=_subprocess.PIPE, stderr=_subprocess.PIPE, wait=True)
562 except SubprocessError as e:
563 if 'No upstream configured' in e.stderr:
566 (status, base, stderr) = _git(
567 args=['merge-base', 'HEAD', ref],
568 stdout=_subprocess.PIPE, wait=True)
569 return base != fetch_head
577 index = _index_tags()
578 maybe_deleted = _diff_index(index=index, filter='D')
579 for id, tags in maybe_deleted.items():
580 (_, stdout, stderr) = _spawn(
581 args=['notmuch', 'search', '--output=files', 'id:{0}'.format(id)],
582 stdout=_subprocess.PIPE,
585 status['deleted'][id] = tags
587 status['missing'][id] = tags
588 status['added'] = _diff_index(index=index, filter='A')
594 "Write notmuch tags to the nmbug.index."
595 path = _os.path.join(NMBGIT, 'nmbug.index')
596 query = ' '.join('tag:"{tag}"'.format(tag=tag) for tag in get_tags())
597 prefix = '+{0}'.format(_ENCODED_TAG_PREFIX)
599 args=['read-tree', '--empty'],
600 additional_env={'GIT_INDEX_FILE': path}, wait=True)
602 args=['notmuch', 'dump', '--format=batch-tag', '--', query],
603 stdout=_subprocess.PIPE) as notmuch:
605 args=['update-index', '--index-info'],
606 stdin=_subprocess.PIPE,
607 additional_env={'GIT_INDEX_FILE': path}) as git:
608 for line in notmuch.stdout:
609 (tags_string, id) = [_.strip() for _ in line.split(' -- id:')]
611 _unquote(tag[len(prefix):])
612 for tag in tags_string.split()
613 if tag.startswith(prefix)]
614 id = _xapian_unquote(string=id)
615 for line in _index_tags_for_message(
616 id=id, status='A', tags=tags):
617 git.stdin.write(line)
621 def _index_tags_for_message(id, status, tags):
623 Update the Git index to either create or delete an empty file.
625 Neither 'id' nor the tags in 'tags' should be encoded/escaped.
632 hash = '0000000000000000000000000000000000000000'
635 path = 'tags/{id}/{tag}'.format(
636 id=_hex_quote(string=id), tag=_hex_quote(string=tag))
637 yield '{mode} {hash}\t{path}\n'.format(mode=mode, hash=hash, path=path)
640 def _diff_index(index, filter):
642 Get an {id: {tag, ...}} dict for a given filter.
644 For example, use 'A' to find added tags, and 'D' to find deleted tags.
646 s = _collections.defaultdict(set)
649 'diff-index', '--cached', '--diff-filter', filter,
650 '--name-only', 'HEAD'],
651 additional_env={'GIT_INDEX_FILE': index},
652 stdout=_subprocess.PIPE) as p:
653 # Once we drop Python < 3.3, we can use 'yield from' here
654 for id, tag in _unpack_diff_lines(stream=p.stdout):
659 def _diff_refs(filter, a='HEAD', b='@{upstream}'):
661 args=['diff', '--diff-filter', filter, '--name-only', a, b],
662 stdout=_subprocess.PIPE) as p:
663 # Once we drop Python < 3.3, we can use 'yield from' here
664 for id, tag in _unpack_diff_lines(stream=p.stdout):
668 def _unpack_diff_lines(stream):
669 "Iterate through (id, tag) tuples in a diff stream."
671 match = _TAG_FILE_REGEX.match(line.strip())
674 'Invalid line in diff: {!r}'.format(line.strip()))
675 id = _unquote(match.group('id'))
676 tag = _unquote(match.group('tag'))
680 if __name__ == '__main__':
683 parser = argparse.ArgumentParser(
684 description=__doc__.strip(),
685 formatter_class=argparse.RawDescriptionHelpFormatter)
687 '-v', '--version', action='version',
688 version='%(prog)s {}'.format(__version__))
691 choices=['critical', 'error', 'warning', 'info', 'debug'],
692 help='Log verbosity. Defaults to {!r}.'.format(
693 _logging.getLevelName(_LOG.level).lower()))
695 subparsers = parser.add_subparsers(
698 'For help on a particular command, run: '
699 "'%(prog)s ... <command> --help'."))
712 func = locals()[command]
713 doc = _textwrap.dedent(func.__doc__).strip().replace('%', '%%')
714 subparser = subparsers.add_parser(
716 help=doc.splitlines()[0],
718 formatter_class=argparse.RawDescriptionHelpFormatter)
719 subparser.set_defaults(func=func)
720 if command == 'archive':
721 subparser.add_argument(
722 'treeish', metavar='TREE-ISH', nargs='?', default='HEAD',
724 'The tree or commit to produce an archive for. Defaults '
726 subparser.add_argument(
727 'args', metavar='ARG', nargs='*',
729 "Argument passed through to 'git archive'. Set anything "
730 'before <tree-ish>, see git-archive(1) for details.'))
731 elif command == 'clone':
732 subparser.add_argument(
735 'The (possibly remote) repository to clone from. See the '
736 'URLS section of git-clone(1) for more information on '
737 'specifying repositories.'))
738 elif command == 'commit':
739 subparser.add_argument(
740 'message', metavar='MESSAGE', default='', nargs='?',
741 help='Text for the commit message.')
742 elif command == 'fetch':
743 subparser.add_argument(
744 'remote', metavar='REMOTE', nargs='?',
746 'Override the default configured in branch.<name>.remote '
747 'to fetch from a particular remote repository (e.g. '
749 elif command == 'log':
750 subparser.add_argument(
751 'args', metavar='ARG', nargs='*',
752 help="Additional argument passed through to 'git log'.")
753 elif command == 'merge':
754 subparser.add_argument(
755 'reference', metavar='REFERENCE', default='@{upstream}',
758 'Reference, usually other branch heads, to merge into '
759 "our branch. Defaults to '@{upstream}'."))
760 elif command == 'pull':
761 subparser.add_argument(
762 'repository', metavar='REPOSITORY', default=None, nargs='?',
764 'The "remote" repository that is the source of the pull. '
765 'This parameter can be either a URL (see the section GIT '
766 'URLS in git-pull(1)) or the name of a remote (see the '
767 'section REMOTES in git-pull(1)).'))
768 subparser.add_argument(
769 'refspecs', metavar='REFSPEC', default=None, nargs='*',
771 'Refspec (usually a branch name) to fetch and merge. See '
772 'the <refspec> entry in the OPTIONS section of '
773 'git-pull(1) for other possibilities.'))
774 elif command == 'push':
775 subparser.add_argument(
776 'repository', metavar='REPOSITORY', default=None, nargs='?',
778 'The "remote" repository that is the destination of the '
779 'push. This parameter can be either a URL (see the '
780 'section GIT URLS in git-push(1)) or the name of a remote '
781 '(see the section REMOTES in git-push(1)).'))
782 subparser.add_argument(
783 'refspecs', metavar='REFSPEC', default=None, nargs='*',
785 'Refspec (usually a branch name) to push. See '
786 'the <refspec> entry in the OPTIONS section of '
787 'git-push(1) for other possibilities.'))
789 args = parser.parse_args()
792 level = getattr(_logging, args.log_level.upper())
795 if not getattr(args, 'func', None):
799 (arg_names, varargs, varkw) = _inspect.getargs(args.func.__code__)
800 kwargs = {key: getattr(args, key) for key in arg_names if key in args}
803 except SubprocessError as e:
804 if _LOG.level == _logging.DEBUG:
805 raise # don't mask the traceback