Merge branch 'master' into release/beta

Conflicts:
	mailpile/defaults.py

Dockerfile

@@ -7,9 +7,11 @@ WORKDIR /Mailpile
 ADD . /Mailpile
 
 RUN	groupadd -r mailpile \
-&&  mkdir /mailpile-data \
+&&  mkdir -p /mailpile-data/.gnupg \
 &&	useradd -r -d /mailpile-data -g mailpile mailpile
 
+RUN touch /mailpile-data/.gnupg/docker_placeholder
+
 RUN chown -R mailpile:mailpile /Mailpile
 RUN chown -R mailpile:mailpile /mailpile-data
 

mailpile/commands.py

@@ -21,6 +21,7 @@
 import mailpile.util
 import mailpile.ui
 import mailpile.postinglist
+import mailpile.security as security
 from mailpile.crypto.gpgi import GnuPG
 from mailpile.eventlog import Event
 from mailpile.i18n import gettext as _
@@ -58,6 +59,7 @@ class Command(object):
     SPLIT_ARG = True  # Uses shlex by default
     RAISES = (UsageError, UrlRedirectException)
     WITH_CONTEXT = ()
+    COMMAND_SECURITY = None
 
     # Event logging settings
     LOG_NOTHING = False
@@ -620,6 +622,11 @@ def streetcar():
             return self._run_sync(True, *args, **kwargs)
 
     def run(self, *args, **kwargs):
+        if self.COMMAND_SECURITY is not None:
+            forbidden = security.forbid_command(self)
+            if forbidden:
+                return self._error(forbidden)
+
         with MultiContext(self.WITH_CONTEXT):
             if self.IS_USER_ACTIVITY:
                 try:
@@ -1167,6 +1174,10 @@ def command(self, slowly=False, cron=False):
         if not slowly:
             mailpile.util.LAST_USER_ACTIVITY = 0
 
+        # Cron always runs the rescan command, no matter what else
+        if cron:
+            self._run_rescan_command(session)
+
         if args and args[0].lower().startswith('vcards'):
             return self._success(_('Rescanned vcards'),
                                  result=self._rescan_vcards(session, args[0]))
@@ -1707,6 +1718,7 @@ class ListDir(Command):
     ORDER = ('Internals', 5)
     CONFIG_REQUIRED = False
     IS_USER_ACTIVITY = True
+    COMMAND_SECURITY = security.CC_ACCESS_FILESYSTEM
 
     class CommandResult(Command.CommandResult):
         def as_text(self):
@@ -1735,9 +1747,6 @@ def command(self, args=None):
         if '_method' in self.data:
             args = ['/' + '/'.join(args)]
 
-        if self.session.config.sys.lockdown:
-            return self._error(_('In lockdown, doing nothing.'))
-
         if not args:
             args = ['.']
 
@@ -1752,6 +1761,7 @@ def ls(p):
                     if '-a' in flags or f.raw_fp[:1] != '.']
 
         file_list = []
+        errors = 0
         for path in args:
             try:
                 path = os.path.expanduser(path.encode('utf-8'))
@@ -1765,9 +1775,12 @@ def ls(p):
                             file_list.append(lsf(p))
             except (socket.error, socket.gaierror), e:
                 return self._error(_('Network error: %s') % e)
-            except (OSError, IOError, UnicodeDecodeError, socket.error), e:
-                traceback.print_exc()
-                return self._error(_('Failed to list: %s') % e)
+            except (OSError, IOError, UnicodeDecodeError), e:
+                errors += 1
+
+        if errors and not file_list:
+            traceback.print_exc()
+            return self._error(_('Failed to list: %s') % e)
 
         id_src_map = self.session.config.find_mboxids_and_sources_by_path(
             *[unicode(f['path']) for f in file_list])
@@ -1802,14 +1815,11 @@ class ChangeDir(ListDir):
     ORDER = ('Internals', 5)
     CONFIG_REQUIRED = False
     IS_USER_ACTIVITY = True
+    COMMAND_SECURITY = security.CC_ACCESS_FILESYSTEM
 
     def command(self, args=None):
-        args = list((args is None) and self.args or args or [])
-
-        if self.session.config.sys.lockdown:
-            return self._error(_('In lockdown, doing nothing.'))
-
         try:
+            args = list((args is None) and self.args or args or [])
             os.chdir(os.path.expanduser(args.pop(0).encode('utf-8')))
             return ListDir.command(self, args=['.'])
         except (OSError, IOError, UnicodeEncodeError), e:
@@ -1822,6 +1832,7 @@ class CatFile(Command):
     ORDER = ('Internals', 5)
     CONFIG_REQUIRED = False
     IS_USER_ACTIVITY = True
+    COMMAND_SECURITY = security.CC_ACCESS_FILESYSTEM
 
     class CommandResult(Command.CommandResult):
         def as_text(self):
@@ -1833,10 +1844,6 @@ def as_text(self):
     def command(self, args=None):
         lines = []
         files = list(args or self.args)
-
-        if self.session.config.sys.lockdown:
-            return self._error(_('In lockdown, doing nothing.'))
-
         target = tfd = None
         if files and files[-1] and files[-1][:1] == '>':
             target = files.pop(-1)[1:]
@@ -1900,8 +1907,10 @@ def command(self):
             force = True
             arg = arg[8:]
 
-        if config.sys.lockdown and not force:
-            return self._error(_('In lockdown, doing nothing.'))
+        if not force:
+            fb = security.forbid_command(self, security.CC_CHANGE_CONFIG)
+            if fb:
+                return self._error(fb)
 
         if not config.loaded_config:
             self.session.ui.warning(_('WARNING: Any changes will '
@@ -1981,17 +1990,14 @@ class ConfigAdd(Command):
         'section.variable': 'value|json-string',
     }
     IS_USER_ACTIVITY = True
+    COMMAND_SECURITY = security.CC_CHANGE_CONFIG
 
     def command(self):
         from mailpile.httpd import BLOCK_HTTPD_LOCK, Idle_HTTPD
-
         config = self.session.config
         args = list(self.args)
         ops = []
 
-        if config.sys.lockdown:
-            return self._error(_('In lockdown, doing nothing.'))
-
         for var in self.data.keys():
             parts = ('.' in var) and var.split('.') or var.split('/')
             if parts[0] in config.rules:
@@ -2034,15 +2040,12 @@ class ConfigUnset(Command):
         'var': 'section.variables'
     }
     IS_USER_ACTIVITY = True
+    COMMAND_SECURITY = security.CC_CHANGE_CONFIG
 
     def command(self):
         from mailpile.httpd import BLOCK_HTTPD_LOCK, Idle_HTTPD
-
         session, config = self.session, self.session.config
 
-        if config.sys.lockdown:
-            return self._error(_('In lockdown, doing nothing.'))
-
         def unset(cfg, key):
             if isinstance(cfg[key], dict):
                 if '_any' in cfg[key].rules:
@@ -2129,7 +2132,7 @@ def command(self):
         list_all = not self.data.get('short', ['-short' in args])[0]
         sanitize = not self.data.get('secrets', ['-secrets' in args])[0]
 
-        if config.sys.lockdown:
+        if security.forbid_command(self, security.CC_LIST_PRIVATE_DATA):
             sanitize = True
 
         # FIXME: Shouldn't we suppress critical variables as well?
@@ -2196,6 +2199,7 @@ class ConfigureMailboxes(Command):
         'tag_visible': 'Make new tags visible in sidebar',
         'local_copy': 'Make local copy of mail'
     }
+    COMMAND_SECURITY = security.CC_CHANGE_CONFIG
 
     MAX_PATHS = 50000
 
@@ -2210,9 +2214,6 @@ def command(self):
         paths = list(self.args)
         recurse = False
 
-        if config.sys.lockdown:
-            return self._error(_('In lockdown, doing nothing.'))
-
         # Which tags do we want to apply?
         apply_tags = self.data.get('apply_tags', [])
         atis = [i for i, p in enumerate(paths)
@@ -2420,11 +2421,9 @@ class Pipe(Command):
     ORDER = ('Internals', 5)
     CONFIG_REQUIRED = False
     IS_USER_ACTIVITY = True
+    COMMAND_SECURITY = security.CC_ACCESS_FILESYSTEM
 
     def command(self):
-        if self.session.config.sys.lockdown:
-            return self._error(_('In lockdown, doing nothing.'))
-
         if '--' in self.args:
             dashdash = self.args.index('--')
             target = self.args[0:dashdash]
@@ -2489,11 +2488,9 @@ class Quit(Command):
     ORDER = ("Internals", 2)
     CONFIG_REQUIRED = False
     RAISES = (KeyboardInterrupt,)
+    COMMAND_SECURITY = security.CC_QUIT
 
     def command(self):
-        if self.session.config.sys.lockdown:
-            return self._error(_('In lockdown, doing nothing.'))
-
         mailpile.util.QUITTING = True
         self._background_save(index=True, config=True, wait=True)
         try:
@@ -2511,11 +2508,9 @@ def exiter():
 class TrustingQQQ(Command):
     """Allow anybody to quit the app"""
     SYNOPSIS = (None, "trustingqqq", None, None)
+    COMMAND_SECURITY = security.CC_QUIT
 
     def command(self):
-        if self.session.config.sys.lockdown:
-            return self._error(_('In lockdown, doing nothing.'))
-
         # FIXME: This is a hack to allow Windows deployments to shut
         #        down cleanly. Eventually this will take an argument
         #        specifying a random token that the launcher chooses.
@@ -2533,11 +2528,9 @@ class Abort(Command):
     HTTP_QUERY_VARS = {
         'no_save': 'Do not try to save state'
     }
+    COMMAND_SECURITY = security.CC_QUIT
 
     def command(self):
-        if self.session.config.sys.lockdown:
-            return self._error(_('In lockdown, doing nothing.'))
-
         mailpile.util.QUITTING = True
         if 'no_save' not in self.data:
             self._background_save(index=True, config=True, wait=True,

mailpile/contrib/experiments/experiments.py

@@ -32,7 +32,7 @@ def _CopyAsMultipart(msg, callback, cleaner):
             del msg[hdr]
         elif hdrl == 'mime-version':
             del msg[hdrl]
-    callback('headers', m, None)
+    callback('headers', m, msg)
 
     def att(part):
         if hasattr(part, 'signature_info'):
@@ -45,14 +45,17 @@ def att(part):
             att(part)
     else:
         att(msg)
-    callback('payload', m, None)
+    callback('payload', m, msg)
 
     return m
 
 
 class EmailCryptoTxf(EmailTransform):
     """This is a set of email encryption experiments"""
 
+    # Different methods. True is for backwards compatibility, => 'attach'
+    TRANSFORM_STYLES = ('true', 'attach', 'mime')
+
     # Header protection ignores these...
     DKG_IGNORED_HEADERS = ['mime-version', 'content-type']
 
@@ -66,7 +69,8 @@ class EmailCryptoTxf(EmailTransform):
     }
     DKG_STRIPPED_HEADERS = ['openpgp']
 
-    def DkgHeaderTransformOutgoing(self, msg, crypto_policy, cleaner):
+    def DkgHeaderTransformOutgoing(self, msg, crypto_policy, cleaner,
+                                   transform_style):
         visible, invisible = Message(), Message()
 
         if 'encrypt' in crypto_policy:
@@ -92,25 +96,47 @@ def DkgHeaderTransformOutgoing(self, msg, crypto_policy, cleaner):
         else:
             return msg
 
-        def copy_callback(stage, msg, part):
+        def copy_callback_attach(stage, msg, part):
             if stage == 'headers' and visible.keys():
                 part = _AddCryptoState(MIMEText(visible.as_string(),
                                                 'rfc822-headers'))
-                part.set_param('memoryhole', 'v1,%s' % msg['Message-ID'])
+                part.set_param('protected-headers',
+                               'v1,%s' % msg['Message-ID'])
                 part['Content-Disposition'] = 'inline'
                 del part['MIME-Version']
                 msg.attach(part)
+                return part
 
             elif stage == 'payload' and invisible.keys():
                 part = _AddCryptoState(MIMEText(invisible.as_string(),
                                                 'rfc822-headers'))
-                part.set_param('memoryhole', 'v1,%s' % msg['Message-ID'])
+                part.set_param('protected-headers',
+                               'v1,%s' % msg['Message-ID'])
                 part['Content-Disposition'
                      ] = 'attachment; filename=Secure_Headers.txt'
                 del part['MIME-Version']
                 msg.attach(part)
+                return part
+
+            return None
+
+        def copy_callback_mime(stage, msg, part):
+            if stage == 'headers':
+                new_part = copy_callback_attach(stage, msg, part)
+                if new_part:
+                    for key in invisible.keys():
+                        new_part[key] = invisible[key]
+                        del invisible[key]
 
-        return _CopyAsMultipart(msg, copy_callback, cleaner)
+            elif stage in ('payload', 'part') and invisible.keys():
+                for key in invisible.keys():
+                    part[key] = invisible[key]
+                    del invisible[key]
+
+        if transform_style == 'mime':
+            return _CopyAsMultipart(msg, copy_callback_mime, cleaner)
+        else:
+            return _CopyAsMultipart(msg, copy_callback_attach, cleaner)
 
     def DkgHeaderTransformIncoming(self, msg):
         # FIXME: Parse incoming message/rfc822-headers parts, migrate
@@ -128,8 +154,10 @@ def TransformOutgoing(self, sender, rcpt, msg,
         txf_continue = True
         txf_matched = False
 
-        if self.config.prefs.experiment_dkg_hdrs is True:
-            msg = self.DkgHeaderTransformOutgoing(msg, crypto_policy, cleaner)
+        txf_style = self.config.prefs.get('experiment_dkg_hdrs', 'off').lower()
+        if txf_style in self.TRANSFORM_STYLES:
+            msg = self.DkgHeaderTransformOutgoing(msg, crypto_policy, cleaner,
+                                                  txf_style)
             txf_matched = True
 
         return sender, rcpt, msg, txf_matched, txf_continue
@@ -156,7 +184,7 @@ def paragraph_id_extractor(index, msg, ctype, textpart):
     try:
         if not ctype == 'text/plain':
             return kws
-        if not index.config.prefs.experiment_para_kws:
+        if not index.config.prefs.get('experiment_para_kws'):
             return kws
 
         para = {'text': '', 'qlevel': 0}

mailpile/contrib/experiments/manifest.json

@@ -16,8 +16,8 @@
     "config": {
         "variables": {
             "prefs": {
-                "experiment_dkg_hdrs": ["Enable DKG-style encrypted headers",
-                                        "bool", "False"],
+                "experiment_dkg_hdrs": ["DKG-style encrypted headers (mime|attach|OFF)",
+                                        "str", ""],
                 "experiment_para_kws": ["Make paragraphs searchable by hash",
                                         "bool", "False"]
             }