Fix for broken salt cmd return codes - issue #7013

salt/cli/__init__.py

@@ -24,7 +24,8 @@
 from salt.exceptions import (
     SaltInvocationError,
     SaltClientError,
-    EauthAuthenticationError
+    EauthAuthenticationError,
+    SaltSystemExit
 )
 
 
@@ -130,6 +131,7 @@ def run(self):
                 jid = local.cmd_async(**kwargs)
                 print('Executed command with job ID: {0}'.format(jid))
                 return
+            retcodes = []
             try:
                 # local will be None when there was an error
                 if local:
@@ -143,21 +145,33 @@ def run(self):
                         if self.options.verbose:
                             kwargs['verbose'] = True
                         full_ret = local.cmd_full_return(**kwargs)
-                        ret, out = self._format_ret(full_ret)
+                        ret, out, retcode = self._format_ret(full_ret)
                         self._output_ret(ret, out)
                     elif self.config['fun'] == 'sys.doc':
                         ret = {}
                         out = ''
                         for full_ret in local.cmd_cli(**kwargs):
-                            ret_, out = self._format_ret(full_ret)
+                            ret_, out, retcode = self._format_ret(full_ret)
                             ret.update(ret_)
                         self._output_ret(ret, out)
                     else:
                         if self.options.verbose:
                             kwargs['verbose'] = True
                         for full_ret in cmd_func(**kwargs):
-                            ret, out = self._format_ret(full_ret)
+                            ret, out, retcode = self._format_ret(full_ret)
+                            retcodes.append(retcode)
                             self._output_ret(ret, out)
+
+                    # NOTE: Return code is set here based on if all minions
+                    # returned 'ok' with a retcode of 0.
+                    # This is the final point before the 'salt' cmd returns,
+                    # which is why we set the retcode here.
+                    if retcodes.count(0) < len(retcodes):
+                        err = 'All Minions did not return a retcode of 0. One or more minions had a problem'
+                        # NOTE: This could probably be made more informative.
+                        # I chose 11 since its not in use.
+                        raise SaltSystemExit(code=11, msg=err)
+
             except (SaltInvocationError, EauthAuthenticationError) as exc:
                 ret = str(exc)
                 out = ''
@@ -182,11 +196,14 @@ def _format_ret(self, full_ret):
         '''
         ret = {}
         out = ''
+        retcode = 0
         for key, data in full_ret.items():
             ret[key] = data['ret']
             if 'out' in data:
                 out = data['out']
-        return ret, out
+            if 'retcode' in data:
+                retcode = data['retcode']
+        return ret, out, retcode
 
     def _print_docs(self, ret):
         '''

salt/client/__init__.py

@@ -1100,6 +1100,7 @@ def get_cli_static_event_returns(
         '''
         Get the returns for the command line interface via the event system
         '''
+        log.debug("entered - function get_cli_static_event_returns()")
         minions = set(minions)
         if verbose:
             msg = 'Executing job with jid {0}'.format(jid)
@@ -1174,6 +1175,7 @@ def get_cli_event_returns(
         '''
         Get the returns for the command line interface via the event system
         '''
+        log.debug("func get_cli_event_returns()")
         if not isinstance(minions, set):
             if isinstance(minions, basestring):
                 minions = set([minions])
@@ -1205,6 +1207,11 @@ def get_cli_event_returns(
             # Wait 0 == forever, use a minimum of 1s
             wait = max(1, time_left)
             raw = self.event.get_event(wait, jid)
+            log.debug(
+                "get_cli_event_returns()" +
+                " called self.event.get_event()" +
+                " and recieved : raw = " + str(raw)
+            )
             if raw is not None:
                 if 'minions' in raw.get('data', {}):
                     minions.update(raw['data']['minions'])
@@ -1214,10 +1221,16 @@ def get_cli_event_returns(
                     continue
                 if 'return' not in raw:
                     continue
+
                 found.add(raw.get('id'))
                 ret = {raw['id']: {'ret': raw['return']}}
                 if 'out' in raw:
                     ret[raw['id']]['out'] = raw['out']
+                if 'retcode' in raw:
+                    ret[raw['id']]['retcode'] = raw['retcode']
+                log.trace("raw = " + str(raw))
+                log.trace("ret = " + str(ret))
+                log.trace("yeilding 'ret'")
                 yield ret
                 if len(found.intersection(minions)) >= len(minions):
                     # All minions have returned, break out of the loop
@@ -1279,6 +1292,7 @@ def get_event_iter_returns(self, jid, minions, timeout=None):
         Gather the return data from the event system, break hard when timeout
         is reached.
         '''
+        log.debug("entered - function get_event_iter_returns()")
         if timeout is None:
             timeout = self.opts['timeout']
         jid_dir = salt.utils.jid_dir(jid,

salt/minion.py

@@ -1026,6 +1026,7 @@ def _return_pub(self, ret, ret_cmd='_return'):
             if not os.path.isdir(jdir):
                 os.makedirs(jdir)
             salt.utils.fopen(fn_, 'w+b').write(self.serial.dumps(ret))
+        log.debug('ret_val = ' + str(ret_val))
         return ret_val
 
     def _state_run(self):

salt/output/__init__.py

@@ -39,6 +39,7 @@ def display_output(data, out, opts=None):
         display_data = get_printout('nested', opts)(data).rstrip()
 
     output_filename = opts.get('output_file', None)
+    log.debug("data = " + str(data))
     try:
         if output_filename is not None:
             with salt.utils.fopen(output_filename, 'a') as ofh:

salt/states/test.py

@@ -0,0 +1,235 @@
+# -*- coding: utf-8 -*-
+'''
+Test States
+==================
+
+Provide test case states that enable easy testing of things to do with
+ state calls, e.g. running, calling, logging, output filtering etc.
+
+.. code-block:: yaml
+
+    always-passes:
+      test.succeed_without_changes:
+        - name: foo
+
+    always-fails:
+      test.fail_without_changes:
+        - name: foo
+
+    always-changes-and-succeeds:
+      test.succeed_with_changes:
+        - name: foo
+
+    always-changes-and-fails:
+      test.fail_with_changes:
+        - name: foo
+
+    my-custom-combo:
+      test.configurable_test_state:
+        - name: foo
+        - changes: True
+        - result: False
+        - comment: bar.baz
+
+'''
+
+# Import Python libs
+import logging
+import random
+from salt.exceptions import SaltInvocationError
+
+log = logging.getLogger(__name__)
+
+
+def succeed_without_changes(name):
+    '''
+    Returns successful.
+
+    name
+        A unique string.
+    '''
+    ret = {
+        'name': name,
+        'changes': {},
+        'result': True,
+        'comment': 'This is just a test, nothing actually happened'
+    }
+    if __opts__['test']:
+        ret['result'] = None
+        ret['comment'] = (
+            'Yo dawg I heard you like tests,'
+            ' so I put tests in your tests,'
+            ' so you can test while you test.'
+        )
+    return ret
+
+
+def fail_without_changes(name):
+    '''
+    Returns failure.
+
+    name:
+        A unique string.
+    '''
+    ret = {
+        'name': name,
+        'changes': {},
+        'result': False,
+        'comment': 'This is just a test, nothing actually happened'
+    }
+
+    if __opts__['test']:
+        ret['result'] = None
+        ret['comment'] = (
+            'Yo dawg I heard you like tests,'
+            ' so I put tests in your tests,'
+            ' so you can test while you test.'
+        )
+
+    return ret
+
+
+def succeed_with_changes(name):
+    '''
+    Returns successful and changes is not empty
+
+    name:
+        A unique string.
+    '''
+    ret = {
+        'name': name,
+        'changes': {},
+        'result': True,
+        'comment': 'This is just a test, nothing actually happened'
+    }
+
+    # Following the docs as written here
+    # http://docs.saltstack.com/ref/states/writing.html#return-data
+    ret['changes'] = {
+        'testing': {
+            'old': 'Nothing has changed yet',
+            'new': 'Were pretending really hard that we changed something'
+        }
+    }
+
+    if __opts__['test']:
+        ret['result'] = None
+        ret['comment'] = (
+            'Yo dawg I heard you like tests,'
+            ' so I put tests in your tests,'
+            ' so you can test while you test.'
+        )
+
+    return ret
+
+
+def fail_with_changes(name):
+    '''
+    Returns failure and changes is not empty.
+
+    name:
+        A unique string.
+    '''
+    ret = {
+        'name': name,
+        'changes': {},
+        'result': False,
+        'comment': 'This is just a test, nothing actually happened'
+    }
+
+    # Following the docs as written here
+    # http://docs.saltstack.com/ref/states/writing.html#return-data
+    ret['changes'] = {
+        'testing': {
+            'old': 'Nothing has changed yet',
+            'new': 'Were pretending really hard that we changed something'
+        }
+    }
+
+    if __opts__['test']:
+        ret['result'] = None
+        ret['comment'] = (
+            'Yo dawg I heard you like tests,'
+            ' so I put tests in your tests,'
+            ' so you can test while you test.'
+        )
+
+    return ret
+
+
+def configurable_test_state(name, changes=True, result=True, comment=''):
+    '''
+    A configurable test state which determines its output based on the inputs.
+
+    name:
+        A unique string.
+    changes:
+        Do we return anything in the changes field?
+        Accepts True, False, and 'Random'
+        Default is True
+    result:
+        Do we return sucessfuly or not?
+        Accepts True, False, and 'Random'
+        Default is True
+    comment:
+        String to fill the comment field with.
+        Default is ''
+    '''
+    ret = {
+        'name': name,
+        'changes': {},
+        'result': False,
+        'comment': comment
+    }
+
+    # E8712 is disabled because this code is a LOT cleaner if we allow it.
+    if changes == "Random":
+        if random.choice([True, False]):
+            # Following the docs as written here
+            # http://docs.saltstack.com/ref/states/writing.html#return-data
+            ret['changes'] = {
+            'testing': {
+                'old': 'Nothing has changed yet',
+                'new': 'Were pretending really hard that we changed something'
+                }
+            }
+    elif changes == True:  # pylint: disable=E8712
+        # If changes is True we place our dummy change dictionary into it.
+        # Following the docs as written here
+        # http://docs.saltstack.com/ref/states/writing.html#return-data
+        ret['changes'] = {
+        'testing': {
+            'old': 'Nothing has changed yet',
+            'new': 'Were pretending really hard that we changed something'
+            }
+        }
+    elif changes == False:  # pylint: disable=E8712
+        ret['changes'] = {}
+    else:
+        err = ('You have specified the state option \'Changes\' with'
+            ' invalid arguments. It must be either '
+            ' \'True\', \'False\', or \'Random\'')
+        raise SaltInvocationError(err)
+
+    if result == 'Random':
+        # since result is a boolean, if its random we just set it here,
+        ret['result'] = random.choice([True, False])
+    elif result == True:  # pylint: disable=E8712
+        ret['result'] = True
+    elif result == False:  # pylint: disable=E8712
+        ret['result'] = False
+    else:
+        err = ('You have specified the state option \'Result\' with'
+            ' invalid arguments. It must be either '
+            ' \'True\', \'False\', or \'Random\'')
+        raise SaltInvocationError(err)
+
+    if __opts__['test']:
+        ret['result'] = None
+        ret['comment'] = (
+            'Yo dawg I heard you like tests,'
+            ' so I put tests in your tests,'
+            ' so you can test while you test.'
+        )
+
+    return ret

salt/templates/__init__.py

@@ -0,0 +1 @@
+# -*- coding: utf-8 -*-