Merge branch 'master' into dnstrails

Author: sebdraven

.travis.yml

@@ -6,11 +6,11 @@ services:
 cache: pip
 
 python:
-    - "3.4"
     - "3.5"
     - "3.5-dev"
     - "3.6"
     - "3.6-dev"
+    - "3.7-dev"
 
 install:
     - pip install -U nose codecov pytest

README.md

@@ -38,8 +38,10 @@ For more information: [Extending MISP with Python modules](https://www.circl.lu/
 * [rbl](misp_modules/modules/expansion/rbl.py) - a module to get RBL (Real-Time Blackhost List) values from an attribute.
 * [reversedns](misp_modules/modules/expansion/reversedns.py) - Simple Reverse DNS expansion service to resolve reverse DNS from MISP attributes.
 * [shodan](misp_modules/modules/expansion/shodan.py) - a minimal [shodan](https://www.shodan.io/) expansion module.
+* [Sigma queries](misp_modules/modules/expansion/sigma_queries.py) - Experimental expansion module querying a sigma rule to convert it into all the available SIEM signatures.
 * [Sigma syntax validator](misp_modules/modules/expansion/sigma_syntax_validator.py) - Sigma syntax validator.
 * [sourcecache](misp_modules/modules/expansion/sourcecache.py) - a module to cache a specific link from a MISP instance.
+* [STIX2 pattern syntax validator](misp_modules/modules/expansion/stix2_pattern_syntax_validator.py) - a module to check a STIX2 pattern syntax.
 * [ThreatCrowd](misp_modules/modules/expansion/threatcrowd.py) - an expansion module for [ThreatCrowd](https://www.threatcrowd.org/).
 * [threatminer](misp_modules/modules/expansion/threatminer.py) - an expansion module to expand from [ThreatMiner](https://www.threatminer.org/).
 * [virustotal](misp_modules/modules/expansion/virustotal.py) - an expansion module to pull known resolutions and malware samples related with an IP/Domain from virusTotal (this modules require a VirusTotal private API key)
@@ -380,7 +382,7 @@ Recommended     Plugin.Import_ocr_enabled       true   Enable or disable the ocr
 In this same menu set any other plugin settings that are required for testing.
 
 ## Install misp-module on an offline instance.
-First, you need to grab all necessery packages for example like this : 
+First, you need to grab all necessary packages for example like this :
 
 Use pip wheel to create an archive
 ~~~

REQUIREMENTS

@@ -15,10 +15,12 @@ git+https://github.com/sebdraven/pyonyphe#egg=pyonyphe
 git+https://github.com/sebdraven/pydnstrails#egg=pydnstrails
 pillow
 pytesseract
+wand
 SPARQLWrapper
 domaintools_api
 pygeoip
 bs4
 oauth2
 yara
 sigmatools
+stix2-patterns

misp_modules/__init__.py

@@ -193,7 +193,7 @@ def post(self):
             if dict_payload.get('timeout'):
                 timeout = datetime.timedelta(seconds=int(dict_payload.get('timeout')))
             else:
-                timeout = datetime.timedelta(seconds=30)
+                timeout = datetime.timedelta(seconds=300)
             response = yield tornado.gen.with_timeout(timeout, self.run_request(jsonpayload))
             self.write(response)
         except tornado.gen.TimeoutError:

misp_modules/modules/expansion/__init__.py

@@ -1,3 +1,3 @@
 from . import _vmray
 
-__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns', 'domaintools', 'eupi', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon', 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange', 'sigma_syntax_validator']
+__all__ = ['vmray_submit', 'asn_history', 'circl_passivedns', 'circl_passivessl', 'countrycode', 'cve', 'dns', 'domaintools', 'eupi', 'farsight_passivedns', 'ipasn', 'passivetotal', 'sourcecache', 'virustotal', 'whois', 'shodan', 'reversedns', 'geoip_country', 'wiki', 'iprep', 'threatminer', 'otx', 'threatcrowd', 'vulndb', 'crowdstrike_falcon', 'yara_syntax_validator', 'hashdd', 'onyphe', 'onyphe_full', 'rbl', 'xforceexchange', 'sigma_syntax_validator', 'stix2_pattern_syntax_validator', 'sigma_queries']

modules/expansion/module.py.skeleton …les/modules/expansion/module.py.skeletonmodules/expansion/module.py.skeleton renamed to misp_modules/modules/expansion/module.py.skeleton modules/expansion/module.py.skeleton renamed to misp_modules/modules/expansion/module.py.skeleton

@@ -0,0 +1,50 @@
+import sys, os, io, json
+try:
+    from sigma.parser import SigmaCollectionParser
+    from sigma.config import SigmaConfiguration
+    from sigma.backends import getBackend, BackendOptions
+except ModuleNotFoundError:
+    print("sigma or yaml is missing, use 'pip3 install sigmatools' to install it.")
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['sigma'], 'output': ['text']}
+moduleinfo = {'version': '0.1', 'author': 'Christian Studer', 'module-type': ['expansion', 'hover'],
+              'description': 'An expansion hover module to display the result of sigma queries.'}
+moduleconfig = []
+sigma_targets = ('es-dsl', 'es-qs', 'graylog', 'kibana', 'xpack-watcher', 'logpoint', 'splunk', 'grep', 'wdatp', 'splunkxml', 'arcsight', 'qualys')
+
+def handler(q=False):
+    if q is False:
+        return False
+    request = json.loads(q)
+    if not request.get('sigma'):
+        misperrors['error'] = 'Sigma rule missing'
+        return misperrors
+    config = SigmaConfiguration()
+    backend_options = BackendOptions(None)
+    f = io.TextIOWrapper(io.BytesIO(request.get('sigma').encode()), encoding='utf-8')
+    parser = SigmaCollectionParser(f, config, None)
+    targets = []
+    old_stdout = sys.stdout
+    result = io.StringIO()
+    sys.stdout = result
+    for t in sigma_targets:
+        backend = getBackend(t)(config, backend_options, None)
+        try:
+            parser.generate(backend)
+            backend.finalize()
+            print("#NEXT")
+            targets.append(t)
+        except:
+            continue
+    sys.stdout = old_stdout
+    results = result.getvalue()[:-5].split('#NEXT')
+    d_result = {t: r.strip() for t,r in zip(targets, results)}
+    return {'results': [{'types': mispattributes['output'], 'values': d_result}]}
+
+def introspection():
+    return mispattributes
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo

misp_modules/modules/expansion/sigma_queries.py

@@ -0,0 +1,42 @@
+import json
+try:
+    from stix2patterns.validator import run_validator
+except ModuleNotFoundError:
+    print("stix2 patterns python library is missing, use 'pip3 install stix2-patterns' to install it.")
+
+misperrors = {'error': 'Error'}
+mispattributes = {'input': ['stix2-pattern'], 'output': ['text']}
+moduleinfo = {'version': '0.1', 'author': 'Christian Studer', 'module-type': ['expansion', 'hover'],
+              'description': 'An expansion hover module to perform a syntax check on stix2 patterns.'}
+moduleconfig = []
+
+def handler(q=False):
+    if q is False:
+        return False
+    request = json.loads(q)
+    if not request.get('stix2-pattern'):
+        misperrors['error'] = 'STIX2 pattern missing'
+        return misperrors
+    pattern = request.get('stix2-pattern')
+    syntax_errors = []
+    for p in pattern[2:-2].split(' AND '):
+        syntax_validator = run_validator("[{}]".format(p))
+        if syntax_validator:
+            for error in syntax_validator:
+                syntax_errors.append(error)
+    if syntax_errors:
+        s = 's' if len(syntax_errors) > 1 else ''
+        s_errors = ""
+        for error in  syntax_errors:
+            s_errors += "{}\n".format(error[6:])
+        result = "Syntax error{}: \n{}".format(s, s_errors[:-1])
+    else:
+        result = "Syntax valid"
+    return {'results': [{'types': mispattributes['output'], 'values': result}]}
+
+def introspection():
+    return mispattributes
+
+def version():
+    moduleinfo['config'] = moduleconfig
+    return moduleinfo

misp_modules/modules/expansion/stix2_pattern_syntax_validator.py

@@ -1 +1 @@
-__all__ = ['testexport','cef_export','liteexport','goamlexport','threat_connect_export','pdfexport','threatStream_misp_export']
+__all__ = ['cef_export','liteexport','goamlexport','threat_connect_export','pdfexport','threatStream_misp_export']

misp_modules/modules/export_mod/__init__.py

@@ -1,3 +1,3 @@
 from . import _vmray
 
-__all__ = ['vmray_import', 'testimport', 'ocr', 'cuckooimport', 'goamlimport', 'email_import', 'mispjson', 'openiocimport', 'threatanalyzer_import', 'csvimport']
+__all__ = ['vmray_import', 'ocr', 'cuckooimport', 'goamlimport', 'email_import', 'mispjson', 'openiocimport', 'threatanalyzer_import', 'csvimport']