maldet 0.2

Author: Katja Hahn

progs/maldet.jar

@@ -27,6 +27,7 @@ import scala.None
 import scala.None
 import scala.None
 import com.github.katjahahn.parser.PESignature
+import com.github.katjahahn.parser.FileFormatException
 
 //TODO implement new good file stats
 //TODO remove dependend anomalies from /data/stats file
@@ -71,11 +72,11 @@ object DetectionHeuristic {
   val threshold = 500
   lazy val probabilities = readProbabilities()
 
-  private val version = """version: 0.1
+  private val version = """version: 0.2
     |author: Katja Hahn
     |last update: 21.Jun 2014""".stripMargin
 
-  private val title = """MalDet v0.1
+  private val title = """MalDet v0.2
                         |-----------    
                     |Please note: 
                     |MalDet uses statistical information about file anomalies to assign a probability to a file for being malicious.
@@ -92,7 +93,7 @@ object DetectionHeuristic {
   private type OptionMap = scala.collection.mutable.Map[Symbol, String]
 
   def main(args: Array[String]): Unit = {
-    testHeuristics();
+    invokeCLI(args)
   }
 
   private def invokeCLI(args: Array[String]): Unit = {
@@ -165,7 +166,7 @@ object DetectionHeuristic {
     }
   }
   private def testHeuristics(): Unit = {
-    val folder = new File("/home/deque/portextestfiles/goodfiles")
+    val folder = new File("/home/deque/portextestfiles/badfiles")
     val threshholdA = 0.99
     val threshholdB = 0.80
     val threshholdC = 0.50
@@ -194,7 +195,8 @@ object DetectionHeuristic {
           println("malicious by threshhold 0.50: " + malcounterC + " ratio " + (malcounterC.toDouble / total.toDouble))
         }
       } catch {
-        case e: Exception => notLoaded += 1; System.err.println(e.getMessage);
+        case e: FileFormatException => notLoaded +=1; System.err.println("file is no PE file: " + file.getName());
+        case e: Exception => notLoaded += 1; e.printStackTrace();
       }
     }
     total -= notLoaded

src/main/java/com/github/katjahahn/tools/DetectionHeuristic.scala

@@ -1,40 +1,40 @@
-UNUSUAL_SEC_NAME;3589;16.380648105887722
-DEPRECATED_FILE_CHARACTERISTICS;5591;25.51802829758101
-NOT_FILEALIGNED_SIZE_OF_RAW;21;0.09584664536741214
-TOO_SMALL_OPTIONAL_HEADER;1;0.004564125969876769
-TOO_MANY_SECTIONS;1;0.004564125969876769
-ZERO_IMAGE_BASE;1;0.004564125969876769
-ZERO_VIRTUAL_SIZE;5;0.022820629849383843
-DEPRECATED_PTR_OF_LINE_NR;15;0.06846188954815152
-OVERLAPPING_SEC;414;1.8895481515289823
-NOT_FILEALIGNED_PTR_TO_RAW;14;0.06389776357827476
-DEPRECATED_PTR_TO_SYMB_TABLE;425;1.9397535371976267
-SECTIONLESS;1;0.004564125969876769
-NOT_SEC_ALIGNED_SIZE_OF_IMAGE;7;0.03194888178913738
-UNINIT_DATA_CONTRAINTS_VIOLATION;40;0.18256503879507074
-ZERO_EP;16;0.0730260155180283
-TOO_LARGE_OPTIONAL_HEADER;1;0.004564125969876769
-DEPRECATED_NR_OF_LINE_NR;14;0.06389776357827476
-TOO_SMALL_SECALIGN;9;0.041077133728890915
-NON_DEFAULT_SIZE_OF_HEADERS;11472;52.35965312642629
-RESERVED_SEC_CHARACTERISTICS;37;0.16887266088544045
-NOT_FILEALIGNED_SIZE_OF_HEADERS;23;0.10497489730716568
-ZERO_SIZE_OF_RAW_DATA;1511;6.8963943404837975
-DEPRECATED_SEC_CHARACTERISTICS;2;0.009128251939753538
-RESERVED_DATA_DIR;3;0.013692377909630305
-VIRTUAL_EP;5889;26.87813783660429
-DEPRECATED_PTR_TO_RELOC;4;0.018256503879507075
-COLLAPSED_OPTIONAL_HEADER;1;0.004564125969876769
-COLLAPSED_MSDOS_HEADER;3;0.013692377909630305
-UNUSUAL_DATA_DIR_NR;2;0.009128251939753538
-DEPRECATED_NR_OF_RELOC;3;0.013692377909630305
-TOO_LARGE_SIZE_OF_RAW;5;0.022820629849383843
-LOW_ALIGNMENT_MODE;282;1.2870835235052487
-OBJECT_ONLY_SEC_CHARACTERISTICS;55;0.25102692834322227
-TOO_LARGE_IMAGE_BASE;6502;29.67594705613875
-DUPLICATE_SEC;9;0.041077133728890915
-DEPRECATED_NR_OF_SYMB;321;1.4650844363304427
-NON_DEFAULT_FILEALIGN;2649;12.09036969420356
-TOO_SMALL_FILEALIGN;280;1.2779552715654952
-CTRL_SYMB_IN_SEC_NAME;609;2.779552715654952
-NON_DEFAULT_IMAGE_BASE;20112;91.79370150616157
+DEPRECATED_PTR_OF_LINE_NR;23;0.06932304539152451
+NOT_SEC_ALIGNED_SIZE_OF_IMAGE;7;0.021098318162637894
+SECTIONLESS;1;0.0030140454518054133
+LOW_ALIGNMENT_MODE;380;1.145337271686057
+CTRL_SYMB_IN_SEC_NAME;757;2.281632407016698
+DUPLICATE_SEC;13;0.03918259087347037
+DEPRECATED_NR_OF_SYMB;327;0.9855928627403702
+COLLAPSED_OPTIONAL_HEADER;1;0.0030140454518054133
+NON_DEFAULT_SIZE_OF_HEADERS;18207;54.87672554102116
+COLLAPSED_MSDOS_HEADER;3;0.009042136355416239
+TOO_LARGE_OPTIONAL_HEADER;1;0.0030140454518054133
+DEPRECATED_FILE_CHARACTERISTICS;7598;22.90071734281753
+OBJECT_ONLY_SEC_CHARACTERISTICS;55;0.16577249984929773
+VIRTUAL_EP;9572;28.850443064681414
+DEPRECATED_PTR_TO_RELOC;4;0.012056181807221653
+DEPRECATED_PTR_TO_SYMB_TABLE;428;1.290011453372717
+TOO_LARGE_IMAGE_BASE;7531;22.698776297546566
+OVERLAPPING_SEC;509;1.5341491349689553
+UNUSUAL_DATA_DIR_NR;2;0.006028090903610827
+NOT_FILEALIGNED_PTR_TO_RAW;15;0.0452106817770812
+ZERO_SIZE_OF_RAW_DATA;1596;4.810416541081439
+NON_DEFAULT_IMAGE_BASE;31154;93.89957200554585
+ZERO_EP;16;0.04822472722888661
+ZERO_IMAGE_BASE;1;0.0030140454518054133
+NOT_FILEALIGNED_SIZE_OF_RAW;21;0.06329495448791368
+TOO_LARGE_SIZE_OF_RAW;5;0.015070227259027066
+ZERO_VIRTUAL_SIZE;5;0.015070227259027066
+TOO_MANY_SECTIONS;1;0.0030140454518054133
+UNUSUAL_SEC_NAME;5100;15.371631804207608
+RESERVED_SEC_CHARACTERISTICS;37;0.1115196817168003
+NON_DEFAULT_FILEALIGN;3920;11.81505817107722
+NOT_FILEALIGNED_SIZE_OF_HEADERS;29;0.08740731810235698
+RESERVED_DATA_DIR;3;0.009042136355416239
+DEPRECATED_NR_OF_RELOC;3;0.009042136355416239
+TOO_SMALL_OPTIONAL_HEADER;1;0.0030140454518054133
+TOO_SMALL_SECALIGN;10;0.03014045451805413
+DEPRECATED_NR_OF_LINE_NR;22;0.06630899993971909
+DEPRECATED_SEC_CHARACTERISTICS;2;0.006028090903610827
+UNINIT_DATA_CONTRAINTS_VIOLATION;57;0.17180059075290854
+TOO_SMALL_FILEALIGN;368;1.1091687262643921

src/main/java/data/goodwareanomalystats

@@ -41,8 +41,8 @@
 import com.github.katjahahn.tools.anomalies.PEAnomalyScanner;
 
 public class PortexStats {
-    
-    //TODO add D:\\ partition files from Win 7 machine!
+
+    // TODO add D:\\ partition files from Win 7 machine!
 
     private static final Logger logger = LogManager.getLogger(PortexStats.class
             .getName());
@@ -62,9 +62,9 @@ public class PortexStats {
     private static int written = 0;
 
     public static void main(String[] args) throws IOException {
-       anomalyCount(new File(GOOD_FILES).listFiles(), "GOOD FILES");
+        anomalyCount(new File(GOOD_FILES).listFiles(), "GOOD FILES");
     }
-    
+
     public static void entropies(File[] files) {
         int total = 0;
         int hasHighE = 0;
@@ -91,15 +91,19 @@ public static void entropies(File[] files) {
                 if (entropies.size() != 0) {
                     entAverage += (entSum / entropies.size());
                 }
-                if (hasHighEFlag) hasHighE++;
-                if (hasLowEFlag) hasLowE++;
+                if (hasHighEFlag)
+                    hasHighE++;
+                if (hasLowEFlag)
+                    hasLowE++;
                 total++;
                 if (total % 1000 == 0) {
                     double highPercent = hasHighE / (double) total;
                     double lowPercent = hasLowE / (double) total;
                     System.out.println("files read: " + total);
-                    System.out.println("has high entropy: " + hasHighE + " " + highPercent);
-                    System.out.println("has low entropy: " + hasLowE + " " + lowPercent);
+                    System.out.println("has high entropy: " + hasHighE + " "
+                            + highPercent);
+                    System.out.println("has low entropy: " + hasLowE + " "
+                            + lowPercent);
                     System.out.println();
                 }
             } catch (Exception e) {
@@ -319,6 +323,16 @@ public static void anomalyCount(File[] files, String base) {
                     System.out.println("Files read: " + total + "/"
                             + files.length);
                 }
+            } catch (FileFormatException e) {
+                if (!file.isDirectory()) {
+                    file.delete();
+                    logger.error("file " + file.getAbsolutePath()
+                            + " deleted, no PE");
+                } else {
+                    logger.error("problem with file " + file.getAbsolutePath()
+                            + " file was not loaded!");
+                }
+                notLoaded++;
             } catch (Exception e) {
                 logger.error("problem with file " + file.getAbsolutePath()
                         + " file was not loaded!");