updated jupyter scripts

Author: elitap

scripts/calc_foreground_label_otsu.py

@@ -66,4 +66,4 @@ def gnerateForgroundMap(dir, filter):
                         )
 
     args = parser.parse_args()
-    gnerateForgroundMap(args.dir, args.filter)
+    gnerateForgroundMap(args.datasetpath, args.filter)

scripts/calc_organwise_distance_map.py

@@ -0,0 +1,86 @@
+import os
+import argparse
+
+import shutil
+import SimpleITK as sitk
+import defs
+import uuid
+import subprocess
+from multiprocessing.pool import ThreadPool
+import threading
+
+DMAP_SUBDIR = "dmaps"
+TMP_SUBDIR = "tmp"
+
+
+def create_empty_tmp_dir(dir):
+    tmp = os.path.join(dir, TMP_SUBDIR)
+    if os.path.exists(tmp):
+        shutil.rmtree(tmp)
+    os.makedirs(tmp)
+    return tmp
+
+
+def create_dmap_dir(dir):
+    dmap_dir = os.path.join(dir, DMAP_SUBDIR)
+    if not os.path.exists(dmap_dir):
+        os.makedirs(dmap_dir)
+
+
+def create_dmap((file, dir, tmp_dir)):
+    itk_img = sitk.ReadImage(os.path.join(dir, file), sitk.sitkUInt8)
+    dmap_dir = os.path.join(dir, DMAP_SUBDIR)
+
+    np_img = sitk.GetArrayFromImage(itk_img)
+
+    for key, value in defs.LABELS.iteritems():
+        np_img_cpy = np_img.copy()
+        np_img_cpy[np_img != value] = 0
+        np_img_cpy[np_img == value] = 1
+
+        itk_img_save = sitk.GetImageFromArray(np_img_cpy)
+
+        itk_img_save.SetOrigin(itk_img.GetOrigin())
+        itk_img_save.SetDirection(itk_img.GetDirection())
+        itk_img_save.SetSpacing(itk_img.GetSpacing())
+
+        tmp_file = os.path.join(tmp_dir, str(uuid.uuid4()) + ".nii.gz")
+        sitk.WriteImage(itk_img_save, tmp_file)
+
+        filename = os.path.join(dmap_dir, file[:9] + "_" + key + ".nii.gz")
+        print threading.currentThread(), filename
+        subprocess.check_output(['plastimatch', 'dmap', '--input', tmp_file, '--output', filename])
+
+
+def split_seg_files(dir, filter, threads):
+    tmp_dir = create_empty_tmp_dir(dir)
+    create_dmap_dir(dir)
+    data = list()
+    for f in os.listdir(dir):
+        if filter in f:
+            data.append([f, dir, tmp_dir])
+
+    t = ThreadPool(threads)
+    t.map(create_dmap, data)
+
+    shutil.rmtree(os.path.join(dir, TMP_SUBDIR))
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(prog='')
+
+    parser.add_argument('--data',
+                        required=True,
+                        help="directory containing the segmentation files which should be split to individual organs"
+                        )
+    parser.add_argument('--filter',
+                        required=False,
+                        default='segmentation',
+                        help="filter to find the ")
+    parser.add_argument('--threads','-t',
+                        required=False,
+                        type=int,
+                        default=1)
+
+    args = parser.parse_args()
+    split_seg_files(args.data, args.filter, args.threads)

scripts/collect_dataset_statistics.py

@@ -4,7 +4,7 @@
 import os
 
 
-VALID_FILES = [".mha", ".nrrd", ".mhd"]
+VALID_FILES = [".mha", ".nrrd", ".mhd", ".gz"]
 
 class ImageInfo:
     def __init__(self, file, size, spacing, minMax, labelBB = None):
@@ -35,6 +35,9 @@ def collectImgWithLabels(dataset, labelpostfix, volumepostfix):
 
     dataset = os.path.abspath(dataset)
     imageInfoList = []
+    cnt_datasets = 0
+    cnt_invalid = 0
+    size_dict = dict()
 
     # for test purposes just use some of the files in the dir eg
     # os.listdir(dataset)[:6]
@@ -49,6 +52,17 @@ def collectImgWithLabels(dataset, labelpostfix, volumepostfix):
             for segmentation in os.listdir(dataset):
                 if labelpostfix in segmentation and file[:9] in segmentation:
                     itk_img, segmentationInfo = getImageInfo(segmentation, dataset)
+                    print volumeInfo.file
+                    print segmentationInfo.file
+
+                    size_dict[file[:9]] = (segmentationInfo.size, segmentationInfo.spacing)
+
+                    np_img = sitk.GetArrayFromImage(itk_img)
+                    organ_labels = np.unique(np_img)
+                    cnt_datasets += 1
+                    if not np.array_equal(organ_labels, np.array(range(8))):
+                        print "Not all organs present", organ_labels
+                        cnt_invalid += 1
 
                     #mand_and_par = sitk.BinaryThreshold(itk_img, 5, 8) #lables for right/left parotis and mandibula
 
@@ -61,7 +75,8 @@ def collectImgWithLabels(dataset, labelpostfix, volumepostfix):
                 imageInfoList.append((volumeInfo,segmentationInfo)) #append a tuple (volume, segmentation)
             else:
                 print("no segmentation or foreground found for: ", file)
-
+    print "datsets found", cnt_datasets, "invalid datasets found", cnt_invalid, "valid datasets found", cnt_datasets-cnt_invalid
+    print size_dict
     return imageInfoList
 
 
@@ -76,14 +91,14 @@ def callculateStatistics(imageInfoList):
     for _ ,segmentationInfo in imageInfoList:
         size[label_cnt,:] = segmentationInfo.size
         spacing[label_cnt, :] = segmentationInfo.spacing
-        labelbb[label_cnt, :] = segmentationInfo.labelBB[3:]
+        #labelbb[label_cnt, :] = segmentationInfo.labelBB[3:]
         labelMax[label_cnt] = segmentationInfo.minMax[1]
         label_cnt += 1
 
     medianSpacing = np.around(np.median(spacing,0),2)
     meanSpacing = np.around(np.mean(spacing,0),2)
     print("Volume      size:  max -> x,y,z: ", size.max(0), " min -> x,y,z: ", size.min(0), " avg -> x,y,z: ", size.mean(0), " median -> x,y,z: ", np.median(size,0))
-    print("BoundingBox size:  max -> x,y,z: ", labelbb.max(0), " min -> x,y,z: ", labelbb.min(0), " avg -> x,y,z: ", labelbb.mean(0), " median -> x,y,z: ", np.median(labelbb,0))
+    #print("BoundingBox size:  max -> x,y,z: ", labelbb.max(0), " min -> x,y,z: ", labelbb.min(0), " avg -> x,y,z: ", labelbb.mean(0), " median -> x,y,z: ", np.median(labelbb,0))
     print("Volume   spacing:  max -> x,y,z: ", spacing.max(0), " min -> x,y,z: ", spacing.min(0), " avg -> x,y,z: ", meanSpacing, " medain -> x,y,z: ", medianSpacing)
 
     print("\n")

scripts/combine_segmentations.py

@@ -0,0 +1,108 @@
+import os
+import argparse
+
+import numpy as np
+import SimpleITK as sitk
+import defs
+
+RES_DIR = "../fine_stage/paper/{}/output/50000_{}/orig_size/"
+
+def getSegmentationsFromKey(key, confLst, dataset):
+    segmentations = []
+    for conf in confLst:
+        fstConfDir = RES_DIR.format(confLst[0], dataset)
+        for file in os.listdir(fstConfDir):
+            if key in file:
+                path = os.path.join(fstConfDir, file)
+                segmentations.append(sitk.ReadImage(path, sitk.sitkUInt8))
+                break
+
+    assert len(confLst) == len(segmentations), "not all segfiles found for " + key
+    return segmentations
+
+
+def majorityVote(segmentations):
+    labelForUndecidedPixels = 0
+    return sitk.LabelVoting(segmentations, labelForUndecidedPixels)
+
+
+def staple(segmentations):
+    threshold = 0.8
+
+    result_np = sitk.GetArrayFromImage(segmentations[0]).copy()
+    result_np[result_np != 0] = 0
+
+    for key, value in defs.LABELS.iteritems():
+
+        singleOrganSeg = []
+        for segmentation in segmentations:
+            segmentation_np = sitk.GetArrayFromImage(segmentation).copy()
+            segmentation_np[segmentation_np != value] = 0
+            segmentation_np[segmentation_np == value] = 1
+            singleOrganSeg.append(npToOrigITK(segmentation_np, segmentation))
+
+
+        reference_segmentation_STAPLE_probabilities = sitk.STAPLE(singleOrganSeg, value)
+        reference_segmentation_STAPLE = reference_segmentation_STAPLE_probabilities > threshold
+        STAPLE_np = sitk.GetArrayFromImage(reference_segmentation_STAPLE)
+        result_np[STAPLE_np == 1] = value
+
+    return npToOrigITK(result_np, segmentations[0])
+
+
+def combineSegmentations(configFile, resultpath, dataset):
+
+    if not os.path.exists(resultpath):
+        os.makedirs(resultpath)
+
+    confLst = []
+    with open(configFile, "r") as fptr:
+        for config in fptr:
+            if len(config) == 0 or config[0] == '#':
+                continue
+            confLst.append(config.rstrip().replace('.ini', ''))
+
+    fstConfDir = RES_DIR.format(confLst[0], dataset)
+    for file in os.listdir(fstConfDir):
+        print file
+        key = file[:9]
+        segmentations = getSegmentationsFromKey(key, confLst, dataset)
+
+        result = staple(segmentations)
+        #result = majorityVote(segmentations)
+
+        outfile = os.path.join(resultpath, key+"_out.nii.gz")
+        sitk.WriteImage(result, outfile)
+
+
+def npToOrigITK(np_image, orig_itk_img):
+    new_itk = sitk.GetImageFromArray(np_image)
+    new_itk.SetOrigin(orig_itk_img.GetOrigin())
+    new_itk.SetDirection(orig_itk_img.GetDirection())
+    new_itk.SetSpacing(orig_itk_img.GetSpacing())
+
+    cast_img_filter = sitk.CastImageFilter()
+    cast_img_filter.SetOutputPixelType(sitk.sitkUInt8)
+    new_itk = cast_img_filter.Execute(new_itk)
+
+    return new_itk
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(prog='')
+
+    parser.add_argument('--configfile',
+                        required=True,
+                        help="file containing the models to be consider for the majority voting"
+                        )
+    parser.add_argument('--resdir',
+                        required=True,
+                        help="directory where the results are written to"
+                        )
+    parser.add_argument('--dataset',
+                        choices=['train', 'test'],
+                        default='test'
+                        )
+
+    args = parser.parse_args()
+    combineSegmentations(args.configfile, args.resdir, args.dataset)

scripts/create_final_result_graph.py

@@ -80,8 +80,6 @@ def createbarplot(csvfile):
     fig.savefig("../results/result_figures/95hd_barplot_ead.eps", dpi=150, bbox_inches='tight')
 
 
-
-
 if __name__ == "__main__":
     parser = argparse.ArgumentParser(prog='')
 

scripts/defs.py

@@ -1,8 +1,8 @@
-DATASET_SIZE_MAP = {'0522c0070': ((540, 540, 128), (1.1, 1.1, 3.0)), '0522c0017': ((456, 456, 130), (1.1, 1.1, 3.0)),
-                     '0522c0014': ((470, 470, 132), (1.1, 1.1, 3.0)), '0522c0248': ((456, 456, 147), (1.1, 1.1, 3.0)),
-                     '0522c0013': ((545, 545, 138), (1.1, 1.1, 3.0)), '0522c0159': ((438, 438, 125), (1.1, 1.1, 3.0)),
-                     '0522c0132': ((545, 545, 115), (1.1, 1.1, 3.0)), '0522c0081': ((438, 438, 133), (1.1, 1.1, 3.0)),
-                     '0522c0079': ((545, 545, 130), (1.1, 1.1, 3.0)), '0522c0057': ((456, 456, 121), (1.1, 1.1, 3.0)),
+DATASET_SIZE_MAP_UNISO = {'0522c0070': ((540, 540, 128), (1.1, 1.1, 3.0)), '0522c0017': ((456, 456, 130), (1.1, 1.1, 3.0)),
+                    '0522c0014': ((470, 470, 132), (1.1, 1.1, 3.0)), '0522c0248': ((456, 456, 147), (1.1, 1.1, 3.0)),
+                    '0522c0013': ((545, 545, 138), (1.1, 1.1, 3.0)), '0522c0159': ((438, 438, 125), (1.1, 1.1, 3.0)),
+                    '0522c0132': ((545, 545, 115), (1.1, 1.1, 3.0)), '0522c0081': ((438, 438, 133), (1.1, 1.1, 3.0)),
+                    '0522c0079': ((545, 545, 130), (1.1, 1.1, 3.0)), '0522c0057': ((456, 456, 121), (1.1, 1.1, 3.0)),
                     '0522c0147': ((354, 354, 113), (1.1, 1.1, 3.0)), '0522c0328': ((456, 456, 194), (1.1, 1.1, 3.0)),
                     '0522c0226': ((456, 456, 135), (1.1, 1.1, 3.0)), '0522c0125': ((410, 410, 76), (1.1, 1.1, 3.0)),
                     '0522c0001': ((521, 521, 107), (1.1, 1.1, 3.0)), '0522c0003': ((517, 517, 134), (1.1, 1.1, 3.0)),
@@ -19,12 +19,52 @@
                     '0522c0806': ((456, 456, 124), (1.1, 1.1, 3.0)), '0522c0845': ((591, 591, 139), (1.1, 1.1, 3.0)),
                     '0522c0857': ((545, 545, 151), (1.1, 1.1, 3.0))}
 
+DATASET_SIZE_MAP = {'0522c0878': ((545, 545, 360), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0661': ((456, 456, 412), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0014': ((470, 470, 361), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0248': ((456, 456, 400), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0013': ((545, 545, 376), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0147': ((354, 354, 309), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0017': ((456, 456, 355), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0669': ((517, 517, 311), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0079': ((545, 545, 355), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0057': ((456, 456, 330), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0857': ((545, 545, 412), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0132': ((545, 545, 314), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0328': ((456, 456, 529), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0667': ((521, 521, 332), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0002': ((503, 503, 355), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0081': ((438, 438, 364), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0659': ((549, 549, 375), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0555': ((456, 456, 478), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0226': ((456, 456, 368), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0788': ((545, 545, 357), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0746': ((456, 456, 297), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0708': ((568, 568, 361), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0125': ((410, 410, 207), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0001': ((521, 521, 292), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0003': ((517, 517, 365), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0727': ((545, 545, 357), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0149': ((419, 419, 352), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0161': ((438, 438, 309), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0009': ((535, 535, 393), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0077': ((545, 545, 355), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0845': ((591, 591, 380), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0159': ((438, 438, 341), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0598': ((545, 545, 368), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0190': ((545, 545, 379), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0251': ((456, 456, 270), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0195': ((545, 545, 355), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0253': ((438, 438, 364), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0070': ((540, 540, 349), (1.100000023841858, 1.100000023841858, 1.100000023841858)),
+                    '0522c0806': ((456, 456, 338), (1.100000023841858, 1.100000023841858, 1.100000023841858))}
+
 ORIG_SIZE_MAP = {'0522c0746': ((512, 512, 109), (0.9800000190734862, 0.9800000190734862, 3.0)),
-             '0522c0441': ((512, 512, 360), (1.1699999570846558, 1.1699999570846558, 1.25)),
-             '0522c0708': ((512, 512, 159), (1.2200000286102295, 1.2200000286102295, 2.500629186630249)),
-             '0522c0667': ((512, 512, 146), (1.1200000047683716, 1.1200000047683716, 2.4999997615814205)),
-             '0522c0149': ((512, 512, 129), (0.899999976158142, 0.899999976158142, 3.0)),
-             '0522c0857': ((512, 512, 151), (1.1699999570846558, 1.1699999570846558, 3.0)),
+                 '0522c0441': ((512, 512, 360), (1.1699999570846558, 1.1699999570846558, 1.25)),
+                 '0522c0708': ((512, 512, 159), (1.2200000286102295, 1.2200000286102295, 2.500629186630249)),
+                 '0522c0667': ((512, 512, 146), (1.1200000047683716, 1.1200000047683716, 2.4999997615814205)),
+                 '0522c0149': ((512, 512, 129), (0.899999976158142, 0.899999976158142, 3.0)),
+                 '0522c0857': ((512, 512, 151), (1.1699999570846558, 1.1699999570846558, 3.0)),
                  '0522c0253': ((512, 512, 160), (0.9399999976158141, 0.9399999976158141, 2.5)),
                  '0522c0226': ((512, 512, 162), (0.9800000190734862, 0.9800000190734862, 2.5)),
                  '0522c0457': ((512, 512, 181), (1.25, 1.25, 2.5)),
@@ -92,4 +132,4 @@
 }
 
 DATASET_SCALE_SUBDIR = "upscaled"
-ORIG_SCALE_SUBDIR = "orig_size"
+ORIG_SCALE_SUBDIR = "orig_size"