{ "cells": [ { "cell_type": "code", "execution_count": 1, "metadata": {}, "outputs": [], "source": [ "import datetime\n", "\n", "from util.util import importstr\n", "from util.logconf import logging\n", "log = logging.getLogger('nb')" ] }, { "cell_type": "code", "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "def run(app, *argv):\n", " argv = list(argv)\n", " argv.insert(0, '--num-workers=8') # <1>\n", " log.info(\"Running: {}({!r}).main()\".format(app, argv))\n", " \n", " app_cls = importstr(*app.rsplit('.', 1)) # <2>\n", " app_cls(argv).main()\n", " \n", " log.info(\"Finished: {}.{!r}).main()\".format(app, argv))" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "import os\n", "import shutil\n", "\n", "# clean up any old data that might be around.\n", "# We don't call this by default because it's destructive, \n", "# and would waste a lot of time if it ran when nothing \n", "# on the application side had changed.\n", "def cleanCache():\n", " shutil.rmtree('data-unversioned/cache')\n", " os.mkdir('data-unversioned/cache')\n", "\n", "cleanCache()" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [], "source": [ "training_epochs = 20\n", "experiment_epochs = 10\n", "final_epochs = 50\n", "\n", "training_epochs = 2\n", "experiment_epochs = 2\n", "final_epochs = 5\n", "seg_epochs = 10" ] }, { "cell_type": "code", "execution_count": null, "metadata": {}, "outputs": [], "source": [ "run('code3.prepcache.LunaPrepCacheApp')" ] }, { "cell_type": "code", "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "2025-05-05 10:21:18,587 INFO pid:11504 nb:004:run Running: code3.training.SegmentationTrainingApp(['--num-workers=8', '--epochs=10', '--augmented', 'final-seg']).main()\n", "2025-05-05 10:21:31,180 INFO pid:11504 code3.training:147:initModel Using CUDA; 1 devices.\n", "2025-05-05 10:21:33,343 INFO pid:11504 code3.training:211:main Starting SegmentationTrainingApp, Namespace(batch_size=16, num_workers=8, epochs=10, augmented=True, augment_flip=False, augment_offset=False, augment_scale=False, augment_rotate=False, augment_noise=False, tb_prefix='pytorch', comment='final-seg')\n", "2025-05-05 10:24:51,644 INFO pid:11504 code3.dsets:276:__init__ : 799 training series, 8343 slices, 1028 nodules\n", "2025-05-05 10:25:13,572 INFO pid:11504 code3.dsets:276:__init__ : 89 validation series, 1122 slices, 154 nodules\n", "2025-05-05 10:25:13,573 INFO pid:11504 code3.training:219:main Epoch 1 of 10, 18750/71 batches of size 16*1\n", "2025-05-05 10:25:13,941 WARNING pid:11504 util.util:219:enumerateWithEstimate E1 Training ----/18750, starting\n", "2025-05-05 10:27:32,613 INFO pid:11504 util.util:236:enumerateWithEstimate E1 Training 256/18750, done at 2025-05-05 11:42:30, 1:15:58\n", "2025-05-05 10:27:59,566 INFO pid:11504 util.util:236:enumerateWithEstimate E1 Training 2048/18750, done at 2025-05-05 10:39:55, 0:13:23\n", "2025-05-05 10:31:40,345 INFO pid:11504 util.util:236:enumerateWithEstimate E1 Training 16384/18750, done at 2025-05-05 10:32:24, 0:05:52\n", "2025-05-05 10:32:21,568 WARNING pid:11504 util.util:249:enumerateWithEstimate E1 Training ----/18750, done at 2025-05-05 10:32:21\n", "2025-05-05 10:32:21,573 INFO pid:11504 code3.training:394:logMetrics E1 SegmentationTrainingApp\n", "2025-05-05 10:32:21,581 INFO pid:11504 code3.training:424:logMetrics E1 trn 0.4511 loss, 0.2897 precision, 0.9380 recall, 0.4427 f1 score\n", "2025-05-05 10:32:21,582 INFO pid:11504 code3.training:434:logMetrics E1 trn_all 0.4511 loss, 93.8% tp, 6.2% fn, 230.0% fp\n", "2025-05-05 10:32:21,597 WARNING pid:11504 util.util:219:enumerateWithEstimate E1 Validation ----/71, starting\n", "2025-05-05 10:33:11,598 INFO pid:11504 util.util:236:enumerateWithEstimate E1 Validation 16/71, done at 2025-05-05 10:33:33, 0:00:25\n", "2025-05-05 10:33:19,685 INFO pid:11504 util.util:236:enumerateWithEstimate E1 Validation 32/71, done at 2025-05-05 10:33:37, 0:00:29\n", "2025-05-05 10:33:41,996 INFO pid:11504 util.util:236:enumerateWithEstimate E1 Validation 64/71, done at 2025-05-05 10:33:45, 0:00:37\n", "2025-05-05 10:33:46,620 WARNING pid:11504 util.util:249:enumerateWithEstimate E1 Validation ----/71, done at 2025-05-05 10:33:46\n", "2025-05-05 10:33:46,621 INFO pid:11504 code3.training:394:logMetrics E1 SegmentationTrainingApp\n", "2025-05-05 10:33:46,623 INFO pid:11504 code3.training:424:logMetrics E1 val 0.9552 loss, 0.0182 precision, 0.8209 recall, 0.0356 f1 score\n", "2025-05-05 10:33:46,623 INFO pid:11504 code3.training:434:logMetrics E1 val_all 0.9552 loss, 82.1% tp, 17.9% fn, 4432.3% fp\n", "2025-05-05 10:33:46,665 INFO pid:11504 code3.training:512:saveModel Saved model params to data-unversioned\\part\\models\\pytorch\\seg_2025-05-05_10.21.31_final-seg.300000.state\n", "2025-05-05 10:33:46,669 INFO pid:11504 code3.training:521:saveModel Saved model params to data-unversioned\\part\\models\\pytorch\\seg_2025-05-05_10.21.31_final-seg.best.state\n", "2025-05-05 10:33:46,673 INFO pid:11504 code3.training:524:saveModel SHA1: f8345809152236d6440bdb09f8108c8dbd2de45d\n", "2025-05-05 10:34:34,389 INFO pid:11504 code3.training:219:main Epoch 2 of 10, 18750/71 batches of size 16*1\n", "2025-05-05 10:34:34,713 WARNING pid:11504 util.util:219:enumerateWithEstimate E2 Training ----/18750, starting\n", "2025-05-05 10:35:27,215 INFO pid:11504 util.util:236:enumerateWithEstimate E2 Training 256/18750, done at 2025-05-05 10:39:52, 0:04:29\n", "2025-05-05 10:35:52,454 INFO pid:11504 util.util:236:enumerateWithEstimate E2 Training 2048/18750, done at 2025-05-05 10:39:48, 0:04:24\n", "2025-05-05 10:39:09,172 INFO pid:11504 util.util:236:enumerateWithEstimate E2 Training 16384/18750, done at 2025-05-05 10:39:41, 0:04:18\n", "2025-05-05 10:39:44,360 WARNING pid:11504 util.util:249:enumerateWithEstimate E2 Training ----/18750, done at 2025-05-05 10:39:44\n", "2025-05-05 10:39:44,366 INFO pid:11504 code3.training:394:logMetrics E2 SegmentationTrainingApp\n", "2025-05-05 10:39:44,372 INFO pid:11504 code3.training:424:logMetrics E2 trn 0.3035 loss, 0.4932 precision, 0.9368 recall, 0.6462 f1 score\n", "2025-05-05 10:39:44,373 INFO pid:11504 code3.training:434:logMetrics E2 trn_all 0.3035 loss, 93.7% tp, 6.3% fn, 96.3% fp\n", "2025-05-05 10:39:44,379 INFO pid:11504 code3.training:219:main Epoch 3 of 10, 18750/71 batches of size 16*1\n", "2025-05-05 10:39:44,692 WARNING pid:11504 util.util:219:enumerateWithEstimate E3 Training ----/18750, starting\n", "2025-05-05 10:40:36,394 INFO pid:11504 util.util:236:enumerateWithEstimate E3 Training 256/18750, done at 2025-05-05 10:44:58, 0:04:25\n", "2025-05-05 10:41:01,290 INFO pid:11504 util.util:236:enumerateWithEstimate E3 Training 2048/18750, done at 2025-05-05 10:44:53, 0:04:21\n", "2025-05-05 10:44:21,133 INFO pid:11504 util.util:236:enumerateWithEstimate E3 Training 16384/18750, done at 2025-05-05 10:44:54, 0:04:21\n", "2025-05-05 10:44:56,889 WARNING pid:11504 util.util:249:enumerateWithEstimate E3 Training ----/18750, done at 2025-05-05 10:44:56\n", "2025-05-05 10:44:56,895 INFO pid:11504 code3.training:394:logMetrics E3 SegmentationTrainingApp\n", "2025-05-05 10:44:56,900 INFO pid:11504 code3.training:424:logMetrics E3 trn 0.2747 loss, 0.5313 precision, 0.9420 recall, 0.6794 f1 score\n", "2025-05-05 10:44:56,901 INFO pid:11504 code3.training:434:logMetrics E3 trn_all 0.2747 loss, 94.2% tp, 5.8% fn, 83.1% fp\n", "2025-05-05 10:44:56,907 INFO pid:11504 code3.training:219:main Epoch 4 of 10, 18750/71 batches of size 16*1\n", "2025-05-05 10:44:57,235 WARNING pid:11504 util.util:219:enumerateWithEstimate E4 Training ----/18750, starting\n", "2025-05-05 10:45:49,725 INFO pid:11504 util.util:236:enumerateWithEstimate E4 Training 256/18750, done at 2025-05-05 10:50:05, 0:04:19\n", "2025-05-05 10:46:14,386 INFO pid:11504 util.util:236:enumerateWithEstimate E4 Training 2048/18750, done at 2025-05-05 10:50:04, 0:04:18\n", "2025-05-05 10:49:32,340 INFO pid:11504 util.util:236:enumerateWithEstimate E4 Training 16384/18750, done at 2025-05-05 10:50:04, 0:04:18\n", "2025-05-05 10:50:07,884 WARNING pid:11504 util.util:249:enumerateWithEstimate E4 Training ----/18750, done at 2025-05-05 10:50:07\n", "2025-05-05 10:50:07,889 INFO pid:11504 code3.training:394:logMetrics E4 SegmentationTrainingApp\n", "2025-05-05 10:50:07,894 INFO pid:11504 code3.training:424:logMetrics E4 trn 0.2589 loss, 0.5564 precision, 0.9427 recall, 0.6998 f1 score\n", "2025-05-05 10:50:07,895 INFO pid:11504 code3.training:434:logMetrics E4 trn_all 0.2589 loss, 94.3% tp, 5.7% fn, 75.2% fp\n", "2025-05-05 10:50:07,901 INFO pid:11504 code3.training:219:main Epoch 5 of 10, 18750/71 batches of size 16*1\n", "2025-05-05 10:50:08,205 WARNING pid:11504 util.util:219:enumerateWithEstimate E5 Training ----/18750, starting\n", "2025-05-05 10:51:00,219 INFO pid:11504 util.util:236:enumerateWithEstimate E5 Training 256/18750, done at 2025-05-05 10:55:18, 0:04:22\n", "2025-05-05 10:51:25,012 INFO pid:11504 util.util:236:enumerateWithEstimate E5 Training 2048/18750, done at 2025-05-05 10:55:16, 0:04:19\n", "2025-05-05 10:54:43,576 INFO pid:11504 util.util:236:enumerateWithEstimate E5 Training 16384/18750, done at 2025-05-05 10:55:16, 0:04:19\n", "2025-05-05 10:55:18,900 WARNING pid:11504 util.util:249:enumerateWithEstimate E5 Training ----/18750, done at 2025-05-05 10:55:18\n", "2025-05-05 10:55:18,904 INFO pid:11504 code3.training:394:logMetrics E5 SegmentationTrainingApp\n", "2025-05-05 10:55:18,909 INFO pid:11504 code3.training:424:logMetrics E5 trn 0.2507 loss, 0.5704 precision, 0.9455 recall, 0.7115 f1 score\n", "2025-05-05 10:55:18,910 INFO pid:11504 code3.training:434:logMetrics E5 trn_all 0.2507 loss, 94.5% tp, 5.5% fn, 71.2% fp\n", "2025-05-05 10:55:18,916 WARNING pid:11504 util.util:219:enumerateWithEstimate E5 Validation ----/71, starting\n", "2025-05-05 10:56:06,291 INFO pid:11504 util.util:236:enumerateWithEstimate E5 Validation 16/71, done at 2025-05-05 10:56:26, 0:00:23\n", "2025-05-05 10:56:14,541 INFO pid:11504 util.util:236:enumerateWithEstimate E5 Validation 32/71, done at 2025-05-05 10:56:32, 0:00:29\n", "2025-05-05 10:56:36,245 INFO pid:11504 util.util:236:enumerateWithEstimate E5 Validation 64/71, done at 2025-05-05 10:56:39, 0:00:36\n", "2025-05-05 10:56:40,635 WARNING pid:11504 util.util:249:enumerateWithEstimate E5 Validation ----/71, done at 2025-05-05 10:56:40\n", "2025-05-05 10:56:40,636 INFO pid:11504 code3.training:394:logMetrics E5 SegmentationTrainingApp\n", "2025-05-05 10:56:40,637 INFO pid:11504 code3.training:424:logMetrics E5 val 0.8866 loss, 0.0382 precision, 0.8041 recall, 0.0729 f1 score\n", "2025-05-05 10:56:40,637 INFO pid:11504 code3.training:434:logMetrics E5 val_all 0.8866 loss, 80.4% tp, 19.6% fn, 2025.2% fp\n", "2025-05-05 10:56:40,668 INFO pid:11504 code3.training:512:saveModel Saved model params to data-unversioned\\part\\models\\pytorch\\seg_2025-05-05_10.21.31_final-seg.1500000.state\n", "2025-05-05 10:56:40,672 INFO pid:11504 code3.training:524:saveModel SHA1: 38a247cc304bc8ae2eb2543b66ddf6e7929aa7c0\n", "2025-05-05 10:57:17,682 INFO pid:11504 code3.training:219:main Epoch 6 of 10, 18750/71 batches of size 16*1\n", "2025-05-05 10:57:17,984 WARNING pid:11504 util.util:219:enumerateWithEstimate E6 Training ----/18750, starting\n", "2025-05-05 10:58:09,483 INFO pid:11504 util.util:236:enumerateWithEstimate E6 Training 256/18750, done at 2025-05-05 11:02:33, 0:04:27\n", "2025-05-05 10:58:34,728 INFO pid:11504 util.util:236:enumerateWithEstimate E6 Training 2048/18750, done at 2025-05-05 11:02:30, 0:04:24\n", "2025-05-05 11:02:03,629 INFO pid:11504 util.util:236:enumerateWithEstimate E6 Training 16384/18750, done at 2025-05-05 11:02:37, 0:04:32\n", "2025-05-05 11:02:39,738 WARNING pid:11504 util.util:249:enumerateWithEstimate E6 Training ----/18750, done at 2025-05-05 11:02:39\n", "2025-05-05 11:02:39,750 INFO pid:11504 code3.training:394:logMetrics E6 SegmentationTrainingApp\n", "2025-05-05 11:02:39,770 INFO pid:11504 code3.training:424:logMetrics E6 trn 0.2395 loss, 0.5945 precision, 0.9481 recall, 0.7307 f1 score\n", "2025-05-05 11:02:39,773 INFO pid:11504 code3.training:434:logMetrics E6 trn_all 0.2395 loss, 94.8% tp, 5.2% fn, 64.7% fp\n", "2025-05-05 11:02:39,786 INFO pid:11504 code3.training:219:main Epoch 7 of 10, 18750/71 batches of size 16*1\n", "2025-05-05 11:02:40,135 WARNING pid:11504 util.util:219:enumerateWithEstimate E7 Training ----/18750, starting\n", "2025-05-05 11:03:33,100 INFO pid:11504 util.util:236:enumerateWithEstimate E7 Training 256/18750, done at 2025-05-05 11:07:50, 0:04:20\n", "2025-05-05 11:03:57,710 INFO pid:11504 util.util:236:enumerateWithEstimate E7 Training 2048/18750, done at 2025-05-05 11:07:47, 0:04:17\n", "2025-05-05 11:07:17,833 INFO pid:11504 util.util:236:enumerateWithEstimate E7 Training 16384/18750, done at 2025-05-05 11:07:50, 0:04:21\n", "2025-05-05 11:07:54,509 WARNING pid:11504 util.util:249:enumerateWithEstimate E7 Training ----/18750, done at 2025-05-05 11:07:54\n", "2025-05-05 11:07:54,515 INFO pid:11504 code3.training:394:logMetrics E7 SegmentationTrainingApp\n", "2025-05-05 11:07:54,521 INFO pid:11504 code3.training:424:logMetrics E7 trn 0.2355 loss, 0.5993 precision, 0.9498 recall, 0.7349 f1 score\n", "2025-05-05 11:07:54,522 INFO pid:11504 code3.training:434:logMetrics E7 trn_all 0.2355 loss, 95.0% tp, 5.0% fn, 63.5% fp\n", "2025-05-05 11:07:54,527 INFO pid:11504 code3.training:219:main Epoch 8 of 10, 18750/71 batches of size 16*1\n", "2025-05-05 11:07:54,956 WARNING pid:11504 util.util:219:enumerateWithEstimate E8 Training ----/18750, starting\n", "2025-05-05 11:08:47,184 INFO pid:11504 util.util:236:enumerateWithEstimate E8 Training 256/18750, done at 2025-05-05 11:13:02, 0:04:18\n", "2025-05-05 11:09:11,738 INFO pid:11504 util.util:236:enumerateWithEstimate E8 Training 2048/18750, done at 2025-05-05 11:13:00, 0:04:17\n", "2025-05-05 11:12:28,211 INFO pid:11504 util.util:236:enumerateWithEstimate E8 Training 16384/18750, done at 2025-05-05 11:13:00, 0:04:16\n", "2025-05-05 11:13:03,586 WARNING pid:11504 util.util:249:enumerateWithEstimate E8 Training ----/18750, done at 2025-05-05 11:13:03\n", "2025-05-05 11:13:03,592 INFO pid:11504 code3.training:394:logMetrics E8 SegmentationTrainingApp\n", "2025-05-05 11:13:03,597 INFO pid:11504 code3.training:424:logMetrics E8 trn 0.2299 loss, 0.6067 precision, 0.9518 recall, 0.7410 f1 score\n", "2025-05-05 11:13:03,598 INFO pid:11504 code3.training:434:logMetrics E8 trn_all 0.2299 loss, 95.2% tp, 4.8% fn, 61.7% fp\n", "2025-05-05 11:13:03,604 INFO pid:11504 code3.training:219:main Epoch 9 of 10, 18750/71 batches of size 16*1\n", "2025-05-05 11:13:03,886 WARNING pid:11504 util.util:219:enumerateWithEstimate E9 Training ----/18750, starting\n", "2025-05-05 11:13:56,899 INFO pid:11504 util.util:236:enumerateWithEstimate E9 Training 256/18750, done at 2025-05-05 11:18:09, 0:04:16\n", "2025-05-05 11:14:20,873 INFO pid:11504 util.util:236:enumerateWithEstimate E9 Training 2048/18750, done at 2025-05-05 11:18:04, 0:04:11\n", "2025-05-05 11:17:33,721 INFO pid:11504 util.util:236:enumerateWithEstimate E9 Training 16384/18750, done at 2025-05-05 11:18:05, 0:04:12\n", "2025-05-05 11:18:08,666 WARNING pid:11504 util.util:249:enumerateWithEstimate E9 Training ----/18750, done at 2025-05-05 11:18:08\n", "2025-05-05 11:18:08,672 INFO pid:11504 code3.training:394:logMetrics E9 SegmentationTrainingApp\n", "2025-05-05 11:18:08,677 INFO pid:11504 code3.training:424:logMetrics E9 trn 0.2269 loss, 0.6061 precision, 0.9534 recall, 0.7410 f1 score\n", "2025-05-05 11:18:08,678 INFO pid:11504 code3.training:434:logMetrics E9 trn_all 0.2269 loss, 95.3% tp, 4.7% fn, 62.0% fp\n", "2025-05-05 11:18:08,683 INFO pid:11504 code3.training:219:main Epoch 10 of 10, 18750/71 batches of size 16*1\n", "2025-05-05 11:18:08,991 WARNING pid:11504 util.util:219:enumerateWithEstimate E10 Training ----/18750, starting\n", "2025-05-05 11:19:01,085 INFO pid:11504 util.util:236:enumerateWithEstimate E10 Training 256/18750, done at 2025-05-05 11:23:16, 0:04:19\n", "2025-05-05 11:19:25,980 INFO pid:11504 util.util:236:enumerateWithEstimate E10 Training 2048/18750, done at 2025-05-05 11:23:17, 0:04:20\n", "2025-05-05 11:22:45,169 INFO pid:11504 util.util:236:enumerateWithEstimate E10 Training 16384/18750, done at 2025-05-05 11:23:18, 0:04:20\n", "2025-05-05 11:23:20,885 WARNING pid:11504 util.util:249:enumerateWithEstimate E10 Training ----/18750, done at 2025-05-05 11:23:20\n", "2025-05-05 11:23:20,890 INFO pid:11504 code3.training:394:logMetrics E10 SegmentationTrainingApp\n", "2025-05-05 11:23:20,895 INFO pid:11504 code3.training:424:logMetrics E10 trn 0.2253 loss, 0.6139 precision, 0.9553 recall, 0.7475 f1 score\n", "2025-05-05 11:23:20,896 INFO pid:11504 code3.training:434:logMetrics E10 trn_all 0.2253 loss, 95.5% tp, 4.5% fn, 60.1% fp\n", "2025-05-05 11:23:20,903 WARNING pid:11504 util.util:219:enumerateWithEstimate E10 Validation ----/71, starting\n", "2025-05-05 11:24:06,835 INFO pid:11504 util.util:236:enumerateWithEstimate E10 Validation 16/71, done at 2025-05-05 11:24:25, 0:00:21\n", "2025-05-05 11:24:14,607 INFO pid:11504 util.util:236:enumerateWithEstimate E10 Validation 32/71, done at 2025-05-05 11:24:31, 0:00:27\n", "2025-05-05 11:24:35,808 INFO pid:11504 util.util:236:enumerateWithEstimate E10 Validation 64/71, done at 2025-05-05 11:24:39, 0:00:35\n", "2025-05-05 11:24:40,079 WARNING pid:11504 util.util:249:enumerateWithEstimate E10 Validation ----/71, done at 2025-05-05 11:24:40\n", "2025-05-05 11:24:40,080 INFO pid:11504 code3.training:394:logMetrics E10 SegmentationTrainingApp\n", "2025-05-05 11:24:40,081 INFO pid:11504 code3.training:424:logMetrics E10 val 0.9041 loss, 0.0358 precision, 0.8056 recall, 0.0686 f1 score\n", "2025-05-05 11:24:40,082 INFO pid:11504 code3.training:434:logMetrics E10 val_all 0.9041 loss, 80.6% tp, 19.4% fn, 2168.5% fp\n", "2025-05-05 11:24:40,116 INFO pid:11504 code3.training:512:saveModel Saved model params to data-unversioned\\part\\models\\pytorch\\seg_2025-05-05_10.21.31_final-seg.3000000.state\n", "2025-05-05 11:24:40,120 INFO pid:11504 code3.training:524:saveModel SHA1: ed66359ad64a3103fdba80fb323389cadacc4a99\n", "2025-05-05 11:25:16,678 INFO pid:11504 nb:009:run Finished: code3.training.SegmentationTrainingApp.['--num-workers=8', '--epochs=10', '--augmented', 'final-seg']).main()\n" ] } ], "source": [ "run('code3.training.SegmentationTrainingApp', f'--epochs={seg_epochs}', '--augmented', 'final-seg')" ] } ], "metadata": { "kernelspec": { "display_name": "base", "language": "python", "name": "python3" }, "language_info": { "codemirror_mode": { "name": "ipython", "version": 3 }, "file_extension": ".py", "mimetype": "text/x-python", "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", "version": "3.12.7" }, "orig_nbformat": 4 }, "nbformat": 4, "nbformat_minor": 2 }