diff --git a/requirements-dev.txt b/requirements-dev.txt index 84896de..957a6ca 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,7 +2,6 @@ pyinstaller>=6.0.0 tox pytest pytest-cov -pytest-flake8 pytest-xdist flake8 chardet<5.0.0 diff --git a/src/fosslight_binary/_simple_mode.py b/src/fosslight_binary/_simple_mode.py index 2508d6e..388b649 100644 --- a/src/fosslight_binary/_simple_mode.py +++ b/src/fosslight_binary/_simple_mode.py @@ -11,6 +11,7 @@ import yaml import fosslight_util.constant as constant from fosslight_util.set_log import init_log +from fosslight_util.time import timestamp_for_filename REMOVE_FILE_EXTENSION_SIMPLE = ['ttf', 'otf', 'png', 'gif', 'jpg', 'bmp', 'jpeg'] logger = logging.getLogger(constant.LOGGER_NAME) @@ -47,6 +48,7 @@ def check_output_path(output, start_time): output_path = "" binary_yaml_file = "" compressed_yaml_file = "" + file_time = timestamp_for_filename(start_time) if output != "": if not os.path.isdir(output) and output.endswith('.yaml'): @@ -57,11 +59,11 @@ def check_output_path(output, start_time): compressed_yaml_file = f"{basename_stem}_compressed.yaml" else: output_path = output - binary_yaml_file = f"binary_list_{start_time}.yaml" - compressed_yaml_file = f"compressed_list_{start_time}.yaml" + binary_yaml_file = f"binary_list_{file_time}.yaml" + compressed_yaml_file = f"compressed_list_{file_time}.yaml" else: - binary_yaml_file = f"binary_list_{start_time}.yaml" - compressed_yaml_file = f"compressed_list_{start_time}.yaml" + binary_yaml_file = f"binary_list_{file_time}.yaml" + compressed_yaml_file = f"compressed_list_{file_time}.yaml" if output_path == "": output_path = os.getcwd() @@ -79,7 +81,7 @@ def init_simple(output_file_name, pkg_name, start_time): output_path, binary_yaml_file, compressed_yaml_file = check_output_path(output_file_name, start_time) - log_file = os.path.join(output_path, f"fosslight_log_bin_{start_time}.txt") + log_file = os.path.join(output_path, f"fosslight_log_bin_{timestamp_for_filename(start_time)}.txt") logger, _result_log = init_log(log_file, False, logging.INFO, logging.DEBUG, pkg_name) return _result_log, binary_yaml_file, compressed_yaml_file diff --git a/src/fosslight_binary/binary_analysis.py b/src/fosslight_binary/binary_analysis.py index 2d530be..639c50f 100755 --- a/src/fosslight_binary/binary_analysis.py +++ b/src/fosslight_binary/binary_analysis.py @@ -10,7 +10,6 @@ from binaryornot.check import is_binary import magic import logging -import yaml import stat from fosslight_util.set_log import init_log, move_log_file import fosslight_util.constant as constant @@ -19,6 +18,8 @@ from ._binary import BinaryItem, TLSH_CHECKSUM_NULL from ._jar_analysis import analyze_jar_file, merge_binary_list from ._simple_mode import print_simple_mode, filter_binary, init_simple, REMOVE_FILE_EXTENSION_SIMPLE +from fosslight_util.cover import dump_result_log +from fosslight_util.time import current_timestamp_utc, format_running_time, timestamp_for_filename from fosslight_util.correct import correct_with_yaml from fosslight_util.oss_item import ScannerItem from fosslight_util.exclude import get_excluded_paths @@ -39,6 +40,7 @@ _error_logs = [] _root_path = "" start_time = "" +finish_time = "" windows = False BYTES = 2048 BIN_EXT_HEADER = {'BIN_FL_Binary': ['ID', 'Binary Path', 'OSS Name', @@ -74,6 +76,7 @@ def init(path_to_find_bin, output_file_name, formats, path_to_exclude=[]): global logger, _result_log _json_ext = ".json" + file_time = timestamp_for_filename(start_time) success, msg, output_path, output_files, output_extensions, formats = check_output_formats_v2(output_file_name, formats) if success: @@ -97,19 +100,19 @@ def init(path_to_find_bin, output_file_name, formats, path_to_exclude=[]): to_remove.append(i) else: if formats[i].startswith('spdx'): - output_files[i] = f"fosslight_spdx_bin_{start_time}" + output_files[i] = f"fosslight_spdx_bin_{file_time}" elif formats[i].startswith('cyclonedx'): - output_files[i] = f'fosslight_cyclonedx_bin_{start_time}' + output_files[i] = f'fosslight_cyclonedx_bin_{file_time}' else: if output_extension == _json_ext: - output_files[i] = f"fosslight_opossum_bin_{start_time}" + output_files[i] = f"fosslight_opossum_bin_{file_time}" else: - output_files[i] = f"fosslight_report_bin_{start_time}" + output_files[i] = f"fosslight_report_bin_{file_time}" else: if output_extension == _json_ext: - output_files[i] = f"fosslight_opossum_bin_{start_time}" + output_files[i] = f"fosslight_opossum_bin_{file_time}" else: - output_files[i] = f"fosslight_report_bin_{start_time}" + output_files[i] = f"fosslight_report_bin_{file_time}" for index in sorted(to_remove, reverse=True): # remove elements of spdx format on windows del output_files[index] @@ -123,7 +126,7 @@ def init(path_to_find_bin, output_file_name, formats, path_to_exclude=[]): logger.error(f"Format error - {msg}") sys.exit(1) - log_file = os.path.join(output_path, f"fosslight_log_bin_{start_time}.txt") + log_file = os.path.join(output_path, f"fosslight_log_bin_{file_time}.txt") logger, _result_log = init_log(log_file, True, logging.INFO, logging.DEBUG, PKG_NAME, path_to_find_bin, path_to_exclude) @@ -169,10 +172,10 @@ def get_file_list(path_to_find, excluded_files): def find_binaries(path_to_find_bin, output_dir, formats, dburl="", simple_mode=False, correct_mode=True, correct_filepath="", path_to_exclude=[], all_exclude_mode=()): - global start_time, _root_path, _result_log + global start_time, finish_time, _root_path, _result_log mode = "Normal Mode" - start_time = datetime.now().strftime('%y%m%d_%H%M') + start_time = current_timestamp_utc() _root_path = path_to_find_bin if not path_to_find_bin.endswith(os.path.sep): @@ -262,7 +265,9 @@ def find_binaries(path_to_find_bin, output_dir, formats, dburl="", simple_mode=F return_list = correct_list logger.info("Success to correct with yaml.") + finish_time = current_timestamp_utc() scan_item.set_cover_comment(f"Detected binaries: {len(return_list)} (Scanned Files : {cnt_file_except_skipped})") + scan_item.set_cover_finish_time(finish_time) for combined_path_and_file, output_extension, output_format in zip(result_reports, output_extensions, formats): results.append(write_output_file(combined_path_and_file, output_extension, scan_item, @@ -284,7 +289,7 @@ def find_binaries(path_to_find_bin, output_dir, formats, dburl="", simple_mode=F try: if os.path.isfile(log_file): - move_log_file(log_file, os.path.join(original_output_path, f"fosslight_log_bin_{start_time}.txt")) + move_log_file(log_file, os.path.join(original_output_path, f"fosslight_log_bin_{timestamp_for_filename(start_time)}.txt")) else: logger.debug("Moving binary analysis log file is skipped") except Exception as ex: @@ -425,9 +430,9 @@ def print_result_log(mode="Normal Mode", success=True, result_log={}, file_cnt=" starttime = result_log["Running time"] else: starttime = start_time + finish_time = current_timestamp_utc() result_log["Mode"] = mode - result_log["Running time"] = starttime + " ~ " + \ - datetime.now().strftime('%Y%m%d_%H%M%S') + result_log["Running time"] = format_running_time(starttime, finish_time) result_log["Execution result"] = 'Success' if success else 'Error occurred' result_log["Identified in Binary DB / Binaries"] = f"{auto_bin_cnt}/{bin_file_cnt}" if len(_error_logs) > 0: @@ -437,7 +442,6 @@ def print_result_log(mode="Normal Mode", success=True, result_log={}, file_cnt=" if bin_list: result_log["Binary list"] = bin_list try: - _str_final_result_log = yaml.safe_dump(result_log, allow_unicode=True, sort_keys=True) - logger.info(_str_final_result_log.strip()) + logger.info(dump_result_log(result_log)) except Exception as ex: logger.warning(f"Error to print final log: {ex}") diff --git a/tox.ini b/tox.ini index 2b082ca..f118bd9 100644 --- a/tox.ini +++ b/tox.ini @@ -11,6 +11,8 @@ allowlist_externals = ls rm cat + pytest + flake8 {toxinidir}/dist/cli setenv = PYTHONPATH=. @@ -43,7 +45,7 @@ deps = -r{toxinidir}/requirements-dev.txt commands = pytest -n 4 tests/initial_tox_test.py::test_release_environment - pytest -v --flake8 + flake8 {toxinidir}/src pyinstaller --onefile cli.py -n cli --additional-hooks-dir=hooks --hidden-import=pkg_resources.extern {toxinidir}/dist/cli -p tests -o test_result_cli ; py.test --cov-report term-missing --cov={envsitepackagesdir}/fosslight_binary