diff --git a/ruff.toml b/ruff.toml new file mode 100644 index 0000000..12673e1 --- /dev/null +++ b/ruff.toml @@ -0,0 +1,52 @@ +indent-width = 2 + +[format] +# Exclude commonly ignored directories. +exclude = [ + ".bzr", + ".direnv", + ".eggs", + ".git", + ".git-rewrite", + ".hg", + ".ipynb_checkpoints", + ".mypy_cache", + ".nox", + ".pants.d", + ".pyenv", + ".pytest_cache", + ".pytype", + ".ruff_cache", + ".svn", + ".tox", + ".venv", + ".vscode", + "__pypackages__", + "_build", + "buck-out", + "build", + "dist", + "node_modules", + "site-packages", + "venv", +] + +indent-style = "space" +line-ending = "lf" +quote-style = "single" +docstring-code-format = true + + +[lint] +# Avoid enforcing line-length violations (`E501`) +ignore = ["E501"] + +# Allow unused variables when underscore-prefixed. +dummy-variable-rgx = "^(_+|(_+[a-zA-Z0-9_]*[a-zA-Z0-9]+?))$" + + +# Ignore `E402` (import violations) in all `__init__.py` files, and in select subdirectories. +[lint.per-file-ignores] +"__init__.py" = ["E402", "F401"] +"**/__init__.py" = ["E402", "F401"] +"**/{tests,docs,tools}/*" = ["E402"] diff --git a/src/thread-cli/__init__.py b/src/thread-cli/__init__.py index 32a3091..0739168 100644 --- a/src/thread-cli/__init__.py +++ b/src/thread-cli/__init__.py @@ -10,7 +10,7 @@ Copyright (c) 2020, thread.ngjx.org. All rights reserved. """ -__version__ = "0.1.0" +__version__ = '0.1.0' from .utils.logging import ColorLogger, logging # Export Core @@ -18,9 +18,9 @@ from .process import process as process_cli app.command( - name="process", + name='process', no_args_is_help=True, - context_settings={"allow_extra_args": True}, + context_settings={'allow_extra_args': True}, )(process_cli) @@ -29,4 +29,4 @@ # Wildcard export -__all__ = ["app"] +__all__ = ['app'] diff --git a/src/thread-cli/base.py b/src/thread-cli/base.py index c2e6eb6..34439c5 100644 --- a/src/thread-cli/base.py +++ b/src/thread-cli/base.py @@ -3,15 +3,14 @@ from . import __version__ from .utils import DebugOption, VerboseOption, QuietOption, verbose_args_processor + logger = logging.getLogger('base') cli_base = typer.Typer( - no_args_is_help = True, - rich_markup_mode = 'rich', - context_settings = { - 'help_option_names': ['-h', '--help', 'help'] - } + no_args_is_help=True, + rich_markup_mode='rich', + context_settings={'help_option_names': ['-h', '--help', 'help']}, ) @@ -21,19 +20,19 @@ def version_callback(value: bool): raise typer.Exit() -@cli_base.callback(invoke_without_command = True) +@cli_base.callback(invoke_without_command=True) def callback( version: bool = typer.Option( - None, '--version', - callback = version_callback, - help = 'Get the current installed version', - is_eager = True + None, + '--version', + callback=version_callback, + help='Get the current installed version', + is_eager=True, ), - debug: bool = DebugOption, verbose: bool = VerboseOption, - quiet: bool = QuietOption -): + quiet: bool = QuietOption, + ): """ [b]Thread CLI[/b]\b\n [white]Use thread from the terminal![/white] @@ -45,9 +44,8 @@ def callback( verbose_args_processor(debug, verbose, quiet) - # Help and Others -@cli_base.command(rich_help_panel = 'Help and Others') +@cli_base.command(rich_help_panel='Help and Others') def help(): """Get [yellow]help[/yellow] from the community. :question:""" typer.echo('Feel free to search for or ask questions here!') @@ -55,10 +53,8 @@ def help(): logger.info('Attempting to open in web browser...') import webbrowser - webbrowser.open( - 'https://github.com/python-thread/thread/issues', - new = 2 - ) + + webbrowser.open('https://github.com/python-thread/thread/issues', new=2) typer.echo('Opening in web browser!') except Exception as e: @@ -67,17 +63,16 @@ def help(): typer.echo('https://github.com/python-thread/thread/issues') - -@cli_base.command(rich_help_panel = 'Help and Others') +@cli_base.command(rich_help_panel='Help and Others') def docs(): """View our [yellow]documentation.[/yellow] :book:""" typer.echo('Thanks for using Thread, here is our documentation!') try: logger.info('Attempting to open in web browser...') import webbrowser + webbrowser.open( - 'https://github.com/python-thread/thread/blob/main/docs/command-line.md', - new = 2 + 'https://github.com/python-thread/thread/blob/main/docs/command-line.md', new=2 ) typer.echo('Opening in web browser!') @@ -87,18 +82,15 @@ def docs(): typer.echo('https://github.com/python-thread/thread/blob/main/docs/command-line.md') - -@cli_base.command(rich_help_panel = 'Help and Others') +@cli_base.command(rich_help_panel='Help and Others') def report(): """[yellow]Report[/yellow] an issue. :bug:""" typer.echo('Sorry you run into an issue, report it here!') try: logger.info('Attempting to open in web browser...') import webbrowser - webbrowser.open( - 'https://github.com/python-thread/thread/issues', - new = 2 - ) + + webbrowser.open('https://github.com/python-thread/thread/issues', new=2) typer.echo('Opening in web browser!') except Exception as e: @@ -107,11 +99,10 @@ def report(): typer.echo('https://github.com/python-thread/thread/issues') - # Utils and Configs -@cli_base.command(rich_help_panel = 'Utils and Configs') +@cli_base.command(rich_help_panel='Utils and Configs') def config(configuration: str): """ [blue]Configure[/blue] the system. :wrench: """ - typer.echo('Coming soon!') \ No newline at end of file + typer.echo('Coming soon!') diff --git a/src/thread-cli/process.py b/src/thread-cli/process.py index c75c5e7..344d38d 100644 --- a/src/thread-cli/process.py +++ b/src/thread-cli/process.py @@ -13,33 +13,74 @@ from rich.live import Live from rich.panel import Panel from rich.console import Group -from rich.progress import Progress, TaskID, SpinnerColumn, TextColumn, BarColumn, TimeRemainingColumn, TimeElapsedColumn -from .utils import DebugOption, VerboseOption, QuietOption, verbose_args_processor, kwargs_processor +from rich.progress import ( + Progress, + TaskID, + SpinnerColumn, + TextColumn, + BarColumn, + TimeRemainingColumn, + TimeElapsedColumn, +) +from .utils import ( + DebugOption, + VerboseOption, + QuietOption, + verbose_args_processor, + kwargs_processor, +) logger = logging.getLogger('base') def process( - func: str = typer.Argument(help = '[blue].path.to.file[/blue]:[blue]function_name[/blue] OR [blue]lambda x: x[/blue]'), - dataset: str = typer.Argument(help = '[blue]./path/to/file.txt[/blue] OR [blue][ i for i in range(2) ][/blue]'), - - args: list[str] = typer.Option([], '--arg', '-a', help = '[blue]Arguments[/blue] passed to each thread'), - kargs: list[str] = typer.Option([], '--kwarg', '-kw', help = '[blue]Key-Value arguments[/blue] passed to each thread'), - threads: int = typer.Option(8, '--threads', '-t', help = 'Maximum number of [blue]threads[/blue] (will scale down based on dataset size)'), - - daemon: bool = typer.Option(False, '--daemon', '-d', help = 'Threads to run in [blue]daemon[/blue] mode'), - graceful_exit: bool = typer.Option(True, '--graceful-exit', '-ge', is_flag = True, help = 'Whether to [blue]gracefully exit[/blue] on abrupt exit (etc. CTRL+C)'), - output: str = typer.Option('./output.json', '--output', '-o', help = '[blue]Output[/blue] file location'), - fileout: bool = typer.Option(True, '--fileout', is_flag = True, help = 'Whether to [blue]write[/blue] output to a file'), - stdout: bool = typer.Option(False, '--stdout', is_flag = True, help = 'Whether to [blue]print[/blue] the output'), - + func: str = typer.Argument( + help='[blue].path.to.file[/blue]:[blue]function_name[/blue] OR [blue]lambda x: x[/blue]' + ), + dataset: str = typer.Argument( + help='[blue]./path/to/file.txt[/blue] OR [blue][ i for i in range(2) ][/blue]' + ), + args: list[str] = typer.Option( + [], '--arg', '-a', help='[blue]Arguments[/blue] passed to each thread' + ), + kargs: list[str] = typer.Option( + [], '--kwarg', '-kw', help='[blue]Key-Value arguments[/blue] passed to each thread' + ), + threads: int = typer.Option( + 8, + '--threads', + '-t', + help='Maximum number of [blue]threads[/blue] (will scale down based on dataset size)', + ), + daemon: bool = typer.Option( + False, '--daemon', '-d', help='Threads to run in [blue]daemon[/blue] mode' + ), + graceful_exit: bool = typer.Option( + True, + '--graceful-exit', + '-ge', + is_flag=True, + help='Whether to [blue]gracefully exit[/blue] on abrupt exit (etc. CTRL+C)', + ), + output: str = typer.Option( + './output.json', '--output', '-o', help='[blue]Output[/blue] file location' + ), + fileout: bool = typer.Option( + True, + '--fileout', + is_flag=True, + help='Whether to [blue]write[/blue] output to a file', + ), + stdout: bool = typer.Option( + False, '--stdout', is_flag=True, help='Whether to [blue]print[/blue] the output' + ), debug: bool = DebugOption, verbose: bool = VerboseOption, - quiet: bool = QuietOption -): + quiet: bool = QuietOption, + ): """ [bold]Utilise parallel processing on a dataset[/bold] - + \b\n [bold white]:glowing_star: Important[/bold white] Args and Kwargs can be parsed by adding multiple -a or -kw @@ -47,7 +88,7 @@ def process( [green]$ thread[/green] [blue]process[/blue] ... -a 'an arg' -kw myKey=myValue -arg testing --kwarg a1=a2 [white]=> args = [ [green]'an arg'[/green], [green]'testing'[/green] ][/white] [white] kwargs = { [green]'myKey'[/green]: [green]'myValue'[/green], [green]'a1'[/green]: [green]'a2'[/green] }[/white] - + [blue][u] [/u][/blue] Learn more from our [link=https://github.com/python-thread/thread/blob/main/docs/command-line.md#parallel-processing-thread-process]documentation![/link] @@ -56,22 +97,20 @@ def process( kwargs = kwargs_processor(kargs) logger.debug('Processed kwargs: %s' % kwargs) - # Verify output if not fileout and not stdout: raise typer.BadParameter('No output method specified') - + if fileout and not os.path.exists('/'.join(output.split('/')[:-1])): raise typer.BadParameter('Output file directory does not exist') - - - # Loading function f = None try: logger.info('Attempted to interpret function') - f = eval(func) # I know eval is bad practice, but I have yet to find a safer replacement + f = eval( + func + ) # I know eval is bad practice, but I have yet to find a safer replacement logger.debug('Evaluated function: %s' % f) if not inspect.isfunction(f): @@ -93,21 +132,20 @@ def process( except Exception as e: logger.warning('Failed to fetch function') raise typer.BadParameter('Failed to fetch function') from e - - - # Loading dataset ds: Union[list, tuple, set, None] = None try: logger.info('Attempting to interpret dataset') ds = eval(dataset) - logger.debug('Evaluated dataset: %s' % (str(ds)[:125] + '...' if len(str(ds)) > 125 else ds)) + logger.debug( + 'Evaluated dataset: %s' % (str(ds)[:125] + '...' if len(str(ds)) > 125 else ds) + ) if not isinstance(ds, (list, tuple, set)): logger.info('Invalid dataset literal') ds = None - + except Exception: logger.info('Failed to interpret dataset') @@ -119,27 +157,31 @@ def process( raise Exception('Invalid file path') with open(dataset, 'r') as a: - ds = [ i.endswith('\n') and i[:-2] for i in a.readlines() ] + ds = [i.endswith('\n') and i[:-2] for i in a.readlines()] except Exception as e: logger.warning('Failed to read dataset') raise typer.BadParameter('Failed to read dataset') from e - - logger.info('Interpreted dataset') + logger.info('Interpreted dataset') # Setup logger.debug('Importing module') from thread import Settings, ParallelProcessing - logger.info('Spawning threads... [Expected: {tcount} threads]'.format(tcount=min(len(ds), threads))) + + logger.info( + 'Spawning threads... [Expected: {tcount} threads]'.format( + tcount=min(len(ds), threads) + ) + ) Settings.set_graceful_exit(graceful_exit) newProcess = ParallelProcessing( - function = f, - dataset = list(ds), - args = args, - kwargs = kwargs, - daemon = daemon, - max_threads = threads + function=f, + dataset=list(ds), + args=args, + kwargs=kwargs, + daemon=daemon, + max_threads=threads, ) logger.info('Created parallel process') @@ -151,7 +193,6 @@ def process( logger.info('Started parallel processes') typer.echo('Waiting for parallel processes to complete, this may take a while...') - # Progress bar :D threadCount = len(newProcess._threads) @@ -160,31 +201,25 @@ def process( TextColumn('{task.description}'), '•', TimeRemainingColumn(), - BarColumn(bar_width = 80), - TextColumn('{task.percentage:>3.1f}%') + BarColumn(bar_width=80), + TextColumn('{task.percentage:>3.1f}%'), ) overall_progress = Progress( - TimeElapsedColumn(), - BarColumn(bar_width = 110), - TextColumn('{task.description}') + TimeElapsedColumn(), BarColumn(bar_width=110), TextColumn('{task.description}') ) workerjobs: list[TaskID] = [ - thread_progress.add_task( - f'[bold blue][T {threadNum}]', - total = 100 - ) + thread_progress.add_task(f'[bold blue][T {threadNum}]', total=100) for threadNum in range(threadCount) ] - overalljob = overall_progress.add_task('(0 of ?)', total = 100) - + overalljob = overall_progress.add_task('(0 of ?)', total=100) with Live( Group( Panel(thread_progress), overall_progress, ), - refresh_per_second = 10 + refresh_per_second=10, ): completed = 0 while completed != threadCount: @@ -194,10 +229,10 @@ def process( for jobID in workerjobs: jobProgress = newProcess._threads[i].progress - thread_progress.update(jobID, completed = round(jobProgress * 100, 2)) + thread_progress.update(jobID, completed=round(jobProgress * 100, 2)) if jobProgress == 1: thread_progress.stop_task(jobID) - thread_progress.update(jobID, description = '[bold green]Completed') + thread_progress.update(jobID, description='[bold green]Completed') completed += 1 progressAvg += jobProgress @@ -206,11 +241,10 @@ def process( # Update overall overall_progress.update( overalljob, - description = f'[bold {"green" if completed == threadCount else "#AAAAAA"}]({completed} of {threadCount})', - completed = round((progressAvg / threadCount) * 100, 2) + description=f'[bold {"green" if completed == threadCount else "#AAAAAA"}]({completed} of {threadCount})', + completed=round((progressAvg / threadCount) * 100, 2), ) time.sleep(0.1) - result = newProcess.get_return_values() @@ -219,11 +253,11 @@ def process( typer.echo(f'Writing to {output}') try: with open(output, 'w') as f: - json.dump(result, f, indent = 2) + json.dump(result, f, indent=2) logger.info('Wrote to file') except Exception as e: logger.error('Failed to write to file') logger.debug(str(e)) if stdout: - typer.echo(result) \ No newline at end of file + typer.echo(result) diff --git a/src/thread-cli/utils/__init__.py b/src/thread-cli/utils/__init__.py index 4462f58..30edbd9 100644 --- a/src/thread-cli/utils/__init__.py +++ b/src/thread-cli/utils/__init__.py @@ -2,8 +2,7 @@ from .processors import ( verbose_args_processor, kwargs_processor, - DebugOption, VerboseOption, - QuietOption + QuietOption, ) diff --git a/src/thread-cli/utils/logging.py b/src/thread-cli/utils/logging.py index f6da703..1d598a8 100644 --- a/src/thread-cli/utils/logging.py +++ b/src/thread-cli/utils/logging.py @@ -7,17 +7,23 @@ # Stdout color configuration class ColorFormatter(logging.Formatter): COLORS = { - "DEBUG": Fore.BLUE, - "INFO": Fore.GREEN, - "WARNING": Fore.YELLOW, - "ERROR": Fore.RED, - "CRITICAL": Fore.RED + Style.BRIGHT, + 'DEBUG': Fore.BLUE, + 'INFO': Fore.GREEN, + 'WARNING': Fore.YELLOW, + 'ERROR': Fore.RED, + 'CRITICAL': Fore.RED + Style.BRIGHT, } def format(self, record): - color = self.COLORS.get(record.levelname, "") - record.levelname = record.levelname if not color else color + Style.BRIGHT + f"{record.levelname:<9}|" - record.msg = record.msg if not color else color + Fore.WHITE + Style.NORMAL + record.msg + color = self.COLORS.get(record.levelname, '') + record.levelname = ( + record.levelname + if not color + else color + Style.BRIGHT + f'{record.levelname:<9}|' + ) + record.msg = ( + record.msg if not color else color + Fore.WHITE + Style.NORMAL + record.msg + ) return logging.Formatter.format(self, record) @@ -26,7 +32,7 @@ def format(self, record): class ColorLogger(logging.Logger): def __init__(self, name): logging.Logger.__init__(self, name, logging.DEBUG) - colorFormatter = ColorFormatter("%(levelname)s %(message)s" + Style.RESET_ALL) + colorFormatter = ColorFormatter('%(levelname)s %(message)s' + Style.RESET_ALL) console = logging.StreamHandler() console.setFormatter(colorFormatter) diff --git a/src/thread-cli/utils/processors.py b/src/thread-cli/utils/processors.py index 7356eb2..75c3ab3 100644 --- a/src/thread-cli/utils/processors.py +++ b/src/thread-cli/utils/processors.py @@ -5,19 +5,17 @@ # Verbose Options # DebugOption = typer.Option( - False, '--debug', - help = 'Set verbosity level to [blue]DEBUG[/blue]', - is_flag = True + False, '--debug', help='Set verbosity level to [blue]DEBUG[/blue]', is_flag=True ) VerboseOption = typer.Option( - False, '--verbose', '-v', - help = 'Set verbosity level to [green]INFO[/green]', - is_flag = True + False, + '--verbose', + '-v', + help='Set verbosity level to [green]INFO[/green]', + is_flag=True, ) QuietOption = typer.Option( - False, '--quiet', '-q', - help = 'Set verbosity level to [red]ERROR[/red]', - is_flag = True + False, '--quiet', '-q', help='Set verbosity level to [red]ERROR[/red]', is_flag=True ) @@ -29,20 +27,15 @@ def verbose_args_processor(debug: bool, verbose: bool, quiet: bool): """Handles setting and raising exceptions for verbose""" if verbose and quiet: raise typer.BadParameter('--quiet cannot be used with --verbose') - + if verbose and debug: raise typer.BadParameter('--debug cannot be used with --verbose') - - logging.getLogger('base').setLevel(( - (debug and logging.DEBUG) or - (verbose and logging.INFO) or - logging.ERROR - )) + + logging.getLogger('base').setLevel( + ((debug and logging.DEBUG) or (verbose and logging.INFO) or logging.ERROR) + ) + def kwargs_processor(arguments: list[str]) -> dict[str, str]: """Processes arguments into kwargs""" - return { - kwarg[0]: kwarg[1] - for i in arguments - if (kwarg := i.split('=')) - } \ No newline at end of file + return {kwarg[0]: kwarg[1] for i in arguments if (kwarg := i.split('='))}