From 3da6c167ee2c1e5ff9e5afaa6c377c09c1b82616 Mon Sep 17 00:00:00 2001 From: "pieths.dev@gmail.com" Date: Wed, 9 Oct 2019 16:34:00 -0700 Subject: [PATCH] Initial implementation of NGramExtractor. --- src/python/nimbusml.pyproj | 5 + .../nimbusml/examples/NGramExtractor.py | 37 ++++++ .../NGramExtractor_df.py | 112 ++++++++++++++++++ .../feature_extraction/text/__init__.py | 2 + .../feature_extraction/text/ngramextractor.py | 72 +++++++++++ .../feature_extraction/text/ngramextractor.py | 111 +++++++++++++++++ .../text/test_ngramextractor.py | 36 ++++++ src/python/tests/test_estimator_checks.py | 3 +- src/python/tools/manifest_diff.json | 6 + 9 files changed, 383 insertions(+), 1 deletion(-) create mode 100644 src/python/nimbusml/examples/NGramExtractor.py create mode 100644 src/python/nimbusml/examples/examples_from_dataframe/NGramExtractor_df.py create mode 100644 src/python/nimbusml/feature_extraction/text/ngramextractor.py create mode 100644 src/python/nimbusml/internal/core/feature_extraction/text/ngramextractor.py create mode 100644 src/python/nimbusml/tests/feature_extraction/text/test_ngramextractor.py diff --git a/src/python/nimbusml.pyproj b/src/python/nimbusml.pyproj index a90735af..01e31597 100644 --- a/src/python/nimbusml.pyproj +++ b/src/python/nimbusml.pyproj @@ -91,7 +91,9 @@ + + @@ -240,6 +242,7 @@ + @@ -287,6 +290,7 @@ + @@ -683,6 +687,7 @@ + diff --git a/src/python/nimbusml/examples/NGramExtractor.py b/src/python/nimbusml/examples/NGramExtractor.py new file mode 100644 index 00000000..efed9d7d --- /dev/null +++ b/src/python/nimbusml/examples/NGramExtractor.py @@ -0,0 +1,37 @@ +############################################################################### +# NGramExtractor +from nimbusml import FileDataStream, Pipeline +from nimbusml.datasets import get_dataset +from nimbusml.preprocessing.schema import ColumnDropper +from nimbusml.preprocessing.text import CharTokenizer +from nimbusml.feature_extraction.text import NGramExtractor + +# data input (as a FileDataStream) +path = get_dataset("wiki_detox_train").as_filepath() + +data = FileDataStream.read_csv(path, sep='\t') +print(data.head()) +# Sentiment SentimentText +# 0 1 ==RUDE== Dude, you are rude upload that carl p... +# 1 1 == OK! == IM GOING TO VANDALIZE WILD ONES WIK... +# 2 1 Stop trolling, zapatancas, calling me a liar m... +# 3 1 ==You're cool== You seem like a really cool g... +# 4 1 ::::: Why are you threatening me? I'm not bein... + +# transform usage +pipe = Pipeline([ + CharTokenizer(columns={'SentimentText_Transform': 'SentimentText'}), + NGramExtractor(ngram_length=1, all_lengths=False, columns={'Ngrams': 'SentimentText_Transform'}), + ColumnDropper(columns=['SentimentText_Transform', 'SentimentText', 'Sentiment']) + ]) + +# fit and transform +features = pipe.fit_transform(data) + +print(features.head()) +# Ngrams.<␂> Ngrams.= Ngrams.R Ngrams.U Ngrams.D Ngrams.E ... +# 0 1.0 4.0 1.0 1.0 2.0 1.0 ... +# 1 1.0 4.0 0.0 0.0 2.0 3.0 ... +# 2 1.0 0.0 0.0 0.0 0.0 0.0 ... +# 3 1.0 4.0 0.0 0.0 0.0 0.0 ... +# 4 1.0 0.0 0.0 0.0 0.0 0.0 ... diff --git a/src/python/nimbusml/examples/examples_from_dataframe/NGramExtractor_df.py b/src/python/nimbusml/examples/examples_from_dataframe/NGramExtractor_df.py new file mode 100644 index 00000000..55ccd15b --- /dev/null +++ b/src/python/nimbusml/examples/examples_from_dataframe/NGramExtractor_df.py @@ -0,0 +1,112 @@ +############################################################################### +# Example with NGramExtractor and LogisticRegressionBinaryClassifier +import pandas +from nimbusml import Pipeline +from nimbusml.feature_extraction.text import NGramExtractor +from nimbusml.linear_model import LogisticRegressionBinaryClassifier +from nimbusml.preprocessing.schema import ColumnConcatenator, ColumnDropper +from nimbusml.preprocessing.text import CharTokenizer + +train_reviews = pandas.DataFrame( + data=dict( + review=[ + "This is great", + "I hate it", + "Love it", + "Do not like it", + "Really like it", + "I hate it", + "I like it a lot", + "I kind of hate it", + "I do like it", + "I really hate it", + "It is very good", + "I hate it a bunch", + "I love it a bunch", + "I hate it", + "I like it very much", + "I hate it very much.", + "I really do love it", + "I really do hate it", + "Love it!", + "Hate it!", + "I love it", + "I hate it", + "I love it", + "I hate it", + "I love it"], + like=[ + True, + False, + True, + False, + True, + False, + True, + False, + True, + False, + True, + False, + True, + False, + True, + False, + True, + False, + True, + False, + True, + False, + True, + False, + True])) + +test_reviews = pandas.DataFrame( + data=dict( + review=[ + "This is great", + "I hate it", + "Love it", + "Really like it", + "I hate it", + "I like it a lot", + "I love it", + "I do like it", + "I really hate it", + "I love it"])) + +y = train_reviews['like'] +X = train_reviews.loc[:, train_reviews.columns != 'like'] + +pipeline = Pipeline([ + CharTokenizer(columns={'review_transform': 'review'}), + NGramExtractor(ngram_length=3, all_lengths=False, columns={'ngrams': 'review_transform'}), + ColumnDropper(columns=['review_transform', 'review']) +]) +X = pipeline.fit_transform(X) + +print(X.head()) +# ngrams.<␂>|T|h ngrams.T|h|i ngrams.h|i|s ngrams.i|s|<␠> ... ngrams.i|t|! ngrams.t|!|<␃> ngrams.<␂>|H|a ngrams.H|a|t +# 0 1.0 1.0 1.0 2.0 ... 0.0 0.0 0.0 0.0 +# 1 0.0 0.0 0.0 0.0 ... 0.0 0.0 0.0 0.0 +# 2 0.0 0.0 0.0 0.0 ... 0.0 0.0 0.0 0.0 +# 3 0.0 0.0 0.0 0.0 ... 0.0 0.0 0.0 0.0 +# 4 0.0 0.0 0.0 0.0 ... 0.0 0.0 0.0 0.0 + +model = LogisticRegressionBinaryClassifier().fit(X, y) + +X_test = pipeline.transform(test_reviews) +result = model.predict(X_test) + +print(result) +# 0 True +# 1 False +# 2 True +# 3 True +# 4 False +# 5 True +# 6 True +# 7 True +# 8 False +# 9 True diff --git a/src/python/nimbusml/feature_extraction/text/__init__.py b/src/python/nimbusml/feature_extraction/text/__init__.py index 7dbd24cf..9c16726e 100644 --- a/src/python/nimbusml/feature_extraction/text/__init__.py +++ b/src/python/nimbusml/feature_extraction/text/__init__.py @@ -1,10 +1,12 @@ from .lightlda import LightLda +from .ngramextractor import NGramExtractor from .ngramfeaturizer import NGramFeaturizer from .sentiment import Sentiment from .wordembedding import WordEmbedding __all__ = [ 'LightLda', + 'NGramExtractor', 'NGramFeaturizer', 'Sentiment', 'WordEmbedding' diff --git a/src/python/nimbusml/feature_extraction/text/ngramextractor.py b/src/python/nimbusml/feature_extraction/text/ngramextractor.py new file mode 100644 index 00000000..f27b7004 --- /dev/null +++ b/src/python/nimbusml/feature_extraction/text/ngramextractor.py @@ -0,0 +1,72 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------------------------------------- +# - Generated by tools/entrypoint_compiler.py: do not edit by hand +""" +NGramExtractor +""" + +__all__ = ["NGramExtractor"] + + +from sklearn.base import TransformerMixin + +from ...base_transform import BaseTransform +from ...internal.core.feature_extraction.text.ngramextractor import \ + NGramExtractor as core +from ...internal.utils.utils import trace + + +class NGramExtractor(core, BaseTransform, TransformerMixin): + """ + **Description** + Produces a bag of counts of n-grams (sequences of consecutive values of length 1-n) in a given vector of keys. It does so by building a dictionary of n-grams and using the id in the dictionary as the index in the bag. + + :param columns: see `Columns `_. + + :param ngram_length: Maximum n-gram length. + + :param all_lengths: Whether to store all n-gram lengths up to ngramLength, + or only ngramLength. + + :param skip_length: Maximum number of tokens to skip when constructing an + n-gram. + + :param max_num_terms: Maximum number of n-grams to store in the dictionary. + + :param weighting: The weighting criteria. + + :param params: Additional arguments sent to compute engine. + + """ + + @trace + def __init__( + self, + ngram_length=2, + all_lengths=True, + skip_length=0, + max_num_terms=[10000000], + weighting='Tf', + columns=None, + **params): + + if columns: + params['columns'] = columns + BaseTransform.__init__(self, **params) + core.__init__( + self, + ngram_length=ngram_length, + all_lengths=all_lengths, + skip_length=skip_length, + max_num_terms=max_num_terms, + weighting=weighting, + **params) + self._columns = columns + + def get_params(self, deep=False): + """ + Get the parameters for this operator. + """ + return core.get_params(self) diff --git a/src/python/nimbusml/internal/core/feature_extraction/text/ngramextractor.py b/src/python/nimbusml/internal/core/feature_extraction/text/ngramextractor.py new file mode 100644 index 00000000..c627addd --- /dev/null +++ b/src/python/nimbusml/internal/core/feature_extraction/text/ngramextractor.py @@ -0,0 +1,111 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------------------------------------- +# - Generated by tools/entrypoint_compiler.py: do not edit by hand +""" +NGramExtractor +""" + +__all__ = ["NGramExtractor"] + + +from ....entrypoints.transforms_ngramtranslator import \ + transforms_ngramtranslator +from ....utils.utils import trace +from ...base_pipeline_item import BasePipelineItem, DefaultSignature + + +class NGramExtractor(BasePipelineItem, DefaultSignature): + """ + **Description** + Produces a bag of counts of n-grams (sequences of consecutive values of length 1-n) in a given vector of keys. It does so by building a dictionary of n-grams and using the id in the dictionary as the index in the bag. + + :param ngram_length: Maximum n-gram length. + + :param all_lengths: Whether to store all n-gram lengths up to ngramLength, + or only ngramLength. + + :param skip_length: Maximum number of tokens to skip when constructing an + n-gram. + + :param max_num_terms: Maximum number of n-grams to store in the dictionary. + + :param weighting: The weighting criteria. + + :param params: Additional arguments sent to compute engine. + + """ + + @trace + def __init__( + self, + ngram_length=2, + all_lengths=True, + skip_length=0, + max_num_terms=[10000000], + weighting='Tf', + **params): + BasePipelineItem.__init__( + self, type='transform', **params) + + self.ngram_length = ngram_length + self.all_lengths = all_lengths + self.skip_length = skip_length + self.max_num_terms = max_num_terms + self.weighting = weighting + + @property + def _entrypoint(self): + return transforms_ngramtranslator + + @trace + def _get_node(self, **all_args): + + input_columns = self.input + if input_columns is None and 'input' in all_args: + input_columns = all_args['input'] + if 'input' in all_args: + all_args.pop('input') + + output_columns = self.output + if output_columns is None and 'output' in all_args: + output_columns = all_args['output'] + if 'output' in all_args: + all_args.pop('output') + + # validate input + if input_columns is None: + raise ValueError( + "'None' input passed when it cannot be none.") + + if not isinstance(input_columns, list): + raise ValueError( + "input has to be a list of strings, instead got %s" % + type(input_columns)) + + # validate output + if output_columns is None: + output_columns = input_columns + + if not isinstance(output_columns, list): + raise ValueError( + "output has to be a list of strings, instead got %s" % + type(output_columns)) + + algo_args = dict( + column=[ + dict( + Source=i, + Name=o) for i, + o in zip( + input_columns, + output_columns)] if input_columns else None, + ngram_length=self.ngram_length, + all_lengths=self.all_lengths, + skip_length=self.skip_length, + max_num_terms=self.max_num_terms, + weighting=self.weighting) + + all_args.update(algo_args) + return self._entrypoint(**all_args) diff --git a/src/python/nimbusml/tests/feature_extraction/text/test_ngramextractor.py b/src/python/nimbusml/tests/feature_extraction/text/test_ngramextractor.py new file mode 100644 index 00000000..5914a24f --- /dev/null +++ b/src/python/nimbusml/tests/feature_extraction/text/test_ngramextractor.py @@ -0,0 +1,36 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------------------------------------- +import os +import unittest +import pandas + +from nimbusml import FileDataStream, Pipeline +from nimbusml.datasets import get_dataset +from nimbusml.feature_extraction.text import NGramExtractor +from nimbusml.preprocessing.text import CharTokenizer +from nimbusml.preprocessing.schema import ColumnDropper + + +class TestNGramExtractor(unittest.TestCase): + + def test_ngramfeaturizer(self): + train_df = pandas.DataFrame(data=dict(review=['one', 'two'])) + + pipeline = Pipeline([ + CharTokenizer(columns={'review_transform': 'review'}), + NGramExtractor(ngram_length=3, all_lengths=False, columns={'ngrams': 'review_transform'}), + ColumnDropper(columns=['review_transform', 'review']) + ]) + + result = pipeline.fit_transform(train_df) + self.assertEqual(len(result.columns), 6) + self.assertEqual(result.loc[0, 'ngrams.o|n|e'], 1.0) + self.assertEqual(result.loc[1, 'ngrams.o|n|e'], 0.0) + self.assertEqual(result.loc[0, 'ngrams.t|w|o'], 0.0) + self.assertEqual(result.loc[1, 'ngrams.t|w|o'], 1.0) + + +if __name__ == '__main__': + unittest.main() diff --git a/src/python/tests/test_estimator_checks.py b/src/python/tests/test_estimator_checks.py index 7dfd5eb8..0c9b291a 100644 --- a/src/python/tests/test_estimator_checks.py +++ b/src/python/tests/test_estimator_checks.py @@ -161,7 +161,8 @@ 'PixelExtractor, Loader, Resizer, \ GlobalContrastRowScaler, PcaTransformer, ' 'ColumnConcatenator, Sentiment, CharTokenizer, LightLda, ' - 'NGramFeaturizer, WordEmbedding, LpScaler, WordTokenizer', + 'NGramFeaturizer, WordEmbedding, LpScaler, WordTokenizer' + 'NGramExtractor', 'check_transformer_data_not_an_array, check_pipeline_consistency, ' 'check_fit2d_1feature, check_estimators_fit_returns_self,\ check_fit2d_1sample, ' diff --git a/src/python/tools/manifest_diff.json b/src/python/tools/manifest_diff.json index cddfaf25..68ab2fa5 100644 --- a/src/python/tools/manifest_diff.json +++ b/src/python/tools/manifest_diff.json @@ -751,6 +751,12 @@ "Module": "feature_extraction.text", "Type": "Transform" }, + { + "Name": "Transforms.NGramTranslator", + "NewName": "NGramExtractor", + "Module": "feature_extraction.text", + "Type": "Transform" + }, { "Name": "Transforms.WordEmbeddings", "NewName": "WordEmbedding",