Add files via upload

Author: Hellisotherpeople

card_classification.csv

@@ -2,7 +2,7 @@
 import csv
 from flair.data import Sentence
 from flair.models import SequenceTagger
-from flair.embeddings import WordEmbeddings, FlairEmbeddings, StackedEmbeddings, DocumentPoolEmbeddings, BertEmbeddings, ELMoEmbeddings, OpenAIGPTEmbeddings
+from flair.embeddings import WordEmbeddings, FlairEmbeddings, StackedEmbeddings, DocumentPoolEmbeddings, BertEmbeddings, ELMoEmbeddings, OpenAIGPTEmbeddings, RoBERTaEmbeddings, XLNetEmbeddings
 import torch
 from torch import tensor
 import numpy as np
@@ -30,10 +30,12 @@
 from keras.callbacks import ModelCheckpoint
 from keras.wrappers.scikit_learn import KerasClassifier
 from keras.models import Sequential
-from keras.layers import Dense
+from keras.layers import Dense, Conv1D, MaxPooling1D, Flatten, Embedding, Reshape, Input, SimpleRNN, LSTM
+import torch.nn as nn
+import torch.nn.functional as F
 
 
-keras = False
+keras = True
 
 def parse_string(a_str):
     to_ret = "".join([c.lower() for c in a_str if c in string.ascii_letters or c in string.whitespace])
@@ -54,7 +56,8 @@ def transform(self, X):
         size_of_emb = list_of_embeddings[1].size
         if not isinstance(X, str):
             for doc in X:
-                p_str = parse_string(doc)
+                #p_str = parse_string(doc)
+                p_str = doc
                 if not p_str:
                     list_of_emb.append(np.zeros((size_of_emb,), dtype=np.float32))##TODO: don't hard code vector size 
                 else:
@@ -64,7 +67,8 @@ def transform(self, X):
             to_ret = np.array(list_of_emb)
         else:
             try:
-                p_str = parse_string(X)
+                #p_str = parse_string(X)
+                p_str = X
                 if not p_str:
                     to_ret = np.zeros((size_of_emb,), dtype=np.float32)##TODO here too
                 else:
@@ -78,8 +82,9 @@ def transform(self, X):
 
 
 
-stacked_embeddings = DocumentPoolEmbeddings([WordEmbeddings('en'),
-                                        WordEmbeddings('glove'), WordEmbeddings('extvec')])
+stacked_embeddings = DocumentPoolEmbeddings([#WordEmbeddings('en'),
+                                        #WordEmbeddings('glove'),
+                                        WordEmbeddings('en-crawl')])
 
 with open('card_classification.csv') as csvfile:
     reader = csv.reader(csvfile)
@@ -89,31 +94,37 @@ def transform(self, X):
     for row in reader:
         list_of_labels.append(row[0])
         parsed_string = parse_string(row[1])
+        parsed_string = row[1]
         list_of_sentences.append(parsed_string)
         set_obj = Sentence(parsed_string)
         stacked_embeddings.embed(set_obj)
         list_of_embeddings.append(set_obj.get_embedding().cpu().detach().numpy())
 
 
-X_train, X_val, Y_train, Y_val, Emb_train, Emb_val = train_test_split(list_of_sentences, list_of_labels, list_of_embeddings, test_size = 0.30, stratify = list_of_labels, random_state=42)
-
+X_train, X_val, Y_train, Y_val, Emb_train, Emb_val = train_test_split(np.asarray(list_of_sentences), np.asarray(list_of_labels), np.asarray(list_of_embeddings), test_size = 0.30, stratify = list_of_labels, random_state=42)
 
+print(list_of_embeddings[1].size)
 
 def create_model(optimizer='adam', kernel_initializer='glorot_uniform', epochs = 5):
         model = Sequential()
+        #model.add(Reshape((137, 1, 400), input_shape = (137, 400)))
+        #model.add(Conv1D(64, 1, activation='relu'))
         model.add(Dense(list_of_embeddings[1].size, activation='relu',kernel_initializer='he_uniform', use_bias = False))
-        model.add(Dense(11,activation='softmax',kernel_initializer=kernel_initializer, use_bias = False))
+        #model.add(LSTM(list_of_embeddings[1].size, return_sequences = True,))
+        model.add(Dense(len(np.unique(Y_val)),activation='softmax',kernel_initializer=kernel_initializer, use_bias = False))
         model.compile(loss='categorical_crossentropy',optimizer=optimizer, metrics=['accuracy'])
         return model
 
 
 
 if keras:
     checkpointer = ModelCheckpoint(filepath='/tmp/weights.hdf5', verbose=1, save_best_only=True)    
-    model = KerasClassifier(build_fn=create_model, batch_size = 32, epochs = 100, callbacks=[checkpointer], validation_split = 0.2)
+    model = KerasClassifier(build_fn=create_model, batch_size = 32, epochs = 150, callbacks=[checkpointer], validation_split = 0.2)
+
+
 
 #model = SVC(kernel = "rbf", probability = True)
-model = KNeighborsClassifier(n_neighbors=5, metric='cosine', weights = 'distance')
+#model = KNeighborsClassifier(n_neighbors=5, metric='cosine', weights = 'distance')
 #model  = AdaBoostClassifier(n_estimators = 100, random_state = 42)
 #model = RandomForestClassifier(n_jobs = -1, n_estimators = 100, max_features = "auto", criterion = "entropy")
 #model = MLPClassifier(hidden_layer_sizes=(500,), activation = 'relu', solver = 'adam', verbose = True, max_iter = 100) #early_stopping = True, validation_fraction = 0.3, n_iter_no_change = 100)
@@ -138,7 +149,7 @@ def create_model(optimizer='adam', kernel_initializer='glorot_uniform', epochs =
 probs = pipe.predict_proba(X_val)
 a_df = pd.DataFrame(probs, index=Y_val, columns=labels)
 a_df[a_df.eq(0)] = np.nan
-print(a_df)
+print(a_df.round(2))
 
 if keras:
     pipe.named_steps['model'].model.save('keras_model.h5')

classification.py

@@ -38,13 +38,13 @@
 from keras.layers import Dense
 from keras.models import load_model
 
-keras = False
+keras = True
 increment = False
 
 stacked_embeddings = DocumentPoolEmbeddings([
-                                        WordEmbeddings('en'),
-                                        WordEmbeddings('glove'),
-                                        WordEmbeddings('extvec'),#ELMoEmbeddings('original'),
+                                        #WordEmbeddings('en'),
+                                        #WordEmbeddings('glove'),
+                                        WordEmbeddings('en-crawl'),#ELMoEmbeddings('original'),
                                         #BertEmbeddings('bert-base-cased'),
                                         #FlairEmbeddings('news-forward-fast'),
                                         #FlairEmbeddings('news-backward-fast'),
@@ -107,12 +107,12 @@ def transform(self, X):
     pipe.named_steps['model'].model = load_model('keras_model.h5')
 
 
-te = TextExplainer(random_state=42, n_samples=10000, position_dependent=True)
+te = TextExplainer(random_state=42, n_samples=3000, position_dependent=False)
 
 def explain_pred(sentence):
     te.fit(sentence, pipe.predict_proba)
     #txt = format_as_text(te.explain_prediction(target_names=["green", "neutral", "red"]))
-    t_pred = te.explain_prediction(top = 20, target_names=["ANB", "CAP", "ECON", "EDU", "ENV", "EX", "FED", "HEG", "NAT", "POL", "TOP"])
+    t_pred = te.explain_prediction(top = 20, target_names=["ANB", "CAP", "ECON", "EDU", "ENV", "EX", "FED", "HEG", "NAT", "POL", "TOP", "ORI", "QER","COL",])
     txt = format_as_text(t_pred)
     html = format_as_html(t_pred)
     html_file = open("latest_prediction.html", "a+")
@@ -143,11 +143,11 @@ def print_misclass():
             break
         elif label == "stop":
             csvfile.close()
-            if keras:
+            if keras and increment:
                 pipe.named_steps['model'].model.save('keras_model.h5')
                 pipe.named_steps['model'].model = None
-            joblib.dump(pipe, 'saved_card_classification.pkl')
-            print("Model Dumped!!!!")
+                joblib.dump(pipe, 'saved_card_classification.pkl')
+                print("Model Dumped!!!!")
             done = True
             sys.exit()
         else: