v1

Author: shenwzh3

dataloader.py

@@ -0,0 +1,64 @@
+from dataset import *
+import pickle
+from torch.utils.data.sampler import SubsetRandomSampler
+from torch.utils.data import DataLoader
+import os
+import argparse
+import numpy as np
+from  transformers import BertTokenizer
+
+def get_train_valid_sampler(trainset):
+    size = len(trainset)
+    idx = list(range(size))
+    return SubsetRandomSampler(idx)
+
+
+def load_vocab(dataset_name):
+    speaker_vocab = pickle.load(open('../data/%s/speaker_vocab.pkl' % (dataset_name), 'rb'))
+    label_vocab = pickle.load(open('../data/%s/label_vocab.pkl' % (dataset_name), 'rb'))
+    person_vec_dir = '../data/%s/person_vect.pkl' % (dataset_name)
+    # if os.path.exists(person_vec_dir):
+    #     print('Load person vec from ' + person_vec_dir)
+    #     person_vec = pickle.load(open(person_vec_dir, 'rb'))
+    # else:
+    #     print('Creating personality vectors')
+    #     person_vec = np.random.randn(len(speaker_vocab['itos']), 100)a
+    #     print('Saving personality vectors to' + person_vec_dir)
+    #     with open(person_vec_dir,'wb') as f:
+    #         pickle.dump(person_vec, f, -1)
+    person_vec = None
+
+    return speaker_vocab, label_vocab, person_vec
+
+
+def get_IEMOCAP_loaders(dataset_name = 'IEMOCAP', batch_size=32, num_workers=0, pin_memory=False, args = None):
+    print('building vocab.. ')
+    speaker_vocab, label_vocab, person_vec = load_vocab(dataset_name)
+    print('building datasets..')
+    trainset = IEMOCAPDataset(dataset_name, 'train',  speaker_vocab, label_vocab, args)
+    devset = IEMOCAPDataset(dataset_name, 'dev', speaker_vocab, label_vocab, args)
+    train_sampler = get_train_valid_sampler(trainset)
+    valid_sampler = get_train_valid_sampler(devset)
+
+    train_loader = DataLoader(trainset,
+                              batch_size=batch_size,
+                              sampler=train_sampler,
+                              collate_fn=trainset.collate_fn,
+                              num_workers=num_workers,
+                              pin_memory=pin_memory)
+
+    valid_loader = DataLoader(devset,
+                              batch_size=batch_size,
+                              sampler=valid_sampler,
+                              collate_fn=devset.collate_fn,
+                              num_workers=num_workers,
+                              pin_memory=pin_memory)
+
+    testset = IEMOCAPDataset(dataset_name, 'test',  speaker_vocab, label_vocab, args)
+    test_loader = DataLoader(testset,
+                             batch_size=batch_size,
+                             collate_fn=testset.collate_fn,
+                             num_workers=num_workers,
+                             pin_memory=pin_memory)
+
+    return train_loader, valid_loader, test_loader, speaker_vocab, label_vocab, person_vec

dataset.py

@@ -0,0 +1,163 @@
+import torch
+from torch.utils.data import Dataset
+from torch.nn.utils.rnn import pad_sequence
+import pickle, pandas as pd
+import json
+import numpy as np
+import random
+from pandas import DataFrame
+
+
+class IEMOCAPDataset(Dataset):
+
+    def __init__(self, dataset_name = 'IEMOCAP', split = 'train', speaker_vocab=None, label_vocab=None, args = None, tokenizer = None):
+        self.speaker_vocab = speaker_vocab
+        self.label_vocab = label_vocab
+        self.args = args
+        self.data = self.read(dataset_name, split, tokenizer)
+        print(len(self.data))
+
+        self.len = len(self.data)
+
+    def read(self, dataset_name, split, tokenizer):
+        with open('../data/%s/%s_data_roberta.json.feature'%(dataset_name, split), encoding='utf-8') as f:
+            raw_data = json.load(f)
+
+        # process dialogue
+        dialogs = []
+        # raw_data = sorted(raw_data, key=lambda x:len(x))
+        for d in raw_data:
+            # if len(d) < 5 or len(d) > 6:
+            #     continue
+            utterances = []
+            labels = []
+            speakers = []
+            features = []
+            for i,u in enumerate(d):
+                utterances.append(u['text'])
+                labels.append(self.label_vocab['stoi'][u['label']] if 'label' in u.keys() else -1)
+                speakers.append(self.speaker_vocab['stoi'][u['speaker']])
+                features.append(u['cls'])
+            dialogs.append({
+                'utterances': utterances,
+                'labels': labels,
+                'speakers':speakers,
+                'features': features
+            })
+        random.shuffle(dialogs)
+        return dialogs
+
+    def __getitem__(self, index):
+        '''
+        :param index:
+        :return:
+            feature,
+            label
+            speaker
+            length
+            text
+        '''
+        return torch.FloatTensor(self.data[index]['features']), \
+               torch.LongTensor(self.data[index]['labels']),\
+               self.data[index]['speakers'], \
+               len(self.data[index]['labels']), \
+               self.data[index]['utterances']
+
+    def __len__(self):
+        return self.len
+
+    def get_adj(self, speakers, max_dialog_len):
+        '''
+        get adj matrix
+        :param speakers:  (B, N)
+        :param max_dialog_len:
+        :return:
+            adj: (B, N, N) adj[:,i,:] means the direct predecessors of node i
+        '''
+        adj = []
+        for speaker in speakers:
+            a = torch.zeros(max_dialog_len, max_dialog_len)
+            for i,s in enumerate(speaker):
+                get_local_pred = False
+                get_global_pred = False
+                for j in range(i - 1, -1, -1):
+                    if speaker[j] == s and not get_local_pred:
+                        get_local_pred = True
+                        a[i,j] = 1
+                    elif speaker[j] != s and not get_global_pred:
+                        get_global_pred = True
+                        a[i,j] = 1
+                    if get_global_pred and get_local_pred:
+                        break
+            adj.append(a)
+        return torch.stack(adj)
+
+    def get_adj_v1(self, speakers, max_dialog_len):
+        '''
+        get adj matrix
+        :param speakers:  (B, N)
+        :param max_dialog_len:
+        :return:
+            adj: (B, N, N) adj[:,i,:] means the direct predecessors of node i
+        '''
+        adj = []
+        for speaker in speakers:
+            a = torch.zeros(max_dialog_len, max_dialog_len)
+            for i,s in enumerate(speaker):
+                cnt = 0
+                for j in range(i - 1, -1, -1):             
+                    a[i,j] = 1
+                    if speaker[j] == s:
+                        cnt += 1
+                        if cnt==self.args.windowp:
+                            break
+            adj.append(a)
+        return torch.stack(adj)
+
+    def get_s_mask(self, speakers, max_dialog_len):
+        '''
+        :param speakers:
+        :param max_dialog_len:
+        :return:
+         s_mask: (B, N, N) s_mask[:,i,:] means the speaker informations for predecessors of node i, where 1 denotes the same speaker, 0 denotes the different speaker
+         s_mask_onehot (B, N, N, 2) onehot emcoding of s_mask
+        '''
+        s_mask = []
+        s_mask_onehot = []
+        for speaker in speakers:
+            s = torch.zeros(max_dialog_len, max_dialog_len, dtype = torch.long)
+            s_onehot = torch.zeros(max_dialog_len, max_dialog_len, 2)
+            for i in range(len(speaker)):
+                for j in range(len(speaker)):
+                    if speaker[i] == speaker[j]:
+                        s[i,j] = 1
+                        s_onehot[i,j,1] = 1
+                    else:
+                        s_onehot[i,j,0] = 1
+
+            s_mask.append(s)
+            s_mask_onehot.append(s_onehot)
+        return torch.stack(s_mask), torch.stack(s_mask_onehot)
+
+    def collate_fn(self, data):
+        '''
+        :param data:
+            features, labels, speakers, length, utterances
+        :return:
+            features: (B, N, D) padded
+            labels: (B, N) padded
+            adj: (B, N, N) adj[:,i,:] means the direct predecessors of node i
+            s_mask: (B, N, N) s_mask[:,i,:] means the speaker informations for predecessors of node i, where 1 denotes the same speaker, 0 denotes the different speaker
+            lengths: (B, )
+            utterances:  not a tensor
+        '''
+        max_dialog_len = max([d[3] for d in data])
+        feaures = pad_sequence([d[0] for d in data], batch_first = True) # (B, N, D)
+        labels = pad_sequence([d[1] for d in data], batch_first = True, padding_value = -1) # (B, N )
+        adj = self.get_adj_v1([d[2] for d in data], max_dialog_len)
+        s_mask, s_mask_onehot = self.get_s_mask([d[2] for d in data], max_dialog_len)
+        lengths = torch.LongTensor([d[3] for d in data])
+        speakers = pad_sequence([torch.LongTensor(d[2]) for d in data], batch_first = True, padding_value = -1)
+        utterances = [d[4] for d in data]
+
+        return feaures, labels, adj,s_mask, s_mask_onehot,lengths, speakers, utterances

evaluate.py

@@ -0,0 +1,167 @@
+import os
+os.environ["CUDA_VISIBLE_DEVICES"] = '0' 
+import numpy as np, argparse, time, pickle, random
+import torch
+import torch.nn as nn
+import torch.optim as optim
+from dataloader import IEMOCAPDataset
+from model import *
+from sklearn.metrics import f1_score, confusion_matrix, accuracy_score, classification_report, \
+    precision_recall_fscore_support
+from trainer import  train_or_eval_model, save_badcase
+from dataset import IEMOCAPDataset
+from dataloader import get_IEMOCAP_loaders
+from transformers import AdamW
+import copy
+
+# We use seed = 100 for reproduction of the results reported in the paper.
+seed = 100
+
+
+def seed_everything(seed=seed):
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    torch.cuda.manual_seed(seed)
+    torch.cuda.manual_seed_all(seed)
+    torch.backends.cudnn.benchmark = False
+    torch.backends.cudnn.deterministic = True
+
+
+def evaluate(model,  dataloader, cuda, args, speaker_vocab, label_vocab):
+    preds, labels = [], []
+    scores, vids = [], []
+    dialogs = []
+    speakers = []
+
+    model.eval()
+
+    for data in dataloader:
+
+        features, label, adj,s_mask, s_mask_onehot,lengths, speaker, utterances = data
+        if cuda:
+            features = features.cuda()
+            label = label.cuda()
+            adj = adj.cuda()
+            s_mask_onehot = s_mask_onehot.cuda()
+            s_mask = s_mask.cuda()
+            lengths = lengths.cuda()
+
+        log_prob = model(features, adj,s_mask, s_mask_onehot, lengths) # (B, N, C)
+
+        label = label.cpu().numpy().tolist() # (B, N)
+        pred = torch.argmax(log_prob, dim = 2).cpu().numpy().tolist() # (B, N)
+        preds += pred
+        labels += label
+        dialogs += utterances
+        speakers += speaker
+
+    if preds != []:
+        new_preds = []
+        new_labels = []
+        for i,label in enumerate(labels):
+            for j,l in enumerate(label):
+                if l != -1:
+                    new_labels.append(l)
+                    new_preds.append(preds[i][j])
+    else:
+        return
+
+    avg_accuracy = round(accuracy_score(new_labels, new_preds) * 100, 2)
+    if args.dataset_name in ['IEMOCAP', 'MELD', 'EmoryNLP']:
+        avg_fscore = round(f1_score(new_labels, new_preds, average='weighted') * 100, 2)
+        print('test_accuracy', avg_accuracy)
+        print('test_f1', avg_fscore)
+        return
+    else:
+        avg_micro_fscore = round(f1_score(new_labels, new_preds, average='micro', labels=list(range(1, 7))) * 100, 2)
+        avg_macro_fscore = round(f1_score(new_labels, new_preds, average='macro') * 100, 2)
+        print('test_accuracy', avg_accuracy)
+        print('test_micro_f1', avg_micro_fscore)
+        print('test_macro_f1', avg_macro_fscore)
+        return
+
+if __name__ == '__main__':
+
+    #path = './saved_models/'
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('--bert_model_dir', type=str, default='')
+    parser.add_argument('--bert_tokenizer_dir', type=str, default='')
+
+    parser.add_argument('--state_dict_file', type=str, default='')
+
+    parser.add_argument('--bert_dim', type = int, default=1024)
+    parser.add_argument('--hidden_dim', type = int, default=300)
+    parser.add_argument('--mlp_layers', type=int, default=2, help='Number of output mlp layers.')
+    parser.add_argument('--gnn_layers', type=int, default=2, help='Number of gnn layers.')
+    parser.add_argument('--emb_dim', type=int, default=1024, help='Feature size.')
+
+    parser.add_argument('--attn_type', type=str, default='rgcn', choices=['dotprod','linear','bilinear', 'rgcn'], help='Feature size.')
+    parser.add_argument('--no_rel_attn',  action='store_true', default=False, help='no relation for edges' )
+
+    parser.add_argument('--max_sent_len', type=int, default=200,
+                        help='max content length for each text, if set to 0, then the max length has no constrain')
+
+    parser.add_argument('--no_cuda', action='store_true', default=False, help='does not use GPU')
+
+    parser.add_argument('--dataset_name', default='IEMOCAP', type= str, help='dataset name, IEMOCAP or MELD or DailyDialog')
+
+    parser.add_argument('--windowp', type=int, default=1,
+                        help='context window size for constructing edges in graph model for past utterances')
+
+    parser.add_argument('--windowf', type=int, default=0,
+                        help='context window size for constructing edges in graph model for future utterances')
+
+    parser.add_argument('--max_grad_norm', type=float, default=5.0, help='Gradient clipping.')
+
+    parser.add_argument('--lr', type=float, default=1e-3, metavar='LR', help='learning rate')
+
+
+    parser.add_argument('--dropout', type=float, default=0, metavar='dropout', help='dropout rate')
+
+    parser.add_argument('--batch_size', type=int, default=8, metavar='BS', help='batch size')
+
+    parser.add_argument('--epochs', type=int, default=20, metavar='E', help='number of epochs')
+
+    parser.add_argument('--tensorboard', action='store_true', default=False, help='Enables tensorboard log')
+
+    parser.add_argument('--nodal_att_type', type=str, default=None, choices=['global','past'], help='type of nodal attention')
+
+    args = parser.parse_args()
+    print(args)
+    
+    seed_everything()
+    
+    args.cuda = torch.cuda.is_available() and not args.no_cuda
+    
+    if args.cuda:
+        print('Running on GPU')
+    else:
+        print('Running on CPU')
+
+    if args.tensorboard:
+        from tensorboardX import SummaryWriter
+
+        writer = SummaryWriter()
+
+
+    cuda = args.cuda
+    n_epochs = args.epochs
+    batch_size = args.batch_size
+    train_loader, valid_loader, test_loader, speaker_vocab, label_vocab, person_vec = get_IEMOCAP_loaders(dataset_name=args.dataset_name, batch_size=batch_size, num_workers=0, args = args)
+    n_classes = len(label_vocab['itos'])
+
+    print('building model..')
+    model = DAGERC_fushion(args, n_classes)
+
+
+    if torch.cuda.device_count() > 1:
+        print('Multi-GPU...........')
+        model = nn.DataParallel(model,device_ids = range(torch.cuda.device_count()))
+    if cuda:
+        model.cuda()
+    
+    state_dict = torch.load(args.state_dict_file)
+    model.load_state_dict(state_dict)
+    evaluate(model, test_loader, cuda, args, speaker_vocab, label_vocab)