-
Notifications
You must be signed in to change notification settings - Fork 0
/
Copy pathread_dstc2.py
107 lines (96 loc) · 3.54 KB
/
read_dstc2.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
import json
import os
import pandas as pd
datapath = '/home/dilyara/data/data_files'
dpath = os.path.join(datapath, 'dstc2')
version = None
train_data = []
valid_data = []
test_data = []
with open(os.path.join(dpath, 'dstc2-trn.jsonlist')) as read:
for line in read:
line = line.strip()
# if empty line - it is the end of dialog
if not line:
continue
replica = json.loads(line)
if 'goals' not in replica.keys():
# bot reply
continue
curr_intents = []
if replica['dialog-acts']:
for act in replica['dialog-acts']:
for slot in act['slots']:
if slot[0] == 'slot':
curr_intents.append(act['act'] + '_' + slot[1])
else:
curr_intents.append(act['act'] + '_' + slot[0])
if len(act['slots']) == 0:
curr_intents.append(act['act'])
else:
if replica['text']:
curr_intents.append('unknown')
else:
continue
train_data.append({'text': replica['text'],
'intents': ' '.join(curr_intents)})
with open(os.path.join(dpath, 'dstc2-val.jsonlist')) as read:
for line in read:
line = line.strip()
# if empty line - it is the end of dialog
if not line:
continue
replica = json.loads(line)
if 'goals' not in replica.keys():
# bot reply
continue
curr_intents = []
if replica['dialog-acts']:
for act in replica['dialog-acts']:
for slot in act['slots']:
if slot[0] == 'slot':
curr_intents.append(act['act'] + '_' + slot[1])
else:
curr_intents.append(act['act'] + '_' + slot[0])
if len(act['slots']) == 0:
curr_intents.append(act['act'])
else:
if replica['text']:
curr_intents.append('unknown')
else:
continue
valid_data.append({'text': replica['text'],
'intents': ' '.join(curr_intents)})
with open(os.path.join(dpath, 'dstc2-tst.jsonlist')) as read:
for line in read:
line = line.strip()
# if empty line - it is the end of dialog
if not line:
continue
replica = json.loads(line)
if 'goals' not in replica.keys():
# bot reply
continue
curr_intents = []
if replica['dialog-acts']:
for act in replica['dialog-acts']:
for slot in act['slots']:
if slot[0] == 'slot':
curr_intents.append(act['act'] + '_' + slot[1])
else:
curr_intents.append(act['act'] + '_' + slot[0])
if len(act['slots']) == 0:
curr_intents.append(act['act'])
else:
if replica['text']:
curr_intents.append('unknown')
else:
continue
test_data.append({'text': replica['text'],
'intents': ' '.join(curr_intents)})
train_data = pd.DataFrame(train_data)
train_data.to_csv(os.path.join(dpath, 'dstc2_train.csv'), index=False)
valid_data = pd.DataFrame(valid_data)
valid_data.to_csv(os.path.join(dpath, 'dstc2_valid.csv'), index=False)
test_data = pd.DataFrame(test_data)
test_data.to_csv(os.path.join(dpath, 'dstc2_test.csv'), index=False)