-
Notifications
You must be signed in to change notification settings - Fork 1
/
run_coling_2024.sh
152 lines (130 loc) · 5.67 KB
/
run_coling_2024.sh
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
#!/bin/bash
datasets="multi_woz snips amz_en"
echo "Starting evaluation -- zero-shot and few-shot..."
models="google/flan-t5-xxl
bigscience/bloomz-7b1
lmsys/vicuna-13b-v1.5
WizardLM/WizardLM-13B-V1.2
tiiuae/falcon-7b-instruct"
prompts_domain="dc-001"
prompts_intent="ic-001 ic-002 ic-003 ic-004"
prompts_slot="sf-sp-001 sf-mp-001"
# Inference domain classification, non-fine-tuned
#for dataset in $datasets;
#do
# for model in $models;
# do
# for prompt in $prompts_domain;
# do
# python -m evaluation.run +data=${dataset}-domain +prompt=zero-shot-${prompt} model=${model}
# python -m evaluation.run +data=${dataset}-domain +prompt=few-shot-${prompt} model=${model}
# done
# done
#done
# Inference intent classification, non-fine-tuned
for dataset in $datasets;
do
for model in $models;
do
for prompt in $prompts_intent;
do
python -m evaluation.run +data=${dataset}-intent +prompt=zero-shot-${prompt} model=${model}
python -m evaluation.run +data=${dataset}-intent +prompt=few-shot-${prompt} model=${model}
done
done
done
# Inference slot filling, non-fine-tuned
for dataset in $datasets;
do
for model in $models;
do
for prompt in $prompts_slot;
do
python -m evaluation.run +data=${dataset}-slot +prompt=zero-shot-${prompt} model=${model}
python -m evaluation.run +data=${dataset}-slot +prompt=few-shot-${prompt} model=${model}
done
done
done
echo "Starting parameter-efficient-fine-tuning -- LoRA..."
models="flan-t5-xxl t5-large
bloomz-7b1 bloom-7b1
vicuna-13b-v1.5
WizardLM-13B-V1.2
falcon-7b-instruct falcon-7b"
# Fine-tune domain classification, LoRA
#for model in $models;
#do
# python -m fine_tuning.run +data=10/multi_woz-domain-10 +prompt=dc model=${model} main.model_output_dir=lora_adapters/ peft=lora trainer.max_steps=-1 trainer.num_train_epochs=10
# python -m fine_tuning.run +data=10/snips-domain-10 +prompt=dc model=${model} main.model_output_dir=lora_adapters/ peft=lora trainer.max_steps=-1 trainer.num_train_epochs=10
# python -m fine_tuning.run +data=10/amz_en-domain-10 +prompt=dc model=${model} main.model_output_dir=lora_adapters/ peft=lora trainer.max_steps=500
#done
# Fine-tune intent classification, LoRA
for model in $models;
do
python -m fine_tuning.run +data=10/multi_woz-intent-10 +prompt=ic model=${model} main.model_output_dir=lora_adapters/ peft=lora trainer.max_steps=-1 trainer.num_train_epochs=10
python -m fine_tuning.run +data=10/snips-intent-10 +prompt=ic model=${model} main.model_output_dir=lora_adapters/ peft=lora trainer.max_steps=-1 trainer.num_train_epochs=10
python -m fine_tuning.run +data=10/amz_en-intent-10 +prompt=ic model=${model} main.model_output_dir=lora_adapters/ peft=lora trainer.max_steps=500
done
# Fine-tune slot filling, LoRA
for model in $models;
do
python -m fine_tuning.run +data=10/multi_woz-slot-10 +prompt=sf-sp model=${model} main.model_output_dir=lora_adapters/ peft=lora trainer.max_steps=-1 trainer.num_train_epochs=10
python -m fine_tuning.run +data=10/snips-slot-10 +prompt=sf-sp model=${model} main.model_output_dir=lora_adapters/ peft=lora trainer.max_steps=-1 trainer.num_train_epochs=10
python -m fine_tuning.run +data=10/amz_en-slot-10 +prompt=sf-sp model=${model} main.model_output_dir=lora_adapters/ peft=lora trainer.max_steps=500
python -m fine_tuning.run +data=10/multi_woz-slot-10 +prompt=sf-mp model=${model} main.model_output_dir=lora_adapters/ peft=lora trainer.max_steps=-1 trainer.num_train_epochs=10
python -m fine_tuning.run +data=10/snips-slot-10 +prompt=sf-mp model=${model} main.model_output_dir=lora_adapters/ peft=lora trainer.max_steps=-1 trainer.num_train_epochs=10
python -m fine_tuning.run +data=10/amz_en-slot-10 +prompt=sf-mp model=${model} main.model_output_dir=lora_adapters/ peft=lora trainer.max_steps=500
done
echo "Starting evaluation -- zero-shot and few-shot with fine-tuned models..."
models="flan-t5-xxl t5-large
bloomz-7b1 bloom-7b1
vicuna-13b-v1.5
WizardLM-13B-V1.2
falcon-7b-instruct falcon-7b"
prompts_domain="dc-001"
prompts_intent="ic-001 ic-002 ic-003 ic-004"
peft_methods="lora"
# Inference domain classification, LoRA
for dataset in $datasets;
do
for model in $models;
do
for prompt in $prompts_domain;
do
for peft in $peft_methods;
do
python -m evaluation.run +data=${dataset}-domain +prompt=zero-shot-${prompt} model=sft/${dataset}/dc/${model}_${peft}
# python -m evaluation.run +data=${dataset}-domain +prompt=few-shot-${prompt} model=sft/${dataset}/dc/${model}_${peft}
done
done
done
done
# Inference intent classification, LoRA
for dataset in $datasets;
do
for model in $models;
do
for prompt in $prompts_intent;
do
for peft in $peft_methods;
do
python -m evaluation.run +data=${dataset}-intent +prompt=zero-shot-${prompt} model=sft/${dataset}/ic/${model}_${peft}
# python -m evaluation.run +data=${dataset}-intent +prompt=few-shot-${prompt} model=sft/${dataset}/ic/${model}_${peft}
done
done
done
done
# Inference slot filling, LoRA
for dataset in $datasets;
do
for model in $models;
do
for peft in $peft_methods;
do
python -m evaluation.run +data=${dataset}-slot +prompt=zero-shot-sf-sp-001 model=sft/${dataset}/sf-sp/${model}_${peft}
# python -m evaluation.run +data=${dataset}-slot +prompt=few-shot-sf-sp-001 model=sft/${dataset}/sf-sp/${model}_${peft}
python -m evaluation.run +data=${dataset}-slot +prompt=zero-shot-sf-mp-001 model=sft/${dataset}/sf-mp/${model}_${peft}
# python -m evaluation.run +data=${dataset}-slot +prompt=few-shot-sf-mp-001 model=sft/${dataset}/sf-mp/${model}_${peft}
done
done
done