Skip to content

Commit 31aa8d3

Browse files
oscarandersson8218oscarandersson8218
authored
Arm backend: Update vgf model tests (#16199)
### Summary Updates VgfTestPipeline to also take an optional argument quantize which defaults to True. Also updates vgf tests in models/ to use the quantize flag instead of passing a tosa_version to VgfTestPipeline. Also rename the tests from vgf_INT/FP to vgf_quant and vgf_no_quant. Signed-off-by: Oscar Andersson <[email protected]>
1 parent bdbffce commit 31aa8d3

16 files changed

+95
-69
lines changed

‎backends/arm/scripts/parse_test_names.py‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,16 @@
3838
ALL_EDGE_OPS = SAMPLE_INPUT.keys() | CUSTOM_EDGE_OPS
3939

4040
# Add all targets and TOSA profiles we support here.
41-
TARGETS = ["tosa_FP", "tosa_INT", "u55_INT", "u85_INT", "vgf_INT", "vgf_FP"]
41+
TARGETS = [
42+
"tosa_FP",
43+
"tosa_INT",
44+
"u55_INT",
45+
"u85_INT",
46+
"vgf_INT",
47+
"vgf_FP",
48+
"vgf_quant",
49+
"vgf_no_quant",
50+
]
4251

4352

4453
def get_op_name_map():

‎backends/arm/test/models/stable_diffusion/test_CLIPTextModelWithProjection.py‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -51,6 +51,9 @@ class TestCLIPTextModelWithProjection:
5151
"torch.ops.higher_order.executorch_call_delegate": 2,
5252
}
5353

54+
ops_after_partitioner_vgf_quantize = ops_after_partitioner_FP
55+
ops_after_partitioner_vgf_no_quantize = ops_after_partitioner_FP
56+
5457
def _prepare_inputs(
5558
self,
5659
batch_size=12,
@@ -119,7 +122,7 @@ def test_CLIPTextModelWithProjection_tosa_INT():
119122

120123

121124
@common.SkipIfNoModelConverter
122-
def test_CLIPTextModelWithProjection_vgf_FP():
125+
def test_CLIPTextModelWithProjection_vgf_no_quant():
123126
text_encoder_model, text_encoder_model_inputs = (
124127
TestCLIPTextModelWithProjection().prepare_model_and_inputs()
125128
)
@@ -129,23 +132,24 @@ def test_CLIPTextModelWithProjection_vgf_FP():
129132
text_encoder_model_inputs,
130133
aten_op=[],
131134
exir_op=[],
132-
tosa_version="TOSA-1.0+FP",
133135
use_to_edge_transform_and_lower=True,
134-
atol=4, # TODO: Investiage numerical issue: MAX Diff ~50%
136+
atol=4,
135137
transform_passes=[
136138
ConvertInt64ConstOpsToInt32Pass(),
137139
ConvertInt64OutputOpsToInt32Pass(),
138140
InsertInt32CastsAfterInt64PlaceholdersPass(),
139141
],
142+
quantize=False,
140143
)
141144
pipeline.change_args(
142-
"check_count.exir", TestCLIPTextModelWithProjection.ops_after_partitioner_FP
145+
"check_count.exir",
146+
TestCLIPTextModelWithProjection.ops_after_partitioner_vgf_no_quantize,
143147
)
144148
pipeline.run()
145149

146150

147151
@common.SkipIfNoModelConverter
148-
def test_CLIPTextModelWithProjection_vgf_INT():
152+
def test_CLIPTextModelWithProjection_vgf_quant():
149153
text_encoder_model, text_encoder_model_inputs = (
150154
TestCLIPTextModelWithProjection().prepare_model_and_inputs()
151155
)
@@ -155,12 +159,12 @@ def test_CLIPTextModelWithProjection_vgf_INT():
155159
text_encoder_model_inputs,
156160
aten_op=[],
157161
exir_op=[],
158-
tosa_version="TOSA-1.0+INT",
159162
use_to_edge_transform_and_lower=True,
160163
atol=0.8,
164+
quantize=True,
161165
)
162166
pipeline.change_args(
163167
"check_count.exir",
164-
TestCLIPTextModelWithProjection.ops_after_partitioner_INT,
168+
TestCLIPTextModelWithProjection.ops_after_partitioner_vgf_quantize,
165169
)
166170
pipeline.run()

‎backends/arm/test/models/stable_diffusion/test_SD3Transformer2DModel.py‎

Lines changed: 14 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,9 @@ class TestSD3Transformer2DModel:
4343
"executorch_exir_dialects_edge__ops_aten_permute_copy_default": 1,
4444
}
4545

46+
ops_after_partitioner_vgf_quantize = ops_after_partitioner_FP
47+
ops_after_partitioner_vgf_no_quantize = ops_after_partitioner_FP
48+
4649
def _prepare_inputs(
4750
self,
4851
batch_size=2,
@@ -141,7 +144,7 @@ def test_SD3Transformer2DModel_tosa_INT():
141144

142145

143146
@common.SkipIfNoModelConverter
144-
def test_SD3Transformer2DModel_vgf_FP():
147+
def test_SD3Transformer2DModel_vgf_no_quant():
145148
sd35_transformer2D_model, sd35_transformer2D_model_inputs = (
146149
TestSD3Transformer2DModel().prepare_model_and_inputs()
147150
)
@@ -151,19 +154,20 @@ def test_SD3Transformer2DModel_vgf_FP():
151154
sd35_transformer2D_model_inputs,
152155
aten_op=[],
153156
exir_op=[],
154-
tosa_version="TOSA-1.0+FP",
155157
use_to_edge_transform_and_lower=True,
156-
rtol=1.0, # TODO: MLETORCH-875: Reduce tolerance of SD3Transformer2DModel with FP and INT
158+
rtol=1.0, # TODO: MLETORCH-875: Reduce tolerance of SD3Transformer2DModel with FP and INT,
157159
atol=4.0,
160+
quantize=False,
158161
)
159162
pipeline.change_args(
160-
"check_count.exir", TestSD3Transformer2DModel.ops_after_partitioner_FP
163+
"check_count.exir",
164+
TestSD3Transformer2DModel.ops_after_partitioner_vgf_no_quantize,
161165
)
162166
pipeline.run()
163167

164168

165169
@common.SkipIfNoModelConverter
166-
def test_SD3Transformer2DModel_vgf_INT():
170+
def test_SD3Transformer2DModel_vgf_quant():
167171
sd35_transformer2D_model, sd35_transformer2D_model_inputs = (
168172
TestSD3Transformer2DModel().prepare_model_and_inputs()
169173
)
@@ -173,13 +177,14 @@ def test_SD3Transformer2DModel_vgf_INT():
173177
sd35_transformer2D_model_inputs,
174178
aten_op=[],
175179
exir_op=[],
176-
tosa_version="TOSA-1.0+INT",
177180
use_to_edge_transform_and_lower=True,
178-
qtol=1.0, # TODO: MLETORCH-875: Reduce tolerance of SD3Transformer2DModel with FP and INT
179-
rtol=1.0,
181+
qtol=1.0,
182+
rtol=1.0, # TODO: MLETORCH-875: Reduce tolerance of SD3Transformer2DModel with FP and INT,
180183
atol=4.0,
184+
quantize=True,
181185
)
182186
pipeline.change_args(
183-
"check_count.exir", TestSD3Transformer2DModel.ops_after_partitioner_INT
187+
"check_count.exir",
188+
TestSD3Transformer2DModel.ops_after_partitioner_vgf_quantize,
184189
)
185190
pipeline.run()

‎backends/arm/test/models/stable_diffusion/test_T5EncoderModel.py‎

Lines changed: 13 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,13 @@ class TestT5EncoderModel:
4444
"torch.ops.higher_order.executorch_call_delegate": 3,
4545
}
4646

47+
ops_after_partitioner_vgf_quantize = {
48+
"executorch_exir_dialects_edge__ops_dim_order_ops__to_dim_order_copy_default": 1,
49+
"torch.ops.higher_order.executorch_call_delegate": 1,
50+
}
51+
52+
ops_after_partitioner_vgf_no_quantize = ops_after_partitioner_vgf_quantize
53+
4754
def _prepare_inputs(
4855
self,
4956
batch_size=12,
@@ -110,7 +117,7 @@ def test_T5EncoderModel_tosa_INT():
110117

111118

112119
@common.SkipIfNoModelConverter
113-
def test_T5EncoderModel_vgf_FP():
120+
def test_T5EncoderModel_vgf_no_quant():
114121
t5_encoder_model, t5_encoder_model_inputs = (
115122
TestT5EncoderModel().prepare_model_and_inputs()
116123
)
@@ -120,22 +127,22 @@ def test_T5EncoderModel_vgf_FP():
120127
t5_encoder_model_inputs,
121128
aten_op=[],
122129
exir_op=[],
123-
tosa_version="TOSA-1.0+FP",
124130
use_to_edge_transform_and_lower=True,
125131
transform_passes=[
126132
ConvertInt64ConstOpsToInt32Pass(),
127133
ConvertInt64OutputOpsToInt32Pass(),
128134
InsertInt32CastsAfterInt64PlaceholdersPass(),
129135
],
136+
quantize=False,
130137
)
131138
pipeline.change_args(
132-
"check_count.exir", TestT5EncoderModel.ops_after_partitioner_FP
139+
"check_count.exir", TestT5EncoderModel.ops_after_partitioner_vgf_no_quantize
133140
)
134141
pipeline.run()
135142

136143

137144
@common.SkipIfNoModelConverter
138-
def test_T5EncoderModel_vgf_INT():
145+
def test_T5EncoderModel_vgf_quant():
139146
t5_encoder_model, t5_encoder_model_inputs = (
140147
TestT5EncoderModel().prepare_model_and_inputs()
141148
)
@@ -145,10 +152,10 @@ def test_T5EncoderModel_vgf_INT():
145152
t5_encoder_model_inputs,
146153
aten_op=[],
147154
exir_op=[],
148-
tosa_version="TOSA-1.0+INT",
149155
use_to_edge_transform_and_lower=True,
156+
quantize=True,
150157
)
151158
pipeline.change_args(
152-
"check_count.exir", TestT5EncoderModel.ops_after_partitioner_INT
159+
"check_count.exir", TestT5EncoderModel.ops_after_partitioner_vgf_quantize
153160
)
154161
pipeline.run()

‎backends/arm/test/models/stable_diffusion/test_vae_AutoencoderKL.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ def test_AutoencoderKL_tosa_INT():
8484

8585

8686
@common.SkipIfNoModelConverter
87-
def test_AutoencoderKL_vgf_FP():
87+
def test_AutoencoderKL_vgf_no_quant():
8888
auto_encoder_model, auto_encoder_model_inputs = (
8989
TestAutoencoderKL().prepare_model_and_inputs()
9090
)
@@ -94,14 +94,14 @@ def test_AutoencoderKL_vgf_FP():
9494
auto_encoder_model_inputs,
9595
aten_op=[],
9696
exir_op=[],
97-
tosa_version="TOSA-1.0+FP",
9897
use_to_edge_transform_and_lower=True,
98+
quantize=False,
9999
)
100100
pipeline.run()
101101

102102

103103
@common.SkipIfNoModelConverter
104-
def test_AutoencoderKL_vgf_INT():
104+
def test_AutoencoderKL_vgf_quant():
105105
auto_encoder_model, auto_encoder_model_inputs = (
106106
TestAutoencoderKL().prepare_model_and_inputs()
107107
)
@@ -111,8 +111,8 @@ def test_AutoencoderKL_vgf_INT():
111111
auto_encoder_model_inputs,
112112
aten_op=[],
113113
exir_op=[],
114-
tosa_version="TOSA-1.0+INT",
115114
use_to_edge_transform_and_lower=True,
116115
atol=1.0, # TODO: MLETORCH-990 Reduce tolerance of vae(AutoencoderKL) with INT
116+
quantize=True,
117117
)
118118
pipeline.run()

‎backends/arm/test/models/test_conformer.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -130,14 +130,14 @@ def test_conformer_u85_INT():
130130

131131

132132
@common.SkipIfNoModelConverter
133-
def test_conformer_vgf_INT():
133+
def test_conformer_vgf_quant():
134134
pipeline = VgfPipeline[input_t](
135135
TestConformer.conformer,
136136
TestConformer.model_example_inputs,
137137
aten_op=[],
138138
exir_op=[],
139-
tosa_version="TOSA-1.0+INT",
140139
use_to_edge_transform_and_lower=True,
140+
quantize=True,
141141
)
142142
pipeline.pop_stage("check_count.exir")
143143
pipeline.change_args(
@@ -152,13 +152,13 @@ def test_conformer_vgf_INT():
152152

153153

154154
@common.SkipIfNoModelConverter
155-
def test_conformer_vgf_FP():
155+
def test_conformer_vgf_no_quant():
156156
pipeline = VgfPipeline[input_t](
157157
TestConformer.conformer,
158158
TestConformer.model_example_inputs,
159159
aten_op=TestConformer.aten_ops,
160160
exir_op=[],
161-
tosa_version="TOSA-1.0+FP",
162161
use_to_edge_transform_and_lower=True,
162+
quantize=False,
163163
)
164164
pipeline.run()

‎backends/arm/test/models/test_deit_tiny_arm.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -93,28 +93,28 @@ def test_deit_tiny_u85_INT():
9393

9494

9595
@common.SkipIfNoModelConverter
96-
def test_deit_tiny_vgf_INT():
96+
def test_deit_tiny_vgf_quant():
9797
pipeline = VgfPipeline[input_t](
9898
deit_tiny,
9999
model_inputs,
100100
aten_op=[],
101101
exir_op=[],
102-
tosa_version="TOSA-1.0+INT",
103102
use_to_edge_transform_and_lower=True,
104103
atol=1.5,
105104
qtol=1,
105+
quantize=True,
106106
)
107107
pipeline.run()
108108

109109

110110
@common.SkipIfNoModelConverter
111-
def test_deit_tiny_vgf_FP():
111+
def test_deit_tiny_vgf_no_quant():
112112
pipeline = VgfPipeline[input_t](
113113
deit_tiny,
114114
model_inputs,
115115
aten_op=[],
116116
exir_op=[],
117-
tosa_version="TOSA-1.0+FP",
118117
use_to_edge_transform_and_lower=True,
118+
quantize=False,
119119
)
120120
pipeline.run()

‎backends/arm/test/models/test_dl3_arm.py‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -89,15 +89,15 @@ def test_dl3_u85_INT():
8989

9090

9191
@common.SkipIfNoModelConverter
92-
def test_dl3_vgf_INT():
92+
def test_dl3_vgf_quant():
9393
pipeline = VgfPipeline[input_t](
9494
TestDl3.dl3,
9595
TestDl3.model_example_inputs,
9696
aten_op=[],
9797
exir_op=[],
98-
tosa_version="TOSA-1.0+INT",
9998
use_to_edge_transform_and_lower=True,
100-
run_on_vulkan_runtime=True, # TODO: run on vulkan runtime
99+
run_on_vulkan_runtime=True,
100+
quantize=True,
101101
)
102102
pipeline.change_args(
103103
"run_method_and_compare_outputs", rtol=0.1, atol=0.1
@@ -106,13 +106,13 @@ def test_dl3_vgf_INT():
106106

107107

108108
@common.SkipIfNoModelConverter
109-
def test_dl3_vgf_FP():
109+
def test_dl3_vgf_no_quant():
110110
pipeline = VgfPipeline[input_t](
111111
TestDl3.dl3,
112112
TestDl3.model_example_inputs,
113113
aten_op=[],
114114
exir_op=[],
115-
tosa_version="TOSA-1.0+FP",
116115
use_to_edge_transform_and_lower=True,
116+
quantize=False,
117117
)
118118
pipeline.run()

‎backends/arm/test/models/test_inception_v3_arm.py‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -93,28 +93,28 @@ def test_ic3_u85_BI():
9393
@pytest.mark.slow
9494
@pytest.mark.skip(reason="Takes too long to run on CI")
9595
@common.SkipIfNoModelConverter
96-
def test_ic3_vgf_FP():
96+
def test_ic3_vgf_no_quant():
9797
pipeline = VgfPipeline[input_t](
9898
ic3,
9999
model_inputs,
100100
aten_op=[],
101101
exir_op=[],
102-
tosa_version="TOSA-1.0+FP",
103102
use_to_edge_transform_and_lower=True,
103+
quantize=False,
104104
)
105105
pipeline.run()
106106

107107

108108
@pytest.mark.slow
109109
@pytest.mark.skip(reason="Takes too long to run on CI")
110110
@common.SkipIfNoModelConverter
111-
def test_ic3_vgf_INT():
111+
def test_ic3_vgf_quant():
112112
pipeline = VgfPipeline[input_t](
113113
ic3,
114114
model_inputs,
115115
aten_op=[],
116116
exir_op=[],
117-
tosa_version="TOSA-1.0+INT",
118117
use_to_edge_transform_and_lower=True,
118+
quantize=True,
119119
)
120120
pipeline.run()

0 commit comments

Comments
 (0)