Update accuracy checker for OpenVINO converted models (#3995)

* Auto detect RCNN network outputs

* Add handling of custom ouput order for some ssd models

* Document new 'custom ouput order' parameter for ssd adapter

* Add autoconvertion of output with NHWC to NCHW layout for yolo adapter

* Add ouput_layout param to yolo_v3 adapter

Author: pwolnows

tools/accuracy_checker/accuracy_checker/adapters/README.md

@@ -68,6 +68,7 @@ AccuracyChecker supports following set of adapters:
   * `outputs` - the list of output layers names.
   * `raw_output` - enabling additional preprocessing for raw YOLO output format (default `False`).
   * `output_format` - setting output layer format - boxes first (`BHW`)(default, also default for generated IRs), boxes last (`HWB`). Applicable only if network output not 3D (4D with batch) tensor.
+  * `output_layout` - setting output layout - channel first (`NCHW`)(default), channel last (`NHWC`).
   * `cells` - sets grid size for each layer, according `outputs` filed. Works only with `do_reshape=True` or when output tensor dimensions not equal 3.
   * `do_reshape` - forces reshape output tensor to [B,Cy,Cx] or [Cy,Cx,B] format, depending on `output_format` value ([B,Cy,Cx] by default). You may need to specify `cells` value.
   * `transpose` - transpose output tensor to specified format (optional).
@@ -107,6 +108,7 @@ AccuracyChecker supports following set of adapters:
   * `vocabulary_file` - file with recognition symbols for decoding.
   * `remove_duplicates` - allow removing of duplicated symbols (Optional, default value - `True`).
 * `ssd` - converting  output of SSD model to `DetectionPrediction` representation.
+  * `custom_output_order` - Use custom output data order: bbox, score, label (Optional, default `False`).
 * `ssd_mxnet` - converting output of SSD-based models from MXNet framework to `DetectionPrediction` representation.
 * `pytorch_ssd_decoder` - converts output of SSD model from PyTorch without embedded decoder.
   * `scores_out` - name of output layer with bounding boxes scores.

tools/accuracy_checker/accuracy_checker/adapters/mask_rcnn.py

@@ -84,14 +84,14 @@ def is_box_outputs(config, box_outputs):
             for elem in box_outputs:
                 if not config.get(elem):
                     return False
-
             return True
 
         box_outputs = ['classes_out', 'scores_out', 'boxes_out']
         if is_detection_out(self.launcher_config) and is_box_outputs(self.launcher_config, box_outputs):
             raise ConfigError('only detection output or [{}] should be provided'.format(', '.join(box_outputs)))
 
         self.raw_masks_out = self.get_value_from_config('raw_masks_out')
+        self.outputs_verified = False
 
         if is_detection_out(self.launcher_config):
             self.detection_out = self.get_value_from_config('detection_out')
@@ -106,12 +106,28 @@ def is_box_outputs(config, box_outputs):
                 if not is_box_outputs(self.launcher_config, box_outputs):
                     raise ConfigError('all related outputs should be specified: {}'.format(', '.join(box_outputs)))
                 self.realisation = self._process_tf_obj_detection_api_outputs
+                self.outputs_verified = False
                 return
 
             self.realisation = self._process_pytorch_outputs
-        self.outputs_verified = False
+
+    def assign_matching_outputs_names(self, outputs):
+        def _get_matching_output(name):
+            return [output for output in outputs if name in output][0]
+
+        self.num_detections_out = _get_matching_output('num_detections')
+        self.boxes_out = _get_matching_output('boxes')
+        self.raw_masks_out = _get_matching_output('masks')
+        self.scores_out = _get_matching_output('scores')
+        self.classes_out = _get_matching_output('classes')
 
     def select_output_blob(self, outputs):
+        # handle case where config has detection_out and raw_masks_out but model has different outputs
+        if hasattr(self, 'detection_out') and len(outputs) == 5:
+            self.assign_matching_outputs_names(outputs)
+            self.realisation = self._process_tf_obj_detection_api_outputs
+            self.outputs_verified = True
+            return
         if self.raw_masks_out:
             self.raw_masks_out = self.check_output_name(self.raw_masks_out, outputs)
         if hasattr(self, 'detection_out'):

tools/accuracy_checker/accuracy_checker/adapters/ssd.py

@@ -33,8 +33,19 @@ class SSDAdapter(Adapter):
     __provider__ = 'ssd'
     prediction_types = (DetectionPrediction, )
 
+    @classmethod
+    def parameters(cls):
+        parameters = super().parameters()
+        parameters.update({
+            'custom_output_order': BoolField(
+                optional=True, default=False,
+                description='Use custom output data order: bbox, score, label')
+        })
+        return parameters
+
     def configure(self):
         super().configure()
+        self.custom_output_order = self.get_value_from_config('custom_output_order')
         self.outputs_verified = False
 
     def select_output_blob(self, outputs):
@@ -62,7 +73,12 @@ def process(self, raw, identifiers, frame_meta):
             prediction_mask = np.where(prediction_batch[:, 0] == batch_index)
             detections = prediction_batch[prediction_mask]
             detections = detections[:, 1::]
-            result.append(DetectionPrediction(identifier, *zip(*detections)))
+            if self.custom_output_order:
+                y_mins, x_mins, y_maxs, x_maxs, scores, labels = detections.T
+            else:
+                labels, scores, x_mins, y_mins, x_maxs, y_maxs = detections.T
+
+            result.append(DetectionPrediction(identifier, labels, scores, x_mins, y_mins, x_maxs, y_maxs))
 
         return result
 

tools/accuracy_checker/accuracy_checker/adapters/yolo.py

@@ -273,6 +273,7 @@ def process(self, raw, identifiers, frame_meta):
         predictions = predictions[self.output_blob]
         out_precision = frame_meta[0].get('output_precision', {})
         out_layout = frame_meta[0].get('output_layout', {})
+
         if self.output_blob in out_precision and predictions.dtype != out_precision[self.output_blob]:
             predictions = predictions.view(out_precision[self.output_blob])
         if self.output_blob in out_layout and out_layout[self.output_blob] == 'NHWC':
@@ -370,6 +371,10 @@ def parameters(cls):
                 choices=['BHW', 'HWB'], optional=True,
                 description="Set output layer format", default='BHW',
             ),
+            'output_layout': StringField(
+                choices=['NCHW', 'NHWC'], optional=True, default='NCHW',
+                description="Set output layout format"
+            ),
             'multiple_labels': BoolField(
                 optional=True, default=False,
                 description="Allow multiple labels for detection objects"
@@ -420,6 +425,7 @@ def configure(self):
 
         self.raw_output = self.get_value_from_config('raw_output')
         self.output_format = self.get_value_from_config('output_format')
+        self.output_layout = self.get_value_from_config('output_layout')
         if self.raw_output:
             self.processor = YoloOutputProcessor(coord_correct=lambda x: 1.0 / (1.0 + np.exp(-x)),
                                                  conf_correct=lambda x: 1.0 / (1.0 + np.exp(-x)),
@@ -517,6 +523,10 @@ def prepare_predictions(self, batch, raw_outputs, out_precision, out_layout):
             if blob in out_layout and out_layout[blob] == 'NHWC':
                 shape = out_blob.shape
                 out_blob = np.transpose(out_blob, (0, 3, 1, 2)).reshape(shape)
+            elif  self.output_layout == 'NHWC' and len(out_blob.shape) == 4:
+                # layout is NHWC turn it to NCHW
+                out_blob = np.transpose(out_blob, (0, 3, 1, 2))
+
             if batch == 1 and out_blob.shape[0] != batch:
                 out_blob = np.expand_dims(out_blob, 0)