Merge pull request openvinotoolkit#1657 from IRDonch/yolo-auto

Support per-model pre-convert scripts and use it for yolo-v2-tiny-tf

ci/get-jobs-for-changes.py

@@ -52,13 +52,17 @@ def main():
     jobs = {}
 
     for changed_file in changed_files:
-        if models_dir in changed_file.parents and changed_file.name == "model.yml":
+        if models_dir in changed_file.parents and \
+                (changed_file.name == "model.yml" or changed_file.suffix == ".py"):
             if Path(changed_file).exists(): # it might've been deleted in the branch
-                jobs.setdefault("models", []).append(changed_file.parent.name)
+                jobs.setdefault("models", set()).add(changed_file.parent.name)
             else:
                 # make sure no models.lst files reference the deleted model
                 jobs["models_lst"] = True
 
+    if "models" in jobs:
+        jobs["models"] = sorted(jobs["models"]) # JSON can't work with a set
+
     git_check_attr_output = subprocess.run(
         ["git", "check-attr", "--stdin", "-z", "--all"],
         input=git_diff_output, stdout=subprocess.PIPE, check=True).stdout

models/public/yolo-v2-tiny-tf/model.yml

@@ -18,17 +18,57 @@ description: >-
   to TensorFlow* framework. This model was pretrained on COCO* dataset with 80 classes.
 task_type: detection
 files:
-  - name: yolo-v2-tiny.pb
-    size: 44962543
-    sha256: 84ccafde4a5ae0d41a0f201342814718bd35bac52492a0e72c175fecf4231e58
-    source: https://download.01.org/opencv/public_models/082020/yolo-v2-tiny-tf/yolo-v2-tiny.pb
+  - name: yolov2-tiny.weights
+    size: 44948600
+    sha256: 16f4e870f1aed83f0089cb69bfda6b53cb7b2a4a01721be56eaf5c899dfac45f
+    source: https://pjreddie.com/media/files/yolov2-tiny.weights
+  - name: keras-YOLOv3-model-set/tools/model_converter/convert.py
+    size: 12937
+    sha256: 12350fbfa942b66fad8a80af100bc14605c9cdcd8508f3eb5cf26647f64d733e
+    source: https://github.com/david8862/keras-YOLOv3-model-set/raw/d38c3d865f7190ee9b19a30e91f2b750a31320c1/tools/model_converter/convert.py
+  - name: keras-YOLOv3-model-set/tools/model_converter/keras_to_tensorflow.py
+    size: 8370
+    sha256: 1cc02cf4ec76760c05bc1635654247ba56b9a1864b63fd05534b836bc93e44e7
+    source: https://github.com/david8862/keras-YOLOv3-model-set/raw/d38c3d865f7190ee9b19a30e91f2b750a31320c1/tools/model_converter/keras_to_tensorflow.py
+  - name: keras-YOLOv3-model-set/common/utils.py
+    size: 5002
+    sha256: 90146775879f294e1f1a82c7e35a5be29d815b7a9b14dbe12ba29dd16b3d10a8
+    source: https://github.com/david8862/keras-YOLOv3-model-set/raw/d38c3d865f7190ee9b19a30e91f2b750a31320c1/common/utils.py
+  - name: keras-YOLOv3-model-set/cfg/yolov2-tiny.cfg
+    size: 1488
+    sha256: 58e8f4390c8080a90a40f3b7ec5868ee9ae32519f3c08f57d104e494de04b34d
+    source: https://github.com/david8862/keras-YOLOv3-model-set/raw/d38c3d865f7190ee9b19a30e91f2b750a31320c1/cfg/yolov2-tiny.cfg
+postprocessing:
+  # disable imports that aren't needed for this model and code that uses them
+  - $type: regex_replace
+    file: keras-YOLOv3-model-set/tools/model_converter/convert.py
+    pattern: 'from yolo4\.'
+    replacement: '# \g<0>'
+  - $type: regex_replace
+    file: keras-YOLOv3-model-set/common/utils.py
+    pattern: 'import cv2,|from (matplotlib|PIL|common\.backbones|yolo4)\b'
+    replacement: '# \g<0>'
+  - $type: regex_replace
+    file: keras-YOLOv3-model-set/common/utils.py
+    pattern: '''(swish|hard_\w+|mish)'''
+    replacement: '# \g<0>'
+
+  # patch TensorFlow 2 compatibility
+  - $type: regex_replace
+    file: keras-YOLOv3-model-set/tools/model_converter/keras_to_tensorflow.py
+    pattern: 'import tensorflow as tf'
+    replacement: 'import tensorflow.compat.v1 as tf\ntf.disable_v2_behavior()'
+  - $type: regex_replace
+    file: keras-YOLOv3-model-set/tools/model_converter/keras_to_tensorflow.py
+    pattern: 'from tensorflow\.keras\b'
+    replacement: 'from tensorflow.compat.v1.keras'
 model_optimizer_args:
   - --input_shape=[1,416,416,3]
   - --input=image_input
   - --scale_values=image_input[255]
   - --reverse_input_channels
   - --transformations_config=$mo_dir/extensions/front/tf/yolo_v2_tiny.json
-  - --input_model=$dl_dir/yolo-v2-tiny.pb
+  - --input_model=$conv_dir/yolo-v2-tiny.pb
 framework: tf
 quantizable: yes
 license: https://raw.githubusercontent.com/david8862/keras-YOLOv3-model-set/master/LICENSE

models/public/yolo-v2-tiny-tf/pre-convert.py

@@ -0,0 +1,44 @@
+#!/usr/bin/env python3
+
+# Copyright (c) 2020 Intel Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import argparse
+import subprocess
+import sys
+
+from pathlib import Path
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument('input_dir', type=Path)
+    parser.add_argument('output_dir', type=Path)
+    args = parser.parse_args()
+
+    subprocess.run([sys.executable, '--',
+        str(args.input_dir / 'keras-YOLOv3-model-set/tools/model_converter/convert.py'),
+        '--',
+        str(args.input_dir / 'keras-YOLOv3-model-set/cfg/yolov2-tiny.cfg'),
+        str(args.input_dir / 'yolov2-tiny.weights'),
+        str(args.output_dir / 'yolo-v2-tiny.h5'),
+    ], check=True)
+
+    subprocess.run([sys.executable, '--',
+        str(args.input_dir / 'keras-YOLOv3-model-set/tools/model_converter/keras_to_tensorflow.py'),
+        '--input_model={}'.format(args.output_dir / 'yolo-v2-tiny.h5'),
+        '--output_model={}'.format(args.output_dir / 'yolo-v2-tiny.pb'),
+    ], check=True);
+
+if __name__ == '__main__':
+    main()

models/public/yolo-v2-tiny-tf/yolo-v2-tiny-tf.md

@@ -4,25 +4,6 @@
 
 YOLO v2 Tiny is a real-time object detection model implemented with Keras\* from this [repository](https://github.com/david8862/keras-YOLOv3-model-set) and converted to TensorFlow\* framework. This model was pretrained on COCO\* dataset with 80 classes.
 
-## Conversion
-
-1. Download or clone the original [repository](https://github.com/david8862/keras-YOLOv3-model-set) (tested on `d38c3d8` commit).
-2. Use the following commands to get original model (named `yolov2_tiny` in repository) and convert it to Keras\* format (see details in the [README.md](https://github.com/david8862/keras-YOLOv3-model-set/blob/d38c3d865f7190ee9b19a30e91f2b750a31320c1/README.md)  file in the official repository):
-
-   1. Download YOLO v2 Tiny weights:
-        ```
-        wget -O weights/yolov2-tiny.weights https://pjreddie.com/media/files/yolov2-tiny.weights
-        ```
-
-   2. Convert model weights to Keras\*:
-        ```
-        python tools/model_converter/convert.py cfg/yolov2-tiny.cfg weights/yolov2-tiny.weights weights/yolov2-tiny.h5
-        ```
-3. Convert model to protobuf:
-    ```
-    python tools/model_converter/keras_to_tensorflow.py --input_model weights/yolov2-tiny.h5 --output_model=weights/yolo-v2-tiny.pb
-    ```
-
 ## Specification
 
 | Metric            | Value         |

tools/downloader/converter.py

@@ -25,6 +25,25 @@
 import common
 
 
+def run_pre_convert(reporter, model, output_dir, args):
+    script = common.MODEL_ROOT / model.subdirectory / 'pre-convert.py'
+    if not script.exists():
+        return True
+
+    reporter.print_section_heading('{}Running pre-convert script for {}',
+        '(DRY RUN) ' if args.dry_run else '', model.name)
+
+    cmd = [str(args.python), '--', str(script), '--',
+        str(args.download_dir / model.subdirectory), str(output_dir / model.subdirectory)]
+
+    reporter.print('Pre-convert command: {}', common.command_string(cmd))
+    reporter.print(flush=True)
+
+    success = True if args.dry_run else reporter.job_context.subprocess(cmd)
+    reporter.print()
+
+    return success
+
 def convert_to_onnx(reporter, model, output_dir, args):
     reporter.print_section_heading('{}Converting {} to ONNX',
         '(DRY RUN) ' if args.dry_run else '', model.name)
@@ -119,6 +138,11 @@ def convert(reporter, model):
             reporter.print()
             return True
 
+        (output_dir / model.subdirectory).mkdir(parents=True, exist_ok=True)
+
+        if not run_pre_convert(reporter, model, output_dir, args):
+            return False
+
         model_format = model.framework
 
         if model.conversion_to_onnx_args: