diff --git a/ci/get-jobs-for-changes.py b/ci/get-jobs-for-changes.py index f690e0b3cfb..7a21a958d96 100755 --- a/ci/get-jobs-for-changes.py +++ b/ci/get-jobs-for-changes.py @@ -52,13 +52,17 @@ def main(): jobs = {} for changed_file in changed_files: - if models_dir in changed_file.parents and changed_file.name == "model.yml": + if models_dir in changed_file.parents and \ + (changed_file.name == "model.yml" or changed_file.suffix == ".py"): if Path(changed_file).exists(): # it might've been deleted in the branch - jobs.setdefault("models", []).append(changed_file.parent.name) + jobs.setdefault("models", set()).add(changed_file.parent.name) else: # make sure no models.lst files reference the deleted model jobs["models_lst"] = True + if "models" in jobs: + jobs["models"] = sorted(jobs["models"]) # JSON can't work with a set + git_check_attr_output = subprocess.run( ["git", "check-attr", "--stdin", "-z", "--all"], input=git_diff_output, stdout=subprocess.PIPE, check=True).stdout diff --git a/models/public/yolo-v2-tiny-tf/model.yml b/models/public/yolo-v2-tiny-tf/model.yml index 4b3d19d7be9..fb3846ca85f 100644 --- a/models/public/yolo-v2-tiny-tf/model.yml +++ b/models/public/yolo-v2-tiny-tf/model.yml @@ -18,17 +18,57 @@ description: >- to TensorFlow* framework. This model was pretrained on COCO* dataset with 80 classes. task_type: detection files: - - name: yolo-v2-tiny.pb - size: 44962543 - sha256: 84ccafde4a5ae0d41a0f201342814718bd35bac52492a0e72c175fecf4231e58 - source: https://download.01.org/opencv/public_models/082020/yolo-v2-tiny-tf/yolo-v2-tiny.pb + - name: yolov2-tiny.weights + size: 44948600 + sha256: 16f4e870f1aed83f0089cb69bfda6b53cb7b2a4a01721be56eaf5c899dfac45f + source: https://pjreddie.com/media/files/yolov2-tiny.weights + - name: keras-YOLOv3-model-set/tools/model_converter/convert.py + size: 12937 + sha256: 12350fbfa942b66fad8a80af100bc14605c9cdcd8508f3eb5cf26647f64d733e + source: https://github.com/david8862/keras-YOLOv3-model-set/raw/d38c3d865f7190ee9b19a30e91f2b750a31320c1/tools/model_converter/convert.py + - name: keras-YOLOv3-model-set/tools/model_converter/keras_to_tensorflow.py + size: 8370 + sha256: 1cc02cf4ec76760c05bc1635654247ba56b9a1864b63fd05534b836bc93e44e7 + source: https://github.com/david8862/keras-YOLOv3-model-set/raw/d38c3d865f7190ee9b19a30e91f2b750a31320c1/tools/model_converter/keras_to_tensorflow.py + - name: keras-YOLOv3-model-set/common/utils.py + size: 5002 + sha256: 90146775879f294e1f1a82c7e35a5be29d815b7a9b14dbe12ba29dd16b3d10a8 + source: https://github.com/david8862/keras-YOLOv3-model-set/raw/d38c3d865f7190ee9b19a30e91f2b750a31320c1/common/utils.py + - name: keras-YOLOv3-model-set/cfg/yolov2-tiny.cfg + size: 1488 + sha256: 58e8f4390c8080a90a40f3b7ec5868ee9ae32519f3c08f57d104e494de04b34d + source: https://github.com/david8862/keras-YOLOv3-model-set/raw/d38c3d865f7190ee9b19a30e91f2b750a31320c1/cfg/yolov2-tiny.cfg +postprocessing: + # disable imports that aren't needed for this model and code that uses them + - $type: regex_replace + file: keras-YOLOv3-model-set/tools/model_converter/convert.py + pattern: 'from yolo4\.' + replacement: '# \g<0>' + - $type: regex_replace + file: keras-YOLOv3-model-set/common/utils.py + pattern: 'import cv2,|from (matplotlib|PIL|common\.backbones|yolo4)\b' + replacement: '# \g<0>' + - $type: regex_replace + file: keras-YOLOv3-model-set/common/utils.py + pattern: '''(swish|hard_\w+|mish)''' + replacement: '# \g<0>' + + # patch TensorFlow 2 compatibility + - $type: regex_replace + file: keras-YOLOv3-model-set/tools/model_converter/keras_to_tensorflow.py + pattern: 'import tensorflow as tf' + replacement: 'import tensorflow.compat.v1 as tf\ntf.disable_v2_behavior()' + - $type: regex_replace + file: keras-YOLOv3-model-set/tools/model_converter/keras_to_tensorflow.py + pattern: 'from tensorflow\.keras\b' + replacement: 'from tensorflow.compat.v1.keras' model_optimizer_args: - --input_shape=[1,416,416,3] - --input=image_input - --scale_values=image_input[255] - --reverse_input_channels - --transformations_config=$mo_dir/extensions/front/tf/yolo_v2_tiny.json - - --input_model=$dl_dir/yolo-v2-tiny.pb + - --input_model=$conv_dir/yolo-v2-tiny.pb framework: tf quantizable: yes license: https://raw.githubusercontent.com/david8862/keras-YOLOv3-model-set/master/LICENSE diff --git a/models/public/yolo-v2-tiny-tf/pre-convert.py b/models/public/yolo-v2-tiny-tf/pre-convert.py new file mode 100755 index 00000000000..680980c36f9 --- /dev/null +++ b/models/public/yolo-v2-tiny-tf/pre-convert.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 + +# Copyright (c) 2020 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import subprocess +import sys + +from pathlib import Path + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('input_dir', type=Path) + parser.add_argument('output_dir', type=Path) + args = parser.parse_args() + + subprocess.run([sys.executable, '--', + str(args.input_dir / 'keras-YOLOv3-model-set/tools/model_converter/convert.py'), + '--', + str(args.input_dir / 'keras-YOLOv3-model-set/cfg/yolov2-tiny.cfg'), + str(args.input_dir / 'yolov2-tiny.weights'), + str(args.output_dir / 'yolo-v2-tiny.h5'), + ], check=True) + + subprocess.run([sys.executable, '--', + str(args.input_dir / 'keras-YOLOv3-model-set/tools/model_converter/keras_to_tensorflow.py'), + '--input_model={}'.format(args.output_dir / 'yolo-v2-tiny.h5'), + '--output_model={}'.format(args.output_dir / 'yolo-v2-tiny.pb'), + ], check=True); + +if __name__ == '__main__': + main() diff --git a/models/public/yolo-v2-tiny-tf/yolo-v2-tiny-tf.md b/models/public/yolo-v2-tiny-tf/yolo-v2-tiny-tf.md index 91173a2039d..a3b94c2a8d4 100644 --- a/models/public/yolo-v2-tiny-tf/yolo-v2-tiny-tf.md +++ b/models/public/yolo-v2-tiny-tf/yolo-v2-tiny-tf.md @@ -4,25 +4,6 @@ YOLO v2 Tiny is a real-time object detection model implemented with Keras\* from this [repository](https://github.com/david8862/keras-YOLOv3-model-set) and converted to TensorFlow\* framework. This model was pretrained on COCO\* dataset with 80 classes. -## Conversion - -1. Download or clone the original [repository](https://github.com/david8862/keras-YOLOv3-model-set) (tested on `d38c3d8` commit). -2. Use the following commands to get original model (named `yolov2_tiny` in repository) and convert it to Keras\* format (see details in the [README.md](https://github.com/david8862/keras-YOLOv3-model-set/blob/d38c3d865f7190ee9b19a30e91f2b750a31320c1/README.md) file in the official repository): - - 1. Download YOLO v2 Tiny weights: - ``` - wget -O weights/yolov2-tiny.weights https://pjreddie.com/media/files/yolov2-tiny.weights - ``` - - 2. Convert model weights to Keras\*: - ``` - python tools/model_converter/convert.py cfg/yolov2-tiny.cfg weights/yolov2-tiny.weights weights/yolov2-tiny.h5 - ``` -3. Convert model to protobuf: - ``` - python tools/model_converter/keras_to_tensorflow.py --input_model weights/yolov2-tiny.h5 --output_model=weights/yolo-v2-tiny.pb - ``` - ## Specification | Metric | Value | diff --git a/tools/downloader/converter.py b/tools/downloader/converter.py index 7664205e693..a00959eeb82 100755 --- a/tools/downloader/converter.py +++ b/tools/downloader/converter.py @@ -25,6 +25,25 @@ import common +def run_pre_convert(reporter, model, output_dir, args): + script = common.MODEL_ROOT / model.subdirectory / 'pre-convert.py' + if not script.exists(): + return True + + reporter.print_section_heading('{}Running pre-convert script for {}', + '(DRY RUN) ' if args.dry_run else '', model.name) + + cmd = [str(args.python), '--', str(script), '--', + str(args.download_dir / model.subdirectory), str(output_dir / model.subdirectory)] + + reporter.print('Pre-convert command: {}', common.command_string(cmd)) + reporter.print(flush=True) + + success = True if args.dry_run else reporter.job_context.subprocess(cmd) + reporter.print() + + return success + def convert_to_onnx(reporter, model, output_dir, args): reporter.print_section_heading('{}Converting {} to ONNX', '(DRY RUN) ' if args.dry_run else '', model.name) @@ -119,6 +138,11 @@ def convert(reporter, model): reporter.print() return True + (output_dir / model.subdirectory).mkdir(parents=True, exist_ok=True) + + if not run_pre_convert(reporter, model, output_dir, args): + return False + model_format = model.framework if model.conversion_to_onnx_args: