Add tflitehub

tank/tflitehub/.gitignore

@@ -0,0 +1,2 @@
+tmp/
+.lit_test_times.txt

tank/tflitehub/README.md

@@ -0,0 +1,15 @@
+# Sample compile and execution of TFLite models
+
+This directory contains test scripts to compile/run/compare various TFLite
+models from TFHub. It aims for simplicity and hackability.
+
+Follow the instructions at the repository root to install a functioning
+python venv. Then you can just run individual python files.
+
+Or, use something like the following to collect all artifacts and traces,
+which can be fed to other tools:
+
+```
+export IREE_SAVE_TEMPS="/tmp/iree/models/{main}/{id}"
+for i in *.py; do export IREE_SAVE_CALLS=/tmp/iree/traces/$i; python $i; done
+```

tank/tflitehub/albert_test.py

@@ -0,0 +1,34 @@
+# RUN: %PYTHON %s
+
+import absl.testing
+import numpy
+import test_util
+
+model_path = "https://tfhub.dev/tensorflow/lite-model/albert_lite_base/squadv1/1?lite-format=tflite"
+
+class AlbertTest(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(AlbertTest, self).__init__(model_path, *args, **kwargs)
+
+  # Inputs modified to be useful albert inputs.
+  def generate_inputs(self, input_details):
+    for input in input_details:
+      absl.logging.info("\t%s, %s", str(input["shape"]), input["dtype"].__name__)
+
+    args = []
+    args.append(numpy.random.randint(low=0, high=256, size=input_details[0]["shape"], dtype=input_details[0]["dtype"]))
+    args.append(numpy.ones(shape=input_details[1]["shape"], dtype=input_details[1]["dtype"]))
+    args.append(numpy.zeros(shape=input_details[2]["shape"], dtype=input_details[2]["dtype"]))
+    return args
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(AlbertTest, self).compare_results(iree_results, tflite_results, details)
+    self.assertTrue(numpy.isclose(iree_results[0], tflite_results[0], atol=1e-4).all())
+    self.assertTrue(numpy.isclose(iree_results[1], tflite_results[1], atol=1e-4).all())
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+

tank/tflitehub/asr_conformer_test.py

@@ -0,0 +1,19 @@
+# RUN: %PYTHON %s
+# XFAIL: *
+
+import absl.testing
+import test_util
+
+model_path = "https://tfhub.dev/neso613/lite-model/ASR_TFLite/pre_trained_models/English/1?lite-format=tflite"
+
+# Failure is due to dynamic shapes:
+# - Some improvements to tfl.strided_slice lowering are next steps
+class AsrConformerTest(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(AsrConformerTest, self).__init__(model_path, *args, **kwargs)
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()

tank/tflitehub/bird_classifier_test.py

@@ -0,0 +1,34 @@
+# RUN: %PYTHON %s
+
+import absl.testing
+import numpy
+import test_util
+import urllib.request
+
+from PIL import Image
+
+model_path = "https://tfhub.dev/google/lite-model/aiy/vision/classifier/birds_V1/3?lite-format=tflite"
+
+class BirdClassifierTest(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(BirdClassifierTest, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(BirdClassifierTest, self).compare_results(iree_results, tflite_results, details)
+    self.assertTrue(numpy.isclose(iree_results[0], tflite_results[0], atol=1e-3).all())
+
+  def generate_inputs(self, input_details):
+    img_path = "https://github.com/google-coral/test_data/raw/master/bird.bmp"
+    local_path = "/".join([self.workdir, "bird.bmp"])
+    urllib.request.urlretrieve(img_path, local_path)
+
+    shape = input_details[0]["shape"]
+    im = numpy.array(Image.open(local_path).resize((shape[1], shape[2])))
+    args = [im.reshape(shape)]
+    return args
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()

tank/tflitehub/cartoon_gan_test.py

@@ -0,0 +1,19 @@
+# RUN: %PYTHON %s
+# REQUIRES: hugetest
+
+import absl.testing
+import test_util
+
+model_path = "https://tfhub.dev/sayakpaul/lite-model/cartoongan/dr/1?lite-format=tflite"
+
+class CartoonGanTest(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(CartoonGanTest, self).__init__(model_path, *args, **kwargs)
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+
+

tank/tflitehub/coco_test_data.py

@@ -0,0 +1,15 @@
+import numpy as np
+import urllib.request
+
+from PIL import Image
+
+# Returns a sample image in the COCO 2017 dataset in uint8.
+def generate_input(workdir, input_details):
+    # We use an image of a bear since this is an easy example.
+    img_path = "https://storage.googleapis.com/iree-model-artifacts/coco_2017_000000000285.jpg"
+    local_path = "/".join([workdir, "coco_2017_000000000285.jpg"])
+    urllib.request.urlretrieve(img_path, local_path)
+
+    shape = input_details[0]["shape"]
+    im = np.array(Image.open(local_path).resize((shape[1], shape[2])))
+    return im.reshape(shape)

tank/tflitehub/craft_text_test.py

@@ -0,0 +1,24 @@
+# RUN: %PYTHON %s
+# XFAIL: *
+
+import absl.testing
+import test_util
+
+model_path = "https://tfhub.dev/tulasiram58827/lite-model/craft-text-detector/dr/1?lite-format=tflite"
+
+# Failure: Resize lowering does not handle inferred dynamic shapes. Furthermore, the entire model
+# requires dynamic shape support.
+class CraftTextTest(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(CraftTextTest, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(CraftTextTest, self).compare_results(iree_results, tflite_results, details)
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+
+

tank/tflitehub/deeplab_v3_test.py

@@ -0,0 +1,22 @@
+# RUN: %PYTHON %s
+
+import absl.testing
+import numpy
+import test_util
+
+model_path = "https://tfhub.dev/tensorflow/lite-model/deeplabv3/1/metadata/2?lite-format=tflite"
+
+class DeepLabV3Test(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(DeepLabV3Test, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(DeepLabV3Test, self).compare_results(iree_results, tflite_results, details)
+    self.assertTrue(numpy.isclose(iree_results[0], tflite_results[0], atol=1e-3).all())
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+

tank/tflitehub/densenet_test.py

@@ -0,0 +1,22 @@
+# RUN: %PYTHON %s
+
+import absl.testing
+import numpy
+import test_util
+
+model_path = "https://tfhub.dev/tensorflow/lite-model/densenet/1/metadata/1?lite-format=tflite"
+
+class DenseNetTest(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(DenseNetTest, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(DenseNetTest, self).compare_results(iree_results, tflite_results, details)
+    self.assertTrue(numpy.isclose(iree_results[0], tflite_results[0], atol=1e-5).all())
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+

tank/tflitehub/east_text_detector_test.py

@@ -0,0 +1,34 @@
+# RUN: %PYTHON %s
+
+import absl.testing
+import numpy
+import test_util
+
+model_path = "https://tfhub.dev/sayakpaul/lite-model/east-text-detector/dr/1?lite-format=tflite"
+
+class EastTextDetectorTest(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(EastTextDetectorTest, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(EastTextDetectorTest, self).compare_results(iree_results, tflite_results, details)
+    self.assertTrue(numpy.isclose(iree_results[0], tflite_results[0], atol=1e-3).all())
+
+    # The second return is extremely noisy as it is not a binary classification. To handle we
+    # check normalized correlation with an expectation of "close enough".
+    iree_norm = numpy.sqrt(iree_results[1] * iree_results[1])
+    tflite_norm = numpy.sqrt(tflite_results[1] * tflite_results[1])
+
+    correlation = numpy.average(iree_results[1] * tflite_results[1] / iree_norm / tflite_norm)
+    self.assertTrue(numpy.isclose(correlation, 1.0, atol=1e-2).all())
+
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+
+
+
+

tank/tflitehub/efficientnet_lite0_int8_test.py

@@ -0,0 +1,31 @@
+# RUN: %PYTHON %s
+
+import absl.testing
+import imagenet_test_data
+import numpy
+import test_util
+
+# Source https://tfhub.dev/tensorflow/lite-model/efficientnet/lite0/int8/2
+model_path = "https://storage.googleapis.com/iree-model-artifacts/efficientnet_lite0_int8_2.tflite"
+
+class EfficientnetLite0Int8Test(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(EfficientnetLite0Int8Test, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(EfficientnetLite0Int8Test, self).compare_results(iree_results, tflite_results, details)
+    # Dequantize outputs.
+    zero_point = details[0]['quantization_parameters']['zero_points'][0]
+    scale = details[0]['quantization_parameters']['scales'][0]
+    dequantized_iree_results = (iree_results - zero_point) * scale
+    dequantized_tflite_results = (tflite_results - zero_point) * scale
+    self.assertTrue(numpy.isclose(dequantized_iree_results, dequantized_tflite_results, atol=5e-3).all())
+
+  def generate_inputs(self, input_details):
+    return [imagenet_test_data.generate_input(self.workdir, input_details)]
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()

tank/tflitehub/efficientnet_lite0_test.py

@@ -0,0 +1,29 @@
+# RUN: %PYTHON %s
+
+import absl.testing
+import imagenet_test_data
+import numpy
+import test_util
+
+# Source https://tfhub.dev/tensorflow/lite-model/efficientnet/lite0/fp32/2
+model_path = "https://storage.googleapis.com/iree-model-artifacts/efficientnet_lite0_fp32_2.tflite"
+
+class EfficientnetLite0Test(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(EfficientnetLite0Test, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(EfficientnetLite0Test, self).compare_results(iree_results, tflite_results, details)
+    self.assertTrue(numpy.isclose(iree_results, tflite_results, atol=1e-4).all())
+
+  def generate_inputs(self, input_details):
+    inputs = imagenet_test_data.generate_input(self.workdir, input_details)
+    # Normalize inputs to [-1, 1].
+    inputs = (inputs.astype('float32') / 127.5) - 1
+    return [inputs]
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()

tank/tflitehub/efficientnet_test.py

@@ -0,0 +1,30 @@
+# RUN: %PYTHON %s
+# XFAIL: *
+
+import absl.testing
+import imagenet_test_data
+import numpy
+import test_util
+
+# Source https://tfhub.dev/sayannath/lite-model/image-scene/1
+model_path = "https://storage.googleapis.com/iree-model-artifacts/efficientnet_224_fp32.tflite"
+
+class EfficientnetTest(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(EfficientnetTest, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(EfficientnetTest, self).compare_results(iree_results, tflite_results, details)
+    self.assertTrue(numpy.isclose(iree_results, tflite_results, atol=1e-4).all())
+
+  def generate_inputs(self, input_details):
+    inputs = imagenet_test_data.generate_input(self.workdir, input_details)
+    # Normalize inputs to [-1, 1].
+    inputs = (inputs.astype('float32') / 127.5) - 1
+    return [inputs]
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()

tank/tflitehub/gpt2_test.py

@@ -0,0 +1,31 @@
+# RUN: %PYTHON %s
+# REQUIRES: horrendoustest
+
+import absl.testing
+import numpy
+import test_util
+
+model_path = "https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-64.tflite"
+
+# This test is a massive download and excluded due to causing timeouts.
+class GPT2Test(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(GPT2Test, self).__init__(model_path, *args, **kwargs)
+
+  # Inputs modified to be useful mobilebert inputs.
+  def generate_inputs(self, input_details):
+    args = []
+    args.append(numpy.random.randint(low=0, high=256, size=input_details[0]["shape"], dtype=input_details[0]["dtype"]))
+    return args
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(GPT2Test, self).compare_results(iree_results, tflite_results, details)
+    for i in range(len(iree_results)):
+      self.assertTrue(numpy.isclose(iree_results[i], tflite_results[i], atol=5e-3).all())
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+

tank/tflitehub/image_stylization_test.py

@@ -0,0 +1,25 @@
+# RUN: %PYTHON %s
+
+import absl.testing
+import numpy
+import test_util
+
+model_path = "https://tfhub.dev/sayakpaul/lite-model/arbitrary-image-stylization-inceptionv3-dynamic-shapes/int8/predict/1?lite-format=tflite"
+
+# Failure is due to avg_pool2d.
+class ImageStylizationTest(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(ImageStylizationTest, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(ImageStylizationTest, self).compare_results(iree_results, tflite_results, details)
+    iree = iree_results[0].flatten().astype(numpy.single)
+    tflite = tflite_results[0].flatten().astype(numpy.single)
+    # Error is not tiny but appears close.
+    self.assertTrue(numpy.isclose(numpy.max(numpy.abs(iree - tflite)), 0.0, atol = 5e-2).all())
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()

tank/tflitehub/imagenet_test_data.py

@@ -0,0 +1,15 @@
+import numpy as np
+import urllib.request
+
+from PIL import Image
+
+# Returns a sample image in the Imagenet dataset in uint8.
+def generate_input(workdir, input_details):
+    # We use an image of apples since this is an easy example.
+    img_path = "https://storage.googleapis.com/iree-model-artifacts/ILSVRC2012_val_00000023.JPEG"
+    local_path = "/".join([workdir, "ILSVRC2012_val_00000023.JPEG"])
+    urllib.request.urlretrieve(img_path, local_path)
+
+    shape = input_details[0]["shape"]
+    im = np.array(Image.open(local_path).resize((shape[1], shape[2])))
+    return im.reshape(shape)

tank/tflitehub/inception_v4_test.py

@@ -0,0 +1,33 @@
+# RUN: %PYTHON %s
+
+import absl.testing
+import imagenet_test_data
+import numpy
+import test_util
+
+# Source https://tfhub.dev/tensorflow/lite-model/inception_v4/1/default/1
+model_path = "https://storage.googleapis.com/iree-model-artifacts/inception_v4_299_fp32.tflite"
+
+class InceptionV4Test(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(InceptionV4Test, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(InceptionV4Test, self).compare_results(iree_results, tflite_results, details)
+    self.assertTrue(numpy.isclose(iree_results, tflite_results, atol=1e-4).all())
+
+  def generate_inputs(self, input_details):
+    inputs = imagenet_test_data.generate_input(self.workdir, input_details)
+    # Normalize inputs to [-1, 1].
+    inputs = (inputs.astype('float32') / 127.5) - 1
+    return [inputs]
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+
+
+
+

tank/tflitehub/inception_v4_uint8_test.py

@@ -0,0 +1,32 @@
+# RUN: %PYTHON %s
+
+import absl.testing
+import imagenet_test_data
+import numpy
+import test_util
+
+# Source https://tfhub.dev/tensorflow/lite-model/inception_v4_quant/1/default/1
+model_path = "https://storage.googleapis.com/iree-model-artifacts/inception_v4_299_uint8.tflite"
+
+class InceptionV4Uint8Test(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(InceptionV4Uint8Test, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(InceptionV4Uint8Test, self).compare_results(iree_results, tflite_results, details)
+    # Dequantize outputs.
+    zero_point = details[0]['quantization_parameters']['zero_points'][0]
+    scale = details[0]['quantization_parameters']['scales'][0]
+    dequantized_iree_results = (iree_results - zero_point) * scale
+    dequantized_tflite_results = (tflite_results - zero_point) * scale
+    self.assertTrue(numpy.isclose(dequantized_iree_results, dequantized_tflite_results, atol=5e-3).all())
+
+  def generate_inputs(self, input_details):
+    return [imagenet_test_data.generate_input(self.workdir, input_details)]
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+

tank/tflitehub/keras_ocr_test.py

@@ -0,0 +1,18 @@
+# RUN: %PYTHON %s
+# XFAIL: *
+
+import absl.testing
+import test_util
+
+model_path = "https://tfhub.dev/tulasiram58827/lite-model/keras-ocr/dr/2?lite-format=tflite"
+
+class KerasOCRTest(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(KerasOCRTest, self).__init__(model_path, *args, **kwargs)
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+

tank/tflitehub/lightning_fp16_test.py

@@ -0,0 +1,19 @@
+# RUN: %PYTHON %s
+
+import absl.testing
+import numpy
+import test_util
+
+model_path = "https://tfhub.dev/google/lite-model/movenet/singlepose/lightning/tflite/float16/4?lite-format=tflite"
+
+# Currently failing further in the linalg stack:
+#   Bug related to linalg fusion. Collapsing dimension despite linalg index.
+class LightningFp16Test(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(LightningFp16Test, self).__init__(model_path, *args, **kwargs)
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()

tank/tflitehub/lightning_i8_test.py

@@ -0,0 +1,69 @@
+# RUN: %PYTHON %s
+
+import absl.testing
+import numpy
+import test_util
+import urllib.request
+
+from PIL import Image
+
+model_path = "https://tfhub.dev/google/lite-model/movenet/singlepose/lightning/tflite/int8/4?lite-format=tflite"
+
+# Currently failing further in the linalg stack:
+#   Invalid cast from ui8 to f32 TODO: make tfl.cast insert a rescale for ui8
+class LightningI8Test(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(LightningI8Test, self).__init__(model_path, *args, **kwargs)
+
+  # Optional utility for visualizing results on the input image. This verifies
+  # the model works-ish as the dots are in roughly the same area. Still need to
+  # debug the source of numerical differences.
+  def plot_results(self, iree_results, tflite_results, details):
+    from matplotlib import pyplot as plt
+    local_path = "/".join([self.workdir, "person.jpg"])
+    im = numpy.array(Image.open(local_path))
+
+    width = im.shape[0]
+    height = im.shape[1]
+
+    tflite_result = tflite_results[0]
+    iree_result = iree_results[0]
+
+    tflite_x = tflite_result[0, 0, :, 0] * width
+    tflite_y = tflite_result[0, 0, :, 1] * height
+
+    iree_x = iree_result[0, 0, :, 0] * width
+    iree_y = iree_result[0, 0, :, 1] * height
+
+    plt.imshow(im)
+    plt.scatter(tflite_y, tflite_x, label='tflite')
+    plt.scatter(iree_y, iree_x, label='iree')
+    plt.legend()
+    plt.show()
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(LightningI8Test, self).compare_results(iree_results, tflite_results, details)
+    # This value is a discretized location of the persons joints. If we are
+    # *close* to the expected position we can consider this good enough.
+    self.assertTrue(numpy.isclose(iree_results[0][:, :, :, 0],
+                                  tflite_results[0][:, :, :, 0], atol=25e-3).all())
+    self.assertTrue(numpy.isclose(iree_results[0][:, :, :, 1],
+                                  tflite_results[0][:, :, :, 1], atol=25e-3).all())
+    # self.plot_results(iree_results, tflite_results, details)
+
+  def generate_inputs(self, input_details):
+    img_path = "https://github.com/tensorflow/examples/raw/master/lite/examples/pose_estimation/raspberry_pi/test_data/image3.jpeg"
+    local_path = "/".join([self.workdir, "person.jpg"])
+    urllib.request.urlretrieve(img_path, local_path)
+
+    shape = input_details[0]["shape"]
+    im = numpy.array(Image.open(local_path).resize((shape[1], shape[2])))
+    args = [im.reshape(shape)]
+    return args
+
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()

tank/tflitehub/lightning_test.py

@@ -0,0 +1,19 @@
+# RUN: %PYTHON %s
+
+import absl.testing
+import numpy
+import test_util
+
+model_path = "https://tfhub.dev/google/lite-model/movenet/singlepose/lightning/3?lite-format=tflite"
+
+# Currently failing further in the linalg stack:
+#   Fusion appears to produce an invalid IR.
+class LightningTest(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(LightningTest, self).__init__(model_path, *args, **kwargs)
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()

tank/tflitehub/lit.cfg.py

@@ -0,0 +1,46 @@
+import os
+import sys
+
+import lit.formats
+import lit.util
+
+import lit.llvm
+
+# Configuration file for the 'lit' test runner.
+lit.llvm.initialize(lit_config, config)
+
+# name: The name of this test suite.
+config.name = 'TFLITEHUB'
+
+config.test_format = lit.formats.ShTest()
+
+# suffixes: A list of file extensions to treat as test files.
+config.suffixes = ['.py']
+
+# test_source_root: The root path where tests are located.
+config.test_source_root = os.path.dirname(__file__)
+
+#config.use_default_substitutions()
+config.excludes = [
+  'coco_test_data.py',
+  'imagenet_test_data.py',
+  'lit.cfg.py',
+  'lit.site.cfg.py',
+  'manual_test.py',
+  'squad_test_data.py',
+  'test_util.py',
+]
+
+config.substitutions.extend([
+    ('%PYTHON', sys.executable),
+])
+
+config.environment['PYTHONPATH'] = ":".join(sys.path)
+
+project_root = os.path.dirname(os.path.dirname(__file__))
+
+# Enable features based on -D FEATURES=hugetest,vulkan
+# syntax.
+features_param = lit_config.params.get('FEATURES')
+if features_param:
+  config.available_features.update(features_param.split(','))

tank/tflitehub/magenta_test.py

@@ -0,0 +1,22 @@
+# RUN: %PYTHON %s
+
+import absl.testing
+import numpy
+import test_util
+
+model_path = "https://tfhub.dev/google/lite-model/magenta/arbitrary-image-stylization-v1-256/int8/prediction/1?lite-format=tflite"
+
+class MagentaTest(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(MagentaTest, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(MagentaTest, self).compare_results(iree_results, tflite_results, details)
+    self.assertTrue(numpy.isclose(iree_results[0], tflite_results[0], atol=2e-1).all())
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+

tank/tflitehub/manual_test.py

@@ -0,0 +1,20 @@
+import absl.flags
+import absl.testing
+import test_util
+
+absl.flags.DEFINE_string('model', None, 'model path to execute')
+
+class ManualTest(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(ManualTest, self).__init__(absl.flags.FLAGS.model, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(ManualTest, self).compare_results(iree_results, tflite_results, details)
+
+  def test_compile_tflite(self):
+    if self.model_path is not None:
+      self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+

tank/tflitehub/midas_test.py

@@ -0,0 +1,22 @@
+# RUN: %PYTHON %s
+
+import absl.testing
+import numpy
+import test_util
+
+model_path = "https://tfhub.dev/intel/lite-model/midas/v2_1_small/1/lite/1?lite-format=tflite"
+
+class MidasTest(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(MidasTest, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(MidasTest, self).compare_results(iree_results, tflite_results, details)
+    self.assertTrue(numpy.isclose(iree_results[0], tflite_results[0], atol=1e-3).all())
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+

tank/tflitehub/mirnet_test.py

@@ -0,0 +1,24 @@
+# RUN: %PYTHON %s
+# REQUIRES: hugetest
+
+import absl.testing
+import numpy
+import test_util
+
+model_path = "https://tfhub.dev/sayakpaul/lite-model/mirnet-fixed/dr/1?lite-format=tflite"
+
+# Note this one takes forever right now. Great for performance work!
+class MirnetTest(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(MirnetTest, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(MirnetTest, self).compare_results(iree_results, tflite_results, details)
+    self.assertTrue(numpy.isclose(iree_results[0], tflite_results[0], atol=5e-3).all())
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+

tank/tflitehub/mnasnet_test.py

@@ -0,0 +1,17 @@
+# RUN: %PYTHON %s
+
+import absl.testing
+import numpy
+import test_util
+
+model_path = "https://tfhub.dev/tensorflow/lite-model/mnasnet_1.0_224/1/metadata/1?lite-format=tflite"
+
+class MnasnetTest(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(MnasnetTest, self).__init__(model_path, *args, **kwargs)
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()

tank/tflitehub/mobilebert_edgetpu_s_float_test.py

@@ -0,0 +1,34 @@
+# RUN: %PYTHON %s
+
+import absl.testing
+import numpy
+import test_util
+
+model_path = "https://storage.googleapis.com/iree-model-artifacts/mobilebert-edgetpu-s-float.tflite"
+
+class MobileBertTest(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(MobileBertTest, self).__init__(model_path, *args, **kwargs)
+
+  # Inputs modified to be useful mobilebert inputs.
+  def generate_inputs(self, input_details):
+    for input in input_details:
+      absl.logging.info("\t%s, %s", str(input["shape"]), input["dtype"].__name__)
+
+    args = []
+    args.append(numpy.random.randint(low=0, high=256, size=input_details[0]["shape"], dtype=input_details[0]["dtype"]))
+    args.append(numpy.ones(shape=input_details[1]["shape"], dtype=input_details[1]["dtype"]))
+    args.append(numpy.zeros(shape=input_details[2]["shape"], dtype=input_details[2]["dtype"]))
+    return args
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(MobileBertTest, self).compare_results(iree_results, tflite_results, details)
+    self.assertTrue(numpy.isclose(iree_results[0], tflite_results[0], atol=1e-4).all())
+    self.assertTrue(numpy.isclose(iree_results[1], tflite_results[1], atol=1e-4).all())
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+

tank/tflitehub/mobilebert_edgetpu_s_quant_test.py

@@ -0,0 +1,35 @@
+# RUN: %PYTHON %s
+# XFAIL: *
+
+import absl.testing
+import numpy
+import test_util
+
+model_path = "https://storage.googleapis.com/iree-model-artifacts/mobilebert-edgetpu-s-quant.tflite"
+
+class MobileBertTest(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(MobileBertTest, self).__init__(model_path, *args, **kwargs)
+
+  # Inputs modified to be useful mobilebert inputs.
+  def generate_inputs(self, input_details):
+    for input in input_details:
+      absl.logging.info("\t%s, %s", str(input["shape"]), input["dtype"].__name__)
+
+    args = []
+    args.append(numpy.random.randint(low=0, high=256, size=input_details[0]["shape"], dtype=input_details[0]["dtype"]))
+    args.append(numpy.ones(shape=input_details[1]["shape"], dtype=input_details[1]["dtype"]))
+    args.append(numpy.zeros(shape=input_details[2]["shape"], dtype=input_details[2]["dtype"]))
+    return args
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(MobileBertTest, self).compare_results(iree_results, tflite_results, details)
+    self.assertTrue(numpy.isclose(iree_results[0], tflite_results[0], atol=1.0).all())
+    self.assertTrue(numpy.isclose(iree_results[1], tflite_results[1], atol=1.0).all())
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+

tank/tflitehub/mobilebert_test.py

@@ -0,0 +1,38 @@
+# RUN: %PYTHON %s
+
+import absl.testing
+import numpy as np
+import squad_test_data
+import test_util
+
+model_path = "https://tfhub.dev/tensorflow/lite-model/mobilebert/1/metadata/1?lite-format=tflite"
+
+class MobileBertTest(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(MobileBertTest, self).__init__(model_path, *args, **kwargs)
+
+  # Inputs modified to be useful mobilebert inputs.
+  def generate_inputs(self, input_details):
+    for input in input_details:
+      absl.logging.info("\t%s, %s", str(input["shape"]), input["dtype"].__name__)
+
+    input_0 = np.asarray(squad_test_data._INPUT_WORD_ID, dtype=input_details[0]["dtype"])
+    input_1 = np.asarray(squad_test_data._INPUT_TYPE_ID, dtype=input_details[1]["dtype"])
+    input_2 = np.asarray(squad_test_data._INPUT_MASK, dtype=input_details[2]["dtype"])
+    return [
+        input_0.reshape(input_details[0]["shape"]),
+        input_1.reshape(input_details[1]["shape"]),
+        input_2.reshape(input_details[2]["shape"])
+    ]
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(MobileBertTest, self).compare_results(iree_results, tflite_results, details)
+    self.assertTrue(np.isclose(iree_results[0], tflite_results[0], atol=1e-4).all())
+    self.assertTrue(np.isclose(iree_results[1], tflite_results[1], atol=1e-4).all())
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+

tank/tflitehub/mobilebert_tf2_float_test.py

@@ -0,0 +1,38 @@
+# RUN: %PYTHON %s
+
+import absl.testing
+import numpy as np
+import squad_test_data
+import test_util
+
+model_path = "https://storage.googleapis.com/iree-model-artifacts/mobilebert-baseline-tf2-float.tflite"
+
+class MobileBertTest(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(MobileBertTest, self).__init__(model_path, *args, **kwargs)
+
+  # Inputs modified to be useful mobilebert inputs.
+  def generate_inputs(self, input_details):
+    for input in input_details:
+      absl.logging.info("\t%s, %s", str(input["shape"]), input["dtype"].__name__)
+
+    input_0 = np.asarray(squad_test_data._INPUT_WORD_ID, dtype=input_details[0]["dtype"])
+    input_1 = np.asarray(squad_test_data._INPUT_TYPE_ID, dtype=input_details[1]["dtype"])
+    input_2 = np.asarray(squad_test_data._INPUT_MASK, dtype=input_details[2]["dtype"])
+    return [
+        input_0.reshape(input_details[0]["shape"]),
+        input_1.reshape(input_details[1]["shape"]),
+        input_2.reshape(input_details[2]["shape"])
+    ]
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(MobileBertTest, self).compare_results(iree_results, tflite_results, details)
+    self.assertTrue(np.isclose(iree_results[0], tflite_results[0], atol=1e-4).all())
+    self.assertTrue(np.isclose(iree_results[1], tflite_results[1], atol=1e-4).all())
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+

tank/tflitehub/mobilebert_tf2_quant_test.py

@@ -0,0 +1,39 @@
+# RUN: %PYTHON %s
+
+import absl.testing
+import numpy as np
+import squad_test_data
+import test_util
+
+model_path = "https://storage.googleapis.com/iree-model-artifacts/mobilebert-baseline-tf2-quant.tflite"
+
+class MobileBertTest(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(MobileBertTest, self).__init__(model_path, *args, **kwargs)
+
+  # Inputs modified to be useful mobilebert inputs.
+  def generate_inputs(self, input_details):
+    for input in input_details:
+      absl.logging.info("\t%s, %s", str(input["shape"]), input["dtype"].__name__)
+
+    input_0 = np.asarray(squad_test_data._INPUT_WORD_ID, dtype=input_details[0]["dtype"])
+    input_1 = np.asarray(squad_test_data._INPUT_TYPE_ID, dtype=input_details[1]["dtype"])
+    input_2 = np.asarray(squad_test_data._INPUT_MASK, dtype=input_details[2]["dtype"])
+    return [
+        input_0.reshape(input_details[0]["shape"]),
+        input_1.reshape(input_details[1]["shape"]),
+        input_2.reshape(input_details[2]["shape"])
+    ]
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(MobileBertTest, self).compare_results(iree_results, tflite_results, details)
+    # We have confirmed in large scale accuracy tests that differences this large is acceptable.
+    self.assertTrue(np.isclose(iree_results[0], tflite_results[0], atol=5.0).all())
+    self.assertTrue(np.isclose(iree_results[1], tflite_results[1], atol=5.0).all())
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+

tank/tflitehub/mobilenet_ssd_quant_test.py

@@ -0,0 +1,37 @@
+# RUN: %PYTHON %s
+
+import absl.testing
+import numpy
+import os
+import test_util
+import urllib.request
+
+from PIL import Image
+
+# Model from https://github.com/google-coral/test_data/raw/master/ssd_mobilenet_v2_face_quant_postprocess.tflite
+# but trimmed the final TFLite_PostProcess op.
+model_path = "https://storage.googleapis.com/iree-shared-files/models/ssd_mobilenet_v2_face_quant.tflite"
+
+class MobilenetSsdQuantTest(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(MobilenetSsdQuantTest, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(MobilenetSsdQuantTest, self).compare_results(iree_results, tflite_results, details)
+    self.assertTrue(numpy.isclose(iree_results[0], tflite_results[0], atol=1.0).all())
+
+  def generate_inputs(self, input_details):
+    img_path = "https://github.com/google-coral/test_data/raw/master/grace_hopper.bmp"
+    local_path = "/".join([self.workdir, "grace_hopper.bmp"])
+    urllib.request.urlretrieve(img_path, local_path)
+
+    shape = input_details[0]["shape"]
+    im = numpy.array(Image.open(local_path).resize((shape[1], shape[2])))
+    args = [im.reshape(shape)]
+    return args
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()

tank/tflitehub/mobilenet_v1_test.py

@@ -0,0 +1,29 @@
+# RUN: %PYTHON %s
+
+import absl.testing
+import imagenet_test_data
+import numpy
+import test_util
+
+model_path = "https://storage.googleapis.com/iree-model-artifacts/mobilenet_v1_224_1.0_float.tflite"
+
+class MobilenetV1Test(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(MobilenetV1Test, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(MobilenetV1Test, self).compare_results(iree_results, tflite_results, details)
+    self.assertTrue(numpy.isclose(iree_results, tflite_results, atol=1e-4).all())
+
+  def generate_inputs(self, input_details):
+    inputs = imagenet_test_data.generate_input(self.workdir, input_details)
+    # Normalize inputs to [-1, 1].
+    inputs = (inputs.astype('float32') / 127.5) - 1
+    return [inputs]
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+

tank/tflitehub/mobilenet_v1_uint8_test.py

@@ -0,0 +1,31 @@
+# RUN: %PYTHON %s
+
+import absl.testing
+import imagenet_test_data
+import numpy
+import test_util
+
+model_path = "https://storage.googleapis.com/iree-model-artifacts/mobilenet_v1_224_1.0_uint8.tflite"
+
+class MobilenetV1Uint8Test(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(MobilenetV1Uint8Test, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(MobilenetV1Uint8Test, self).compare_results(iree_results, tflite_results, details)
+    # Dequantize outputs.
+    zero_point = details[0]['quantization_parameters']['zero_points'][0]
+    scale = details[0]['quantization_parameters']['scales'][0]
+    dequantized_iree_results = (iree_results - zero_point) * scale
+    dequantized_tflite_results = (tflite_results - zero_point) * scale
+    self.assertTrue(numpy.isclose(dequantized_iree_results, dequantized_tflite_results, atol=5e-3).all())
+
+  def generate_inputs(self, input_details):
+    return [imagenet_test_data.generate_input(self.workdir, input_details)]
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+

tank/tflitehub/mobilenet_v2_int8_test.py

@@ -0,0 +1,36 @@
+# RUN: %PYTHON %s
+
+import absl.testing
+import imagenet_test_data
+import numpy
+import test_util
+
+model_path = "https://storage.googleapis.com/tf_model_garden/vision/mobilenet/v2_1.0_int8/mobilenet_v2_1.00_224_int8.tflite"
+
+class MobilenetV2Int8Test(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(MobilenetV2Int8Test, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(MobilenetV2Int8Test, self).compare_results(iree_results, tflite_results, details)
+    # Although this a quantized model, inputs and outputs are in float.
+    # The difference here is quite high for a dequantized output.
+    self.assertTrue(numpy.isclose(iree_results, tflite_results, atol=0.5).all())
+
+    # Make sure the predicted class is the same.
+    iree_predicted_class = numpy.argmax(iree_results[0][0])
+    tflite_predicted_class = numpy.argmax(tflite_results[0][0])
+    self.assertEqual(iree_predicted_class, tflite_predicted_class)
+
+  def generate_inputs(self, input_details):
+    inputs = imagenet_test_data.generate_input(self.workdir, input_details)
+    # Normalize inputs to [-1, 1].
+    inputs = (inputs.astype('float32') / 127.5) - 1
+    return [inputs]
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+

tank/tflitehub/mobilenet_v2_test.py

@@ -0,0 +1,29 @@
+# RUN: %PYTHON %s
+
+import absl.testing
+import imagenet_test_data
+import numpy
+import test_util
+
+model_path = "https://storage.googleapis.com/iree-model-artifacts/mobilenet_v2_1.0_224.tflite"
+
+class MobilenetV2Test(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(MobilenetV2Test, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(MobilenetV2Test, self).compare_results(iree_results, tflite_results, details)
+    self.assertTrue(numpy.isclose(iree_results, tflite_results, atol=1e-4).all())
+
+  def generate_inputs(self, input_details):
+    inputs = imagenet_test_data.generate_input(self.workdir, input_details)
+    # Normalize inputs to [-1, 1].
+    inputs = (inputs.astype('float32') / 127.5) - 1
+    return [inputs]
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+

tank/tflitehub/mobilenet_v2_uint8_test.py

@@ -0,0 +1,31 @@
+# RUN: %PYTHON %s
+
+import absl.testing
+import imagenet_test_data
+import numpy
+import test_util
+
+model_path = "https://storage.googleapis.com/iree-model-artifacts/mobilenet_v2_224_1.0_uint8.tflite"
+
+class MobilenetV2Uint8Test(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(MobilenetV2Uint8Test, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(MobilenetV2Uint8Test, self).compare_results(iree_results, tflite_results, details)
+    # Dequantize outputs.
+    zero_point = details[0]['quantization_parameters']['zero_points'][0]
+    scale = details[0]['quantization_parameters']['scales'][0]
+    dequantized_iree_results = (iree_results - zero_point) * scale
+    dequantized_tflite_results = (tflite_results - zero_point) * scale
+    self.assertTrue(numpy.isclose(dequantized_iree_results, dequantized_tflite_results, atol=5e-3).all())
+
+  def generate_inputs(self, input_details):
+    return [imagenet_test_data.generate_input(self.workdir, input_details)]
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+

tank/tflitehub/mobilenet_v3-large_test.py

@@ -0,0 +1,29 @@
+# RUN: %PYTHON %s
+
+import absl.testing
+import imagenet_test_data
+import numpy
+import test_util
+
+model_path = "https://storage.googleapis.com/iree-model-artifacts/mobilenet_v3-large_224_1.0_float.tflite"
+
+class MobilenetV3LargeTest(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(MobilenetV3LargeTest, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(MobilenetV3LargeTest, self).compare_results(iree_results, tflite_results, details)
+    self.assertTrue(numpy.isclose(iree_results, tflite_results, atol=1e-4).all())
+
+  def generate_inputs(self, input_details):
+    inputs = imagenet_test_data.generate_input(self.workdir, input_details)
+    # Normalize inputs to [-1, 1].
+    inputs = (inputs.astype('float32') / 127.5) - 1
+    return [inputs]
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+

tank/tflitehub/mobilenet_v3-large_uint8_test.py

@@ -0,0 +1,31 @@
+# RUN: %PYTHON %s
+
+import absl.testing
+import imagenet_test_data
+import numpy
+import test_util
+
+model_path = "https://storage.googleapis.com/iree-model-artifacts/mobilenet_v3-large_224_1.0_uint8.tflite"
+
+class MobilenetV3LargeUint8Test(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(MobilenetV3LargeUint8Test, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(MobilenetV3LargeUint8Test, self).compare_results(iree_results, tflite_results, details)
+    # Dequantize outputs.
+    zero_point = details[0]['quantization_parameters']['zero_points'][0]
+    scale = details[0]['quantization_parameters']['scales'][0]
+    dequantized_iree_results = (iree_results - zero_point) * scale
+    dequantized_tflite_results = (tflite_results - zero_point) * scale
+    self.assertTrue(numpy.isclose(dequantized_iree_results, dequantized_tflite_results, atol=5e-3).all())
+
+  def generate_inputs(self, input_details):
+    return [imagenet_test_data.generate_input(self.workdir, input_details)]
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+

tank/tflitehub/mobilenet_v35_int8_test.py

@@ -0,0 +1,34 @@
+# RUN: %PYTHON %s
+
+import absl.testing
+import imagenet_test_data
+import numpy
+import test_util
+
+model_path = "https://storage.googleapis.com/tf_model_garden/vision/mobilenet/v3.5multiavg_1.0_int8/mobilenet_v3.5multiavg_1.00_224_int8.tflite"
+
+class MobilenetV35Int8Test(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(MobilenetV35Int8Test, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(MobilenetV35Int8Test, self).compare_results(iree_results, tflite_results, details)
+    # The difference here is quite high for a dequantized output.
+    self.assertTrue(numpy.isclose(iree_results, tflite_results, atol=0.5).all())
+
+    # Make sure the predicted class is the same.
+    iree_predicted_class = numpy.argmax(iree_results[0][0])
+    tflite_predicted_class = numpy.argmax(tflite_results[0][0])
+    self.assertEqual(iree_predicted_class, tflite_predicted_class)
+
+  def generate_inputs(self, input_details):
+    inputs = imagenet_test_data.generate_input(self.workdir, input_details)
+    # Normalize inputs to [-1, 1].
+    inputs = (inputs.astype('float32') / 127.5) - 1
+    return [inputs]
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()

tank/tflitehub/nasnet_test.py

@@ -0,0 +1,18 @@
+# RUN: %PYTHON %s
+# REQUIRES: hugetest
+
+import absl.testing
+import numpy
+import test_util
+
+model_path = "https://tfhub.dev/tensorflow/lite-model/nasnet/large/1/default/1?lite-format=tflite"
+
+class MnasnetTest(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(MnasnetTest, self).__init__(model_path, *args, **kwargs)
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()

tank/tflitehub/person_detect_test.py

@@ -0,0 +1,51 @@
+# RUN: %PYTHON %s
+
+import absl.testing
+import numpy
+import test_util
+import urllib.request
+
+from PIL import Image
+
+model_path = "https://github.com/tensorflow/tflite-micro/raw/aeac6f39e5c7475cea20c54e86d41e3a38312546/tensorflow/lite/micro/models/person_detect.tflite"
+
+class PersonDetectTest(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(PersonDetectTest, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(PersonDetectTest, self).compare_results(iree_results, tflite_results, details)
+    self.assertTrue(numpy.isclose(iree_results[0], tflite_results[0], atol=1e-3).all())
+
+  # TFLite is broken with this model so we hardcode the input/output details.
+  def setup_tflite(self):
+    self.input_details = [{
+      "shape": [1, 96, 96, 1],
+      "dtype": numpy.int8,
+      "index": 0,
+    }]
+    self.output_details = [{
+      "shape": [1, 2],
+      "dtype" : numpy.int8,
+   }]
+
+  # The input has known expected values. We hardcode this value.
+  def invoke_tflite(self, args):
+    return [numpy.array([[-113, 113]], dtype=numpy.int8)]
+
+  def generate_inputs(self, input_details):
+    img_path = "https://github.com/tensorflow/tflite-micro/raw/aeac6f39e5c7475cea20c54e86d41e3a38312546/tensorflow/lite/micro/examples/person_detection/testdata/person.bmp"
+    local_path = "/".join([self.workdir, "person.bmp"])
+    urllib.request.urlretrieve(img_path, local_path)
+
+    shape = input_details[0]["shape"]
+    im = numpy.array(Image.open(local_path).resize(
+        (shape[1], shape[2]))).astype(input_details[0]["dtype"])
+    args = [im.reshape(shape)]
+    return args
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()

tank/tflitehub/posenet_test.py

@@ -0,0 +1,25 @@
+# RUN: %PYTHON %s
+
+import absl.testing
+import numpy
+import test_util
+
+model_path = "https://storage.googleapis.com/download.tensorflow.org/models/tflite/gpu/multi_person_mobilenet_v1_075_float.tflite"
+
+class PoseTest(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(PoseTest, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(PoseTest, self).compare_results(iree_results, tflite_results, details)
+    self.assertTrue(numpy.isclose(iree_results[0], tflite_results[0], atol=1e-3).all())
+    self.assertTrue(numpy.isclose(iree_results[1], tflite_results[1], atol=1e-2).all())
+    self.assertTrue(numpy.isclose(iree_results[2], tflite_results[2], atol=1e-2).all())
+    self.assertTrue(numpy.isclose(iree_results[3], tflite_results[3], atol=1e-3).all())
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+

tank/tflitehub/resnet_50_int8_test.py

@@ -0,0 +1,22 @@
+# RUN: %PYTHON %s
+
+import absl.testing
+import numpy
+import test_util
+
+model_path = "https://storage.googleapis.com/tf_model_garden/vision/resnet50_imagenet/resnet_50_224_int8.tflite"
+
+class ResNet50Int8Test(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(ResNet50Int8Test, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(ResNet50Int8Test, self).compare_results(iree_results, tflite_results, details)
+    self.assertTrue(numpy.isclose(iree_results[0], tflite_results[0], atol=1.0).all())
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+

tank/tflitehub/rosetta_test.py

@@ -0,0 +1,24 @@
+# RUN: %PYTHON %s
+# XFAIL: *
+
+import absl.testing
+import numpy
+import test_util
+
+model_path = "https://tfhub.dev/tulasiram58827/lite-model/rosetta/dr/1?lite-format=tflite"
+
+# tfl.padv2 cannot be lowered to tosa.pad. May be possible to switch tosa.concat
+class RosettaTest(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(RosettaTest, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(RosettaTest, self).compare_results(iree_results, tflite_results, details)
+    self.assertTrue(numpy.isclose(iree_results[0], tflite_results[0], atol=5e-3).all())
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+

tank/tflitehub/spice_test.py

@@ -0,0 +1,21 @@
+# RUN: %PYTHON %s
+# XFAIL: *
+
+import absl.testing
+import numpy
+import test_util
+
+model_path = "https://tfhub.dev/google/lite-model/spice/1?lite-format=tflite"
+
+# Currently unsupported:
+# 1. Multiple unsupported dynamic operations (tfl.stride, range, gather).
+# 2. Static version blocked by tfl.range not having a lowering for static fixed shapes.
+class SpiceTest(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(SpiceTest, self).__init__(model_path, *args, **kwargs)
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()

tank/tflitehub/squad_test_data.py

@@ -0,0 +1,4 @@
+# An example input combination from the Squad 1.1 dataset.
+_INPUT_WORD_ID = [101, 2129, 2116, 19576, 2015, 2106, 3854, 4679, 2486, 1029, 102, 1996, 14169, 2165, 2019, 2220, 2599, 1999, 3565, 4605, 2753, 1998, 2196, 11145, 1012, 8446, 2001, 3132, 2011, 7573, 1005, 1055, 3639, 1010, 2029, 14159, 2032, 2698, 2335, 1998, 3140, 2032, 2046, 2093, 20991, 2015, 1010, 2164, 1037, 19576, 2029, 2027, 6757, 2005, 1037, 7921, 1012, 7573, 15674, 3854, 4679, 2001, 2315, 3565, 4605, 12041, 1010, 3405, 2274, 3948, 10455, 1010, 1016, 13714, 14918, 1010, 1998, 2048, 3140, 19576, 2015, 1012, 102, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
+_INPUT_TYPE_ID = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
+_INPUT_MASK = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0]

tank/tflitehub/squeezenet_test.py

@@ -0,0 +1,22 @@
+# RUN: %PYTHON %s
+
+import absl.testing
+import test_util
+
+model_path = "https://tfhub.dev/tensorflow/lite-model/squeezenet/1/default/1?lite-format=tflite"
+
+class SqueezeNetTest(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(SqueezeNetTest, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(SqueezeNetTest, self).compare_results(iree_results, tflite_results, details)
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+
+
+

tank/tflitehub/ssd_mobilenet_v1_test.py

@@ -0,0 +1,30 @@
+# RUN: %PYTHON %s
+
+import absl.testing
+import coco_test_data
+import numpy
+import test_util
+
+model_path = "https://storage.googleapis.com/iree-model-artifacts/ssd_mobilenet_v1_320_1.0_float.tflite"
+
+class SsdMobilenetV1Test(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(SsdMobilenetV1Test, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(SsdMobilenetV1Test, self).compare_results(iree_results, tflite_results, details)
+    for i in range(len(iree_results)):
+      self.assertTrue(numpy.isclose(iree_results[i], tflite_results[i], atol=1e-4).all())
+
+  def generate_inputs(self, input_details):
+    inputs = coco_test_data.generate_input(self.workdir, input_details)
+    # Normalize inputs to [-1, 1].
+    inputs = (inputs.astype('float32') / 127.5) - 1
+    return [inputs]
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+

tank/tflitehub/ssd_mobilenet_v1_uint8_test.py

@@ -0,0 +1,32 @@
+# RUN: %PYTHON %s
+
+import absl.testing
+import coco_test_data
+import numpy
+import test_util
+
+model_path = "https://storage.googleapis.com/iree-model-artifacts/ssd_mobilenet_v1_320_1.0_uint8.tflite"
+
+class SsdMobilenetV1Uint8Test(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(SsdMobilenetV1Uint8Test, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(SsdMobilenetV1Uint8Test, self).compare_results(iree_results, tflite_results, details)
+    for i in range(len(iree_results)):
+      # Dequantize outputs.
+      zero_point = details[i]['quantization_parameters']['zero_points'][0]
+      scale = details[i]['quantization_parameters']['scales'][0]
+      dequantized_iree_results = (iree_results[i] - zero_point) * scale
+      dequantized_tflite_results = (tflite_results[i] - zero_point) * scale
+      self.assertTrue(numpy.isclose(dequantized_iree_results, dequantized_tflite_results, atol=0.1).all())
+
+  def generate_inputs(self, input_details):
+    return [coco_test_data.generate_input(self.workdir, input_details)]
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+

tank/tflitehub/ssd_mobilenet_v2_fpnlite_test.py

@@ -0,0 +1,31 @@
+# RUN: %PYTHON %s
+# XFAIL: *
+
+import absl.testing
+import coco_test_data
+import numpy
+import test_util
+
+model_path = "https://storage.googleapis.com/iree-model-artifacts/ssd_mobilenet_v2_fpnlite_dynamic_1.0_float.tflite"
+
+class SsdMobilenetV2FpnliteTest(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(SsdMobilenetV2FpnliteTest, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(SsdMobilenetV2FpnliteTest, self).compare_results(iree_results, tflite_results, details)
+    for i in range(len(iree_results)):
+      self.assertTrue(numpy.isclose(iree_results[i], tflite_results[i], atol=1e-4).all())
+
+  def generate_inputs(self, input_details):
+    inputs = coco_test_data.generate_input(self.workdir, input_details)
+    # Normalize inputs to [-1, 1].
+    inputs = (inputs.astype('float32') / 127.5) - 1
+    return [inputs]
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+

tank/tflitehub/ssd_mobilenet_v2_fpnlite_uint8_test.py

@@ -0,0 +1,33 @@
+# RUN: %PYTHON %s
+# XFAIL: *
+
+import absl.testing
+import coco_test_data
+import numpy
+import test_util
+
+model_path = "https://storage.googleapis.com/iree-model-artifacts/ssd_mobilenet_v2_fpnlite_dynamic_1.0_uint8.tflite"
+
+class SsdMobilenetV2FpnliteUint8Test(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(SsdMobilenetV2FpnliteUint8Test, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(SsdMobilenetV2FpnliteUint8Test, self).compare_results(iree_results, tflite_results, details)
+    for i in range(len(iree_results)):
+      # Dequantize outputs.
+      zero_point = details[i]['quantization_parameters']['zero_points'][0]
+      scale = details[i]['quantization_parameters']['scales'][0]
+      dequantized_iree_results = (iree_results[i] - zero_point) * scale
+      dequantized_tflite_results = (tflite_results[i] - zero_point) * scale
+      self.assertTrue(numpy.isclose(dequantized_iree_results, dequantized_tflite_results, atol=0.1).all())
+
+  def generate_inputs(self, input_details):
+    return [coco_test_data.generate_input(self.workdir, input_details)]
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+

tank/tflitehub/ssd_mobilenet_v2_int8_test.py

@@ -0,0 +1,36 @@
+# RUN: %PYTHON %s
+# XFAIL: *
+
+import absl.testing
+import coco_test_data
+import numpy
+import test_util
+
+model_path = "https://storage.googleapis.com/iree-model-artifacts/ssd_mobilenet_v2_dynamic_1.0_int8.tflite"
+
+class SsdMobilenetV2Test(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(SsdMobilenetV2Test, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(SsdMobilenetV1Test, self).compare_results(iree_results, tflite_results, details)
+    for i in range(len(iree_results)):
+      # Dequantize outputs.
+      zero_point = details[i]['quantization_parameters']['zero_points'][0]
+      scale = details[i]['quantization_parameters']['scales'][0]
+      dequantized_iree_results = (iree_results[i] - zero_point) * scale
+      dequantized_tflite_results = (tflite_results[i] - zero_point) * scale
+      self.assertTrue(numpy.isclose(dequantized_iree_results, dequantized_tflite_results, atol=0.1).all())
+
+  def generate_inputs(self, input_details):
+    inputs = coco_test_data.generate_input(self.workdir, input_details)
+    # Move input values from [0, 255] to [-128, 127].
+    inputs = inputs - 128
+    return [inputs]
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+

tank/tflitehub/ssd_mobilenet_v2_test.py

@@ -0,0 +1,31 @@
+# RUN: %PYTHON %s
+# XFAIL: *
+
+import absl.testing
+import coco_test_data
+import numpy
+import test_util
+
+model_path = "https://storage.googleapis.com/iree-model-artifacts/ssd_mobilenet_v2_dynamic_1.0_float.tflite"
+
+class SsdMobilenetV2Test(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(SsdMobilenetV2Test, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(SsdMobilenetV2Test, self).compare_results(iree_results, tflite_results, details)
+    for i in range(len(iree_results)):
+      self.assertTrue(numpy.isclose(iree_results[i], tflite_results[i], atol=1e-4).all())
+
+  def generate_inputs(self, input_details):
+    inputs = coco_test_data.generate_input(self.workdir, input_details)
+    # Normalize inputs to [-1, 1].
+    inputs = (inputs.astype('float32') / 127.5) - 1
+    return [inputs]
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+

tank/tflitehub/ssd_spaghettinet_large_test.py

@@ -0,0 +1,33 @@
+# RUN: %PYTHON %s
+# XFAIL: *
+
+import absl.testing
+import coco_test_data
+import numpy
+import test_util
+
+model_path = "https://storage.googleapis.com/iree-model-artifacts/ssd_spaghettinet_edgetpu_large.tflite"
+
+class SsdSpaghettinetLargeTest(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(SsdSpaghettinetLargeTest, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(SsdSpaghettinetLargeTest, self).compare_results(iree_results, tflite_results, details)
+    for i in range(len(iree_results)):
+      print('iree_results: ' + str(iree_results[i]))
+      print('tflite_results: ' + str(tflite_results[i]))
+      self.assertTrue(numpy.isclose(iree_results[i], tflite_results[i], atol=1e-4).all())
+
+  def generate_inputs(self, input_details):
+    inputs = coco_test_data.generate_input(self.workdir, input_details)
+    # Normalize inputs to [-1, 1].
+    inputs = (inputs.astype('float32') / 127.5) - 1
+    return [inputs]
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+

tank/tflitehub/ssd_spaghettinet_large_uint8_test.py

@@ -0,0 +1,33 @@
+# RUN: %PYTHON %s
+# XFAIL: *
+
+import absl.testing
+import coco_test_data
+import numpy
+import test_util
+
+model_path = "https://storage.googleapis.com/iree-model-artifacts/ssd_spaghettinet_edgetpu_large_uint8.tflite"
+
+class SsdSpaghettinetLargeUint8Test(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(SsdSpaghettinetLargeUint8Test, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(SsdSpaghettinetLargeUint8Test, self).compare_results(iree_results, tflite_results, details)
+    for i in range(len(iree_results)):
+      # Dequantize outputs.
+      zero_point = details[i]['quantization_parameters']['zero_points'][0]
+      scale = details[i]['quantization_parameters']['scales'][0]
+      dequantized_iree_results = (iree_results[i] - zero_point) * scale
+      dequantized_tflite_results = (tflite_results[i] - zero_point) * scale
+      self.assertTrue(numpy.isclose(dequantized_iree_results, dequantized_tflite_results, atol=0.1).all())
+
+  def generate_inputs(self, input_details):
+    return [coco_test_data.generate_input(self.workdir, input_details)]
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()
+

tank/tflitehub/test_util.py

@@ -0,0 +1,147 @@
+# Lint as: python3
+# Copyright 2021 The IREE Authors
+#
+# Licensed under the Apache License v2.0 with LLVM Exceptions.
+# See https://llvm.org/LICENSE.txt for license information.
+# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+"""Test architecture for a set of tflite tests."""
+
+import absl
+from absl.flags import FLAGS
+import absl.testing as testing
+import iree.compiler.tflite as iree_tflite_compile
+import iree.runtime as iree_rt
+import numpy as np
+import os
+import sys
+import tensorflow.compat.v2 as tf
+import time
+import urllib.request
+
+targets = {
+  'dylib' : 'dylib-llvm-aot',
+  'vulkan' : 'vulkan-spirv',
+}
+
+configs = {
+  'dylib' : 'dylib',
+  'vulkan' : 'vulkan',
+}
+
+absl.flags.DEFINE_string('config', 'dylib', 'model path to execute')
+
+class TFLiteModelTest(testing.absltest.TestCase):
+  def __init__(self, model_path, *args, **kwargs):
+    super(TFLiteModelTest, self).__init__(*args, **kwargs)
+    self.model_path = model_path
+
+  def setUp(self):
+    if self.model_path is None:
+      return
+    exe_basename = os.path.basename(sys.argv[0])
+    self.workdir = os.path.join(os.path.dirname(__file__), "tmp", exe_basename)
+    print(f"TMP_DIR = {self.workdir}")
+    os.makedirs(self.workdir, exist_ok=True)
+    self.tflite_file = '/'.join([self.workdir, 'model.tflite'])
+    self.tflite_ir = '/'.join([self.workdir, 'tflite.mlir'])
+    self.iree_ir = '/'.join([self.workdir, 'tosa.mlir'])
+    if os.path.exists(self.model_path):
+      self.tflite_file = self.model_path
+    else:
+      urllib.request.urlretrieve(self.model_path, self.tflite_file)
+    self.binary = '/'.join([self.workdir, 'module.bytecode'])
+
+  def generate_inputs(self, input_details):
+    args = []
+    for input in input_details:
+      absl.logging.info("\t%s, %s", str(input["shape"]), input["dtype"].__name__)
+      args.append(np.zeros(shape=input["shape"], dtype=input["dtype"]))
+    return args
+
+  def compare_results(self, iree_results, tflite_results, details):
+    self.assertEqual(
+      len(iree_results), len(tflite_results), "Number of results do not match")
+
+    for i in range(len(details)):
+      iree_result = iree_results[i]
+      tflite_result = tflite_results[i]
+      iree_result = iree_result.astype(np.single)
+      tflite_result = tflite_result.astype(np.single)
+      self.assertEqual(iree_result.shape, tflite_result.shape)
+      maxError = np.max(np.abs(iree_result - tflite_result))
+      absl.logging.info("Max error (%d): %f", i, maxError)
+
+  def setup_tflite(self):
+    absl.logging.info("Setting up tflite interpreter")
+    self.tflite_interpreter = tf.lite.Interpreter(model_path=self.tflite_file)
+    self.tflite_interpreter.allocate_tensors()
+    self.input_details = self.tflite_interpreter.get_input_details()
+    self.output_details = self.tflite_interpreter.get_output_details()
+
+  def setup_iree(self):
+    absl.logging.info("Setting up iree runtime")
+    with open(self.binary, 'rb') as f:
+      config = iree_rt.Config(configs[absl.flags.FLAGS.config])
+      self.iree_context = iree_rt.SystemContext(config=config)
+      vm_module = iree_rt.VmModule.from_flatbuffer(f.read())
+      self.iree_context.add_vm_module(vm_module)
+
+  def invoke_tflite(self, args):
+    for i, input in enumerate(args):
+      self.tflite_interpreter.set_tensor(self.input_details[i]['index'], input)
+    start = time.perf_counter()
+    self.tflite_interpreter.invoke()
+    end = time.perf_counter()
+    tflite_results = []
+    absl.logging.info(f"Invocation time: {end - start:0.4f} seconds")
+    for output_detail in self.output_details:
+      tflite_results.append(np.array(self.tflite_interpreter.get_tensor(
+        output_detail['index'])))
+
+    for i in range(len(self.output_details)):
+      dtype = self.output_details[i]["dtype"]
+      tflite_results[i] = tflite_results[i].astype(dtype)
+    return tflite_results
+
+  def invoke_iree(self, args):
+    invoke = self.iree_context.modules.module["main"]
+    start = time.perf_counter()
+    iree_results = invoke(*args)
+    end = time.perf_counter()
+    absl.logging.info(f"Invocation time: {end - start:0.4f} seconds")
+    if not isinstance(iree_results, tuple):
+      iree_results = (iree_results,)
+    return iree_results
+
+  def compile_and_execute(self):
+    self.assertIsNotNone(self.model_path)
+
+    absl.logging.info("Setting up for IREE")
+    iree_tflite_compile.compile_file(
+      self.tflite_file, input_type="tosa",
+      output_file=self.binary,
+      save_temp_tfl_input=self.tflite_ir,
+      save_temp_iree_input=self.iree_ir,
+      target_backends=[targets[absl.flags.FLAGS.config]],
+      import_only=False)
+
+    self.setup_tflite()
+    self.setup_iree()
+
+    absl.logging.info("Setting up test inputs")
+    args = self.generate_inputs(self.input_details)
+
+    absl.logging.info("Invoking TFLite")
+    tflite_results = self.invoke_tflite(args)
+
+    absl.logging.info("Invoke IREE")
+    iree_results = self.invoke_iree(args)
+
+    # Fix type information for unsigned cases.
+    iree_results = list(iree_results)
+    for i in range(len(self.output_details)):
+      dtype = self.output_details[i]["dtype"]
+      iree_results[i] = iree_results[i].astype(dtype)
+
+
+    self.compare_results(iree_results, tflite_results, self.output_details)

tank/tflitehub/visual_wake_words_i8_test.py

@@ -0,0 +1,38 @@
+# RUN: %PYTHON %s
+
+import absl.testing
+import numpy
+import test_util
+import urllib.request
+
+from PIL import Image
+
+model_path = "https://github.com/mlcommons/tiny/raw/0b04bcd402ee28f84e79fa86d8bb8e731d9497b8/v0.5/training/visual_wake_words/trained_models/vww_96_int8.tflite"
+
+# Failure is due to dynamic shapes. This model has a dynamic batch dimension
+# and there is not currently supported. Flatbuffer was modified to use static
+#  shapes and was otherwise numerically correct.
+class VisualWakeWordsTest(test_util.TFLiteModelTest):
+  def __init__(self, *args, **kwargs):
+    super(VisualWakeWordsTest, self).__init__(model_path, *args, **kwargs)
+
+  def compare_results(self, iree_results, tflite_results, details):
+    super(VisualWakeWordsTest, self).compare_results(iree_results, tflite_results, details)
+    self.assertTrue(numpy.isclose(iree_results[0], tflite_results[0], atol=1).all())
+
+  def generate_inputs(self, input_details):
+    img_path = "https://github.com/tensorflow/tflite-micro/raw/main/tensorflow/lite/micro/examples/person_detection/testdata/person.bmp"
+    local_path = "/".join([self.workdir, "person.bmp"])
+    urllib.request.urlretrieve(img_path, local_path)
+
+    shape = input_details[0]["shape"]
+    input_type = input_details[0]["dtype"]
+    im = numpy.array(Image.open(local_path).resize((shape[1], shape[2])).convert(mode="RGB"))
+    args = [im.reshape(shape).astype(input_type)]
+    return args
+
+  def test_compile_tflite(self):
+    self.compile_and_execute()
+
+if __name__ == '__main__':
+  absl.testing.absltest.main()