tests: run modified tests with full coverage

This addition would catch tests being added or modified and would run
those tests with --all and catch issues with non default board
configurations before they get merged into the tree.

Signed-off-by: Anas Nashif <anas.nashif@intel.com>

.shippable.yml

@@ -11,6 +11,7 @@ env:
         - ZEPHYR_GCC_VARIANT=zephyr
         - USE_CCACHE=1
         - MATRIX_BUILDS="2"
+        - MATRIX_BUILDS_EXTRA="3"
     matrix:
         - MATRIX_BUILD="1"
         - MATRIX_BUILD="2"
@@ -64,12 +65,22 @@ build:
             echo "- Verify commit message and coding style";
             ./scripts/ci/check-compliance.py --commits ${COMMIT_RANGE} || true;
           fi;
+      - >
+          # installing due to missing sh module for python3 in image, remove after docker image update
+          sudo pip3 install sh;
+          ./scripts/ci/get_modified_tests.py --commits origin/${PULL_REQUEST_BASE_BRANCH}..HEAD > modified_tests.args;
+
+          if [ -s modified_tests.args ]; then
+            ./scripts/sanitycheck --subset ${MATRIX_BUILD}/${MATRIX_BUILDS_EXTRA} +modified_tests.args;
+            cp ./scripts/sanity_chk/last_sanity.xml modified_tests.xml;
+          fi;
+          rm -f modified_tests.args;
       - >
           if [ "$MATRIX_BUILD" != "3" ]; then
           	./scripts/sanitycheck ${PLATFORMS} --subset ${MATRIX_BUILD}/${MATRIX_BUILDS} ${COVERAGE} ${SANITYCHECK_OPTIONS} || ./scripts/sanitycheck ${PLATFORMS} --subset ${MATRIX_BUILD}/${MATRIX_BUILDS}  ${COVERAGE} ${SANITYCHECK_OPTIONS_RETRY};
           fi;
       - ccache -s
-    on_success:
+    post_ci:
       - rm -rf sanity-out out-2nd-pass
       - mkdir -p shippable/testresults
       - >
@@ -82,18 +93,10 @@ build:
             cp ./scripts/sanity_chk/last_sanity.xml shippable/testresults/;
             aws s3 cp ./scripts/sanity_chk/last_sanity.xml ${S3_PATH}/sanitycheck.xml;
           fi;
-    on_failure:
-      - rm -rf sanity-out out-2nd-pass
-      - mkdir -p shippable/testresults
       - >
-          if [ -e compliance.xml ]; then
-            cp compliance.xml shippable/testresults/;
-            aws s3 cp compliance.xml ${S3_PATH}/;
-          fi;
-      - >
-          if [ -e ./scripts/sanity_chk/last_sanity.xml ]; then
-            cp ./scripts/sanity_chk/last_sanity.xml shippable/testresults/;
-            aws s3 cp ./scripts/sanity_chk/last_sanity.xml ${S3_PATH}/sanitycheck.xml;
+          if [ -e ./modified_tests.xml ]; then
+            cp ./modified_tests.xml shippable/testresults/;
+            aws s3 cp ./modified_tests.xml ${S3_PATH}/modified_tests.xml;
           fi;
 
 integrations:

scripts/ci/get_modified_tests.py

@@ -0,0 +1,73 @@
+#!/usr/bin/env python3
+
+# A script to generate a list of tests that have changed or added and create an
+# arguemnts file for sanitycheck to allow running those tests with --all
+
+import sys
+import re, os
+from email.utils import parseaddr
+import sh
+import logging
+import argparse
+
+if "ZEPHYR_BASE" not in os.environ:
+    logging.error("$ZEPHYR_BASE environment variable undefined.\n")
+    exit(1)
+
+logger = None
+
+repository_path = os.environ['ZEPHYR_BASE']
+sh_special_args = {
+    '_tty_out': False,
+    '_cwd': repository_path
+}
+
+def init_logs():
+    global logger
+    log_lev = os.environ.get('LOG_LEVEL', None)
+    level = logging.INFO
+    if log_lev == "DEBUG":
+        level = logging.DEBUG
+    elif log_lev == "ERROR":
+        level = logging.ERROR
+
+    console = logging.StreamHandler()
+    format = logging.Formatter('%(levelname)-8s: %(message)s')
+    console.setFormatter(format)
+    logger = logging.getLogger('')
+    logger.addHandler(console)
+    logger.setLevel(level)
+
+    logging.debug("Log init completed")
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+                description="Generate a sanitycheck argument for for tests "
+                            " that have changed")
+    parser.add_argument('-c', '--commits', default=None,
+            help="Commit range in the form: a..b")
+    return parser.parse_args()
+
+def main():
+    args = parse_args()
+    if not args.commits:
+        exit(1)
+
+    commit = sh.git("diff","--name-only", args.commits, **sh_special_args)
+    files = commit.split("\n")
+    tests = set()
+    for f in files:
+        d = os.path.dirname(f)
+        while d:
+            if os.path.exists(os.path.join(d, "testcase.yaml")) or os.path.exists(os.path.join(d, "sample.yaml")):
+                tests.add(d)
+                break
+            else:
+                d = os.path.dirname(d)
+
+    if tests:
+        print("-T\n%s\n--all" %("\n-T\n".join(tests)))
+
+if __name__ == "__main__":
+    main()
+