add tests.yml anywhere nbgrader config appears in ui tests

nbgrader/tests/ui-tests/files/tests.yml

@@ -0,0 +1,310 @@
+#kernel_name:
+#    setup:
+#    hash:
+#    dispatch:
+#    normalize:
+#    check:
+#    success:
+#
+#    templates:
+#        default:
+#            - test:
+#              fail:
+#
+#        datatype:
+#            - test:
+#              fail:
+
+
+python3:
+    setup: "from hashlib import sha1"
+    hash: 'sha1({{snippet}}.encode("utf-8")+b"{{salt}}").hexdigest()'
+    dispatch: "type({{snippet}})"
+    normalize: "str({{snippet}})"
+    check: 'assert {{snippet}} == "{{value}}", "{{message}}"'
+    success: "print('Success!')"
+
+    templates:
+        default:
+            - test: "type({{snippet}})"
+              fail: "type of {{snippet}} is not correct"
+
+            - test: "{{snippet}}"
+              fail: "value of {{snippet}} is not correct"
+
+        int:
+            - test: "type({{snippet}})"
+              fail: "type of {{snippet}} is not int. Please make sure it is int and not np.int64, etc. You can cast your value into an int using int()"
+
+            - test: "{{snippet}}"
+              fail: "value of {{snippet}} is not correct"
+
+        float:
+            - test: "type({{snippet}})"
+              fail: "type of {{snippet}} is not float. Please make sure it is float and not np.float64, etc. You can cast your value into a float using float()"
+
+            - test: "round({{snippet}}, 2)"
+              fail: "value of {{snippet}} is not correct (rounded to 2 decimal places)"
+
+        set:
+            - test: "type({{snippet}})"
+              fail: "type of {{snippet}} is not set. {{snippet}} should be a set"
+
+            - test: "len({{snippet}})"
+              fail: "length of {{snippet}} is not correct"
+
+            - test: "{{snippet}}"
+              fail: "value of {{snippet}} is not correct"
+
+        list:
+            - test: "type({{snippet}})"
+              fail: "type of {{snippet}} is not list. {{snippet}} should be a list"
+
+            - test: "len({{snippet}})"
+              fail: "length of {{snippet}} is not correct"
+
+            - test: "sorted(map(str, {{snippet}}))"
+              fail: "values of {{snippet}} are not correct"
+
+            - test: "{{snippet}}"
+              fail: "order of elements of {{snippet}} is not correct"
+
+        tuple:
+            - test: "type({{snippet}})"
+              fail: "type of {{snippet}} is not tuple. {{snippet}} should be a tuple"
+
+            - test: "len({{snippet}})"
+              fail: "length of {{snippet}} is not correct"
+
+            - test: "sorted(map(str, {{snippet}}))"
+              fail: "values of {{snippet}} are not correct"
+
+            - test: "{{snippet}}"
+              fail: "order of elements of {{snippet}} is not correct"
+
+        str:
+
+            - test: "type({{snippet}})"
+              fail: "type of {{snippet}} is not str. {{snippet}} should be an str"
+
+            - test: "len({{snippet}})"
+              fail: "length of {{snippet}} is not correct"
+
+            - test: "{{snippet}}.lower()"
+              fail: "value of {{snippet}} is not correct"
+
+            - test: "{{snippet}}"
+              fail: "correct string value of {{snippet}} but incorrect case of letters"
+
+        dict:
+
+            - test: "type({{snippet}})"
+              fail: "type of {{snippet}} is not dict. {{snippet}} should be a dict"
+
+            - test: "len(list({{snippet}}.keys()))"
+              fail: "number of keys of {{snippet}} is not correct"
+
+            - test: "sorted(map(str, {{snippet}}.keys()))"
+              fail: "keys of {{snippet}} are not correct"
+
+            - test: "sorted(map(str, {{snippet}}.values()))"
+              fail: "correct keys, but values of {{snippet}} are not correct"
+
+            - test: "{{snippet}}"
+              fail: "correct keys and values, but incorrect correspondence in keys and values of {{snippet}}"
+
+        bool:
+            - test: "type({{snippet}})"
+              fail: "type of {{snippet}} is not bool. {{snippet}} should be a bool"
+
+            - test: "{{snippet}}"
+              fail: "boolean value of {{snippet}} is not correct"
+
+        type:
+            - test: "{{snippet}}"
+              fail: "type of {{snippet}} is not correct"
+
+# ---------------------------------------------
+
+python:
+    setup: "from hashlib import sha1"
+    hash: 'sha1({{snippet}}.encode("utf-8")+b"{{salt}}").hexdigest()'
+    dispatch: "type({{snippet}})"
+    normalize: "str({{snippet}})"
+    check: 'assert {{snippet}} == "{{value}}", "{{message}}"'
+    success: "print('Success!')"
+
+    templates:
+        default:
+            - test: "type({{snippet}})"
+              fail: "type of {{snippet}} is not correct"
+
+            - test: "{{snippet}}"
+              fail: "value of {{snippet}} is not correct"
+
+        int:
+            - test: "type({{snippet}})"
+              fail: "type of {{snippet}} is not int. Please make sure it is int and not np.int64, etc. You can cast your value into an int using int()"
+
+            - test: "{{snippet}}"
+              fail: "value of {{snippet}} is not correct"
+
+        float:
+            - test: "type({{snippet}})"
+              fail: "type of {{snippet}} is not float. Please make sure it is float and not np.float64, etc. You can cast your value into a float using float()"
+
+            - test: "round({{snippet}}, 2)"
+              fail: "value of {{snippet}} is not correct (rounded to 2 decimal places)"
+
+        set:
+            - test: "type({{snippet}})"
+              fail: "type of {{snippet}} is not set. {{snippet}} should be a set"
+
+            - test: "len({{snippet}})"
+              fail: "length of {{snippet}} is not correct"
+
+            - test: "{{snippet}}"
+              fail: "value of {{snippet}} is not correct"
+
+        list:
+            - test: "type({{snippet}})"
+              fail: "type of {{snippet}} is not list. {{snippet}} should be a list"
+
+            - test: "len({{snippet}})"
+              fail: "length of {{snippet}} is not correct"
+
+            - test: "sorted(map(str, {{snippet}}))"
+              fail: "values of {{snippet}} are not correct"
+
+            - test: "{{snippet}}"
+              fail: "order of elements of {{snippet}} is not correct"
+
+        tuple:
+            - test: "type({{snippet}})"
+              fail: "type of {{snippet}} is not tuple. {{snippet}} should be a tuple"
+
+            - test: "len({{snippet}})"
+              fail: "length of {{snippet}} is not correct"
+
+            - test: "sorted(map(str, {{snippet}}))"
+              fail: "values of {{snippet}} are not correct"
+
+            - test: "{{snippet}}"
+              fail: "order of elements of {{snippet}} is not correct"
+
+        str:
+
+            - test: "type({{snippet}})"
+              fail: "type of {{snippet}} is not str. {{snippet}} should be an str"
+
+            - test: "len({{snippet}})"
+              fail: "length of {{snippet}} is not correct"
+
+            - test: "{{snippet}}.lower()"
+              fail: "value of {{snippet}} is not correct"
+
+            - test: "{{snippet}}"
+              fail: "correct string value of {{snippet}} but incorrect case of letters"
+
+        dict:
+
+            - test: "type({{snippet}})"
+              fail: "type of {{snippet}} is not dict. {{snippet}} should be a dict"
+
+            - test: "len(list({{snippet}}.keys()))"
+              fail: "number of keys of {{snippet}} is not correct"
+
+            - test: "sorted(map(str, {{snippet}}.keys()))"
+              fail: "keys of {{snippet}} are not correct"
+
+            - test: "sorted(map(str, {{snippet}}.values()))"
+              fail: "correct keys, but values of {{snippet}} are not correct"
+
+            - test: "{{snippet}}"
+              fail: "correct keys and values, but incorrect correspondence in keys and values of {{snippet}}"
+
+        bool:
+            - test: "type({{snippet}})"
+              fail: "type of {{snippet}} is not bool. {{snippet}} should be a bool"
+
+            - test: "{{snippet}}"
+              fail: "boolean value of {{snippet}} is not correct"
+
+        type:
+            - test: "{{snippet}}"
+              fail: "type of {{snippet}} is not correct"
+
+
+
+# --------------------------------------------------------------------------------------------------
+ir:
+    setup: 'library(digest)'
+    hash: 'digest(paste({{snippet}}, "{{salt}}"))'
+    dispatch: 'class({{snippet}})'
+    normalize: 'toString({{snippet}})'
+    check: 'stopifnot("{{message}}"= setequal({{snippet}}, "{{value}}"))'
+    success: "print('Success!')"
+
+    templates:
+        default:
+            - test: "class({{snippet}})"
+              fail: "type of {{snippet}} is not correct"
+
+            - test: "{{snippet}}"
+              fail: "value of {{snippet}} is not correct"
+
+        integer:
+            - test: "class({{snippet}})"
+              fail: "type of {{snippet}} is not integer"
+
+            - test: "length({{snippet}})"
+              fail: "length of {{snippet}} is not correct"
+
+            - test: "sort({{snippet}})"
+              fail: "values of {{snippet}} are not correct"
+
+        numeric:
+            - test: "class({{snippet}})"
+              fail: "type of {{snippet}} is not double"
+
+            - test: "round({{snippet}}, 2)"
+              fail: "value of {{snippet}} is not correct (rounded to 2 decimal places)"
+
+            - test: "length({{snippet}})"
+              fail: "length of {{snippet}} is not correct"
+
+            - test: "sort({{snippet}})"
+              fail: "values of {{snippet}} are not correct"
+
+        list:
+            - test: "class({{snippet}})"
+              fail: "type of {{snippet}} is not list"
+
+            - test: "length({{snippet}})"
+              fail: "length of {{snippet}} is not correct"
+
+            - test: "sort(c(names({{snippet}})))"
+              fail: "values of {{snippet}} names are not correct"
+
+            - test: "{{snippet}}"
+              fail: "order of elements of {{snippet}} is not correct"
+
+        character:
+            - test: "class({{snippet}})"
+              fail: "type of {{snippet}} is not list"
+
+            - test: "length({{snippet}})"
+              fail: "length of {{snippet}} is not correct"
+
+            - test: "tolower({{snippet}})"
+              fail: "value of {{snippet}} is not correct"
+
+            - test: "{{snippet}}"
+              fail: "correct string value of {{snippet}} but incorrect case of letters"
+
+        logical:
+            - test: "class({{snippet}})"
+              fail: "type of {{snippet}} is not logical"
+
+            - test: "{{snippet}}"
+              fail: "logical value of {{snippet}} is not correct"

nbgrader/tests/ui-tests/utils/run_jupyter_lab.py

@@ -18,6 +18,9 @@
 shutil.copyfile("./nbgrader/tests/ui-tests/files/nbgrader_config.py",
                 join(root_dir, "nbgrader_config.py"))
 
+shutil.copyfile("./nbgrader/tests/ui-tests/files/tests.yml",
+                join(root_dir, "tests.yml"))
+
 subprocess.check_call([sys.executable, "-m", "jupyter", "server", "extension", "enable", "--user", "--py", "nbgrader"], env=env)
 subprocess.check_call([sys.executable, "-m", "jupyter", "labextension", "develop", "--overwrite", "."])
 

nbgrader/tests/ui-tests/utils/run_jupyter_notebook.py

@@ -18,6 +18,9 @@
 shutil.copyfile("./nbgrader/tests/ui-tests/files/nbgrader_config.py",
                 join(root_dir, "nbgrader_config.py"))
 
+shutil.copyfile("./nbgrader/tests/ui-tests/files/tests.yml",
+                join(root_dir, "tests.yml"))
+
 subprocess.check_call([sys.executable, "-m", "jupyter", "server", "extension", "enable", "--user", "--py", "nbgrader"], env=env)
 subprocess.check_call([sys.executable, "-m", "jupyter", "labextension", "develop", "--overwrite", "."])