first commit

tests/repo_utils/test_tests_fetcher.py

@@ -0,0 +1,806 @@
+# Copyright 2022 The HuggingFace Team. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import shutil
+import sys
+import tempfile
+import unittest
+from contextlib import ExitStack, contextmanager
+from pathlib import Path
+from unittest.mock import patch
+
+from git import Repo
+
+from transformers.testing_utils import CaptureStdout
+
+
+REPO_PATH = os.path.abspath(os.path.dirname(os.path.dirname(os.path.dirname(__file__))))
+sys.path.append(os.path.join(REPO_PATH, "utils"))
+
+import tests_fetcher  # noqa: E402
+from tests_fetcher import (  # noqa: E402
+    checkout_commit,
+    clean_code,
+    create_reverse_dependency_map,
+    create_reverse_dependency_tree,
+    create_test_list_from_filter,
+    diff_is_docstring_only,
+    extract_imports,
+    get_all_tests,
+    get_diff,
+    get_module_dependencies,
+    get_repo_utils_tests,
+    get_tree_starting_at,
+    infer_tests_to_run,
+    init_test_examples_dependencies,
+    parse_commit_message,
+    print_tree_deps_of,
+    should_run_repo_utils_tests,
+)
+
+
+BERT_MODELING_FILE = "src/transformers/models/bert/modeling_bert.py"
+BERT_MODEL_FILE = """from ...modeling_utils import PreTrainedModel
+from ...utils import is_torch_available
+from .configuration_bert import BertConfig
+
+class BertModel:
+    '''
+    This is the docstring.
+    '''
+    This is the code
+"""
+
+BERT_MODEL_FILE_NEW_DOCSTRING = """from ...modeling_utils import PreTrainedModel
+from ...utils import is_torch_available
+from .configuration_bert import BertConfig
+
+class BertModel:
+    '''
+    This is the docstring. It has been updated.
+    '''
+    This is the code
+"""
+
+BERT_MODEL_FILE_NEW_CODE = """from ...modeling_utils import PreTrainedModel
+from ...utils import is_torch_available
+from .configuration_bert import BertConfig
+
+class BertModel:
+    '''
+    This is the docstring.
+    '''
+    This is the code. It has been updated
+"""
+
+
+def create_tmp_repo(tmp_dir, models=None):
+    """
+    Creates a repository in a temporary directory mimicking the structure of Transformers. Uses the list of models
+    provided (which defaults to just `["bert"]`).
+    """
+    tmp_dir = Path(tmp_dir)
+    if tmp_dir.exists():
+        shutil.rmtree(tmp_dir)
+    tmp_dir.mkdir(exist_ok=True)
+    repo = Repo.init(tmp_dir)
+
+    if models is None:
+        models = ["bert"]
+    class_names = [model[0].upper() + model[1:] for model in models]
+
+    transformers_dir = tmp_dir / "src" / "transformers"
+    transformers_dir.mkdir(parents=True, exist_ok=True)
+    with open(transformers_dir / "__init__.py", "w") as f:
+        init_lines = ["from .utils import cached_file, is_torch_available"]
+        init_lines.extend(
+            [f"from .models.{model} import {cls}Config, {cls}Model" for model, cls in zip(models, class_names)]
+        )
+        f.write("\n".join(init_lines) + "\n")
+    with open(transformers_dir / "configuration_utils.py", "w") as f:
+        f.write("from .utils import cached_file\n\ncode")
+    with open(transformers_dir / "modeling_utils.py", "w") as f:
+        f.write("from .utils import cached_file\n\ncode")
+
+    utils_dir = tmp_dir / "src" / "transformers" / "utils"
+    utils_dir.mkdir(exist_ok=True)
+    with open(utils_dir / "__init__.py", "w") as f:
+        f.write("from .hub import cached_file\nfrom .imports import is_torch_available\n")
+    with open(utils_dir / "hub.py", "w") as f:
+        f.write("import huggingface_hub\n\ncode")
+    with open(utils_dir / "imports.py", "w") as f:
+        f.write("code")
+
+    model_dir = tmp_dir / "src" / "transformers" / "models"
+    model_dir.mkdir(parents=True, exist_ok=True)
+    with open(model_dir / "__init__.py", "w") as f:
+        f.write("\n".join([f"import {model}" for model in models]))
+
+    for model, cls in zip(models, class_names):
+        model_dir = tmp_dir / "src" / "transformers" / "models" / model
+        model_dir.mkdir(parents=True, exist_ok=True)
+        with open(model_dir / "__init__.py", "w") as f:
+            f.write(f"from .configuration_{model} import {cls}Config\nfrom .modeling_{model} import {cls}Model\n")
+        with open(model_dir / f"configuration_{model}.py", "w") as f:
+            f.write("from ...configuration_utils import PreTrainedConfig\ncode")
+        with open(model_dir / f"modeling_{model}.py", "w") as f:
+            modeling_code = BERT_MODEL_FILE.replace("bert", model).replace("Bert", cls)
+            f.write(modeling_code)
+
+    test_dir = tmp_dir / "tests"
+    test_dir.mkdir(exist_ok=True)
+    with open(test_dir / "test_modeling_common.py", "w") as f:
+        f.write("from transformers.modeling_utils import PreTrainedModel\ncode")
+
+    for model, cls in zip(models, class_names):
+        test_model_dir = test_dir / "models" / model
+        test_model_dir.mkdir(parents=True, exist_ok=True)
+        (test_model_dir / "__init__.py").touch()
+        with open(test_model_dir / f"test_modeling_{model}.py", "w") as f:
+            f.write(
+                f"from transformers import {cls}Config, {cls}Model\nfrom ...test_modeling_common import ModelTesterMixin\n\ncode"
+            )
+
+    example_dir = tmp_dir / "examples"
+    example_dir.mkdir(exist_ok=True)
+    framework_dir = example_dir / "pytorch"
+    framework_dir.mkdir(exist_ok=True)
+    with open(framework_dir / "test_pytorch_examples.py", "w") as f:
+        f.write("""test_args = "run_glue.py"\n""")
+    glue_dir = framework_dir / "text-classification"
+    glue_dir.mkdir(exist_ok=True)
+    with open(glue_dir / "run_glue.py", "w") as f:
+        f.write("from transformers import BertModel\n\ncode")
+
+    repo.index.add(["examples", "src", "tests"])
+    repo.index.commit("Initial commit")
+    if "main" not in repo.heads:
+        repo.create_head("main")
+    repo.head.reference = repo.refs.main
+    if "master" in repo.heads:
+        repo.delete_head("master")
+    return repo
+
+
+@contextmanager
+def patch_transformer_repo_path(new_folder):
+    """
+    Temporarily patches the variables defines in `tests_fetcher` to use a different location for the repo.
+    """
+    old_repo_path = tests_fetcher.PATH_TO_REPO
+    tests_fetcher.PATH_TO_REPO = Path(new_folder).resolve()
+    tests_fetcher.PATH_TO_EXAMPLES = tests_fetcher.PATH_TO_REPO / "examples"
+    tests_fetcher.PATH_TO_TRANSFORMERS = tests_fetcher.PATH_TO_REPO / "src/transformers"
+    tests_fetcher.PATH_TO_TESTS = tests_fetcher.PATH_TO_REPO / "tests"
+    try:
+        yield
+    finally:
+        tests_fetcher.PATH_TO_REPO = old_repo_path
+        tests_fetcher.PATH_TO_EXAMPLES = tests_fetcher.PATH_TO_REPO / "examples"
+        tests_fetcher.PATH_TO_TRANSFORMERS = tests_fetcher.PATH_TO_REPO / "src/transformers"
+        tests_fetcher.PATH_TO_TESTS = tests_fetcher.PATH_TO_REPO / "tests"
+
+
+def commit_changes(filenames, contents, repo, commit_message="Commit"):
+    """
+    Commit new `contents` to `filenames` inside a given `repo`.
+    """
+    if not isinstance(filenames, list):
+        filenames = [filenames]
+    if not isinstance(contents, list):
+        contents = [contents]
+
+    folder = Path(repo.working_dir)
+    for filename, content in zip(filenames, contents):
+        with open(folder / filename, "w") as f:
+            f.write(content)
+    repo.index.add(filenames)
+    commit = repo.index.commit(commit_message)
+    return commit.hexsha
+
+
+class TestFetcherTester(unittest.TestCase):
+    def test_checkout_commit(self):
+        with tempfile.TemporaryDirectory() as tmp_folder:
+            tmp_folder = Path(tmp_folder)
+            repo = create_tmp_repo(tmp_folder)
+            initial_sha = repo.head.commit.hexsha
+            new_sha = commit_changes(BERT_MODELING_FILE, BERT_MODEL_FILE_NEW_DOCSTRING, repo)
+
+            assert repo.head.commit.hexsha == new_sha
+            with checkout_commit(repo, initial_sha):
+                assert repo.head.commit.hexsha == initial_sha
+                with open(tmp_folder / BERT_MODELING_FILE) as f:
+                    assert f.read() == BERT_MODEL_FILE
+
+            assert repo.head.commit.hexsha == new_sha
+            with open(tmp_folder / BERT_MODELING_FILE) as f:
+                assert f.read() == BERT_MODEL_FILE_NEW_DOCSTRING
+
+    def test_clean_code(self):
+        # Clean code removes all strings in triple quotes
+        assert clean_code('"""\nDocstring\n"""\ncode\n"""Long string"""\ncode\n') == "code\ncode"
+        assert clean_code("'''\nDocstring\n'''\ncode\n'''Long string'''\ncode\n'''") == "code\ncode"
+
+        # Clean code removes all comments
+        assert clean_code("code\n# Comment\ncode") == "code\ncode"
+        assert clean_code("code  # inline comment\ncode") == "code  \ncode"
+
+    def test_get_all_tests(self):
+        with tempfile.TemporaryDirectory() as tmp_folder:
+            tmp_folder = Path(tmp_folder)
+            create_tmp_repo(tmp_folder)
+            with patch_transformer_repo_path(tmp_folder):
+                assert get_all_tests() == ["tests/models/bert", "tests/test_modeling_common.py"]
+
+    def test_get_all_tests_on_full_repo(self):
+        all_tests = get_all_tests()
+        assert "tests/models/albert" in all_tests
+        assert "tests/models/bert" in all_tests
+        assert "tests/repo_utils" in all_tests
+        assert "tests/test_pipeline_mixin.py" in all_tests
+        assert "tests/models" not in all_tests
+        assert "tests/__pycache__" not in all_tests
+        assert "tests/models/albert/test_modeling_albert.py" not in all_tests
+        assert "tests/repo_utils/test_tests_fetcher.py" not in all_tests
+
+    def test_get_repo_utils_tests_on_full_repo(self):
+        repo_utils_tests = get_repo_utils_tests()
+        assert "tests/repo_utils/test_tests_fetcher.py" in repo_utils_tests
+
+    def test_should_run_repo_utils_tests(self):
+        assert should_run_repo_utils_tests(["utils/check_modeling_structure.py"])
+        assert not should_run_repo_utils_tests(["src/transformers/modeling_utils.py"])
+
+    def test_create_test_list_from_filter_routes_repo_utils_tests(self):
+        with tempfile.TemporaryDirectory() as tmp_folder:
+            create_test_list_from_filter(
+                [
+                    "tests/models/bert/test_modeling_bert.py",
+                    "tests/repo_utils/test_tests_fetcher.py",
+                ],
+                out_path=tmp_folder,
+            )
+
+            with open(Path(tmp_folder) / "tests_repo_utils_test_list.txt", encoding="utf-8") as f:
+                repo_utils_tests = f.read().splitlines()
+
+            assert repo_utils_tests == [
+                "tests/repo_utils/test_tests_fetcher.py",
+            ]
+
+    def test_create_test_list_from_filter_does_not_create_hub_job(self):
+        with tempfile.TemporaryDirectory() as tmp_folder:
+            create_test_list_from_filter(["tests/models/bert/test_modeling_bert.py"], out_path=tmp_folder)
+
+            assert (Path(tmp_folder) / "tests_torch_test_list.txt").exists()
+            assert not (Path(tmp_folder) / "tests_hub_test_list.txt").exists()
+
+    def test_infer_tests_to_run_adds_repo_utils_for_utils_changes(self):
+        with ExitStack() as stack:
+            stack.enter_context(patch.object(tests_fetcher, "commit_flags", {"test_all": False}, create=True))
+            stack.enter_context(
+                patch.object(
+                    tests_fetcher, "get_modified_python_files", return_value=["utils/check_modeling_structure.py"]
+                )
+            )
+            stack.enter_context(patch.object(tests_fetcher, "create_reverse_dependency_map", return_value={}))
+            stack.enter_context(
+                patch.object(tests_fetcher, "get_impacted_files_from_tiny_model_summary", return_value=[])
+            )
+            mock_create_test_list = stack.enter_context(patch.object(tests_fetcher, "create_test_list_from_filter"))
+            stack.enter_context(patch.object(tests_fetcher, "get_doctest_files", return_value=[]))
+            infer_tests_to_run("unused.txt", diff_with_last_commit=True)
+
+        test_files_to_run = mock_create_test_list.call_args.args[0]
+        assert "tests/repo_utils/test_tests_fetcher.py" in test_files_to_run
+
+    def test_diff_is_docstring_only(self):
+        with tempfile.TemporaryDirectory() as tmp_folder:
+            tmp_folder = Path(tmp_folder)
+            repo = create_tmp_repo(tmp_folder)
+
+            branching_point = repo.refs.main.commit
+            bert_file = BERT_MODELING_FILE
+            commit_changes(bert_file, BERT_MODEL_FILE_NEW_DOCSTRING, repo)
+            assert diff_is_docstring_only(repo, branching_point, bert_file)
+
+            commit_changes(bert_file, BERT_MODEL_FILE_NEW_CODE, repo)
+            assert not diff_is_docstring_only(repo, branching_point, bert_file)
+
+    def test_get_diff_ignores_docstring_only_changes(self):
+        """Files whose diff is only in docstrings/comments should be excluded from get_diff results."""
+        with tempfile.TemporaryDirectory() as tmp_folder:
+            tmp_folder = Path(tmp_folder)
+            repo = create_tmp_repo(tmp_folder)
+            branching_commit = repo.head.commit
+
+            # Docstring-only change: should NOT appear in diff
+            commit_changes(BERT_MODELING_FILE, BERT_MODEL_FILE_NEW_DOCSTRING, repo)
+            diff = get_diff(repo, repo.head.commit, [branching_commit])
+            assert BERT_MODELING_FILE not in diff
+
+            # Real code change: should appear in diff
+            commit_changes(BERT_MODELING_FILE, BERT_MODEL_FILE_NEW_CODE, repo)
+            diff = get_diff(repo, repo.head.commit, [branching_commit])
+            assert BERT_MODELING_FILE in diff
+
+    def test_extract_imports_relative(self):
+        with tempfile.TemporaryDirectory() as tmp_folder:
+            tmp_folder = Path(tmp_folder)
+            create_tmp_repo(tmp_folder)
+
+            expected_bert_imports = [
+                ("src/transformers/modeling_utils.py", ["PreTrainedModel"]),
+                ("src/transformers/utils/__init__.py", ["is_torch_available"]),
+                ("src/transformers/models/bert/configuration_bert.py", ["BertConfig"]),
+            ]
+            expected_utils_imports = [
+                ("src/transformers/utils/hub.py", ["cached_file"]),
+                ("src/transformers/utils/imports.py", ["is_torch_available"]),
+            ]
+            with patch_transformer_repo_path(tmp_folder):
+                assert extract_imports(BERT_MODELING_FILE) == expected_bert_imports
+                assert extract_imports("src/transformers/utils/__init__.py") == expected_utils_imports
+
+            with open(tmp_folder / BERT_MODELING_FILE, "w") as f:
+                f.write(
+                    "from ...utils import cached_file, is_torch_available\nfrom .configuration_bert import BertConfig\n"
+                )
+            expected_bert_imports = [
+                ("src/transformers/utils/__init__.py", ["cached_file", "is_torch_available"]),
+                ("src/transformers/models/bert/configuration_bert.py", ["BertConfig"]),
+            ]
+            with patch_transformer_repo_path(tmp_folder):
+                assert extract_imports(BERT_MODELING_FILE) == expected_bert_imports
+
+            # Test with multi-line imports
+            with open(tmp_folder / BERT_MODELING_FILE, "w") as f:
+                f.write(
+                    "from ...utils import (\n    cached_file,\n    is_torch_available\n)\nfrom .configuration_bert import BertConfig\n"
+                )
+            expected_bert_imports = [
+                ("src/transformers/models/bert/configuration_bert.py", ["BertConfig"]),
+                ("src/transformers/utils/__init__.py", ["cached_file", "is_torch_available"]),
+            ]
+            with patch_transformer_repo_path(tmp_folder):
+                assert extract_imports(BERT_MODELING_FILE) == expected_bert_imports
+
+    def test_extract_imports_absolute(self):
+        with tempfile.TemporaryDirectory() as tmp_folder:
+            tmp_folder = Path(tmp_folder)
+            create_tmp_repo(tmp_folder)
+
+            with open(tmp_folder / BERT_MODELING_FILE, "w") as f:
+                f.write(
+                    "from transformers.utils import cached_file, is_torch_available\nfrom transformers.models.bert.configuration_bert import BertConfig\n"
+                )
+            expected_bert_imports = [
+                ("src/transformers/utils/__init__.py", ["cached_file", "is_torch_available"]),
+                ("src/transformers/models/bert/configuration_bert.py", ["BertConfig"]),
+            ]
+            with patch_transformer_repo_path(tmp_folder):
+                assert extract_imports(BERT_MODELING_FILE) == expected_bert_imports
+
+            # Test with multi-line imports
+            with open(tmp_folder / BERT_MODELING_FILE, "w") as f:
+                f.write(
+                    "from transformers.utils import (\n    cached_file,\n    is_torch_available\n)\nfrom transformers.models.bert.configuration_bert import BertConfig\n"
+                )
+            expected_bert_imports = [
+                ("src/transformers/models/bert/configuration_bert.py", ["BertConfig"]),
+                ("src/transformers/utils/__init__.py", ["cached_file", "is_torch_available"]),
+            ]
+            with patch_transformer_repo_path(tmp_folder):
+                assert extract_imports(BERT_MODELING_FILE) == expected_bert_imports
+
+            # Test with base imports
+            with open(tmp_folder / BERT_MODELING_FILE, "w") as f:
+                f.write(
+                    "from transformers.utils import (\n    cached_file,\n    is_torch_available\n)\nfrom transformers import BertConfig\n"
+                )
+            expected_bert_imports = [
+                ("src/transformers/__init__.py", ["BertConfig"]),
+                ("src/transformers/utils/__init__.py", ["cached_file", "is_torch_available"]),
+            ]
+            with patch_transformer_repo_path(tmp_folder):
+                assert extract_imports(BERT_MODELING_FILE) == expected_bert_imports
+
+    def test_get_module_dependencies(self):
+        with tempfile.TemporaryDirectory() as tmp_folder:
+            tmp_folder = Path(tmp_folder)
+            create_tmp_repo(tmp_folder)
+
+            expected_bert_dependencies = [
+                "src/transformers/modeling_utils.py",
+                "src/transformers/models/bert/configuration_bert.py",
+                "src/transformers/utils/imports.py",
+            ]
+            with patch_transformer_repo_path(tmp_folder):
+                assert get_module_dependencies(BERT_MODELING_FILE) == expected_bert_dependencies
+
+            expected_test_bert_dependencies = [
+                "tests/test_modeling_common.py",
+                "src/transformers/models/bert/configuration_bert.py",
+                "src/transformers/models/bert/modeling_bert.py",
+            ]
+
+            with patch_transformer_repo_path(tmp_folder):
+                assert (
+                    get_module_dependencies("tests/models/bert/test_modeling_bert.py")
+                    == expected_test_bert_dependencies
+                )
+
+            # Test with a submodule
+            (tmp_folder / "src/transformers/utils/logging.py").touch()
+            with open(tmp_folder / BERT_MODELING_FILE, "a") as f:
+                f.write("from ...utils import logging\n")
+
+            expected_bert_dependencies = [
+                "src/transformers/modeling_utils.py",
+                "src/transformers/models/bert/configuration_bert.py",
+                "src/transformers/utils/logging.py",
+                "src/transformers/utils/imports.py",
+            ]
+            with patch_transformer_repo_path(tmp_folder):
+                assert get_module_dependencies(BERT_MODELING_FILE) == expected_bert_dependencies
+
+            # Test with an object non-imported in the init
+            create_tmp_repo(tmp_folder)
+            with open(tmp_folder / BERT_MODELING_FILE, "a") as f:
+                f.write("from ...utils import CONSTANT\n")
+
+            expected_bert_dependencies = [
+                "src/transformers/modeling_utils.py",
+                "src/transformers/models/bert/configuration_bert.py",
+                "src/transformers/utils/__init__.py",
+                "src/transformers/utils/imports.py",
+            ]
+            with patch_transformer_repo_path(tmp_folder):
+                assert get_module_dependencies(BERT_MODELING_FILE) == expected_bert_dependencies
+
+            # Test with an example
+            create_tmp_repo(tmp_folder)
+
+            expected_example_dependencies = ["src/transformers/models/bert/modeling_bert.py"]
+
+            with patch_transformer_repo_path(tmp_folder):
+                assert (
+                    get_module_dependencies("examples/pytorch/text-classification/run_glue.py")
+                    == expected_example_dependencies
+                )
+
+    def test_create_reverse_dependency_tree(self):
+        with tempfile.TemporaryDirectory() as tmp_folder:
+            tmp_folder = Path(tmp_folder)
+            create_tmp_repo(tmp_folder)
+            with patch_transformer_repo_path(tmp_folder):
+                tree = create_reverse_dependency_tree()
+
+            init_edges = [
+                "src/transformers/utils/hub.py",
+                "src/transformers/utils/imports.py",
+                "src/transformers/models/bert/configuration_bert.py",
+                "src/transformers/models/bert/modeling_bert.py",
+            ]
+            assert {f for f, g in tree if g == "src/transformers/__init__.py"} == set(init_edges)
+
+            bert_edges = [
+                "src/transformers/modeling_utils.py",
+                "src/transformers/utils/imports.py",
+                "src/transformers/models/bert/configuration_bert.py",
+            ]
+            assert {f for f, g in tree if g == "src/transformers/models/bert/modeling_bert.py"} == set(bert_edges)
+
+            test_bert_edges = [
+                "tests/test_modeling_common.py",
+                "src/transformers/models/bert/configuration_bert.py",
+                "src/transformers/models/bert/modeling_bert.py",
+            ]
+            assert {f for f, g in tree if g == "tests/models/bert/test_modeling_bert.py"} == set(test_bert_edges)
+
+    def test_get_tree_starting_at(self):
+        with tempfile.TemporaryDirectory() as tmp_folder:
+            tmp_folder = Path(tmp_folder)
+            create_tmp_repo(tmp_folder)
+            with patch_transformer_repo_path(tmp_folder):
+                edges = create_reverse_dependency_tree()
+
+                bert_tree = get_tree_starting_at("src/transformers/models/bert/modeling_bert.py", edges)
+                config_utils_tree = get_tree_starting_at("src/transformers/configuration_utils.py", edges)
+
+            expected_bert_tree = [
+                "src/transformers/models/bert/modeling_bert.py",
+                [("src/transformers/models/bert/modeling_bert.py", "tests/models/bert/test_modeling_bert.py")],
+            ]
+            assert bert_tree == expected_bert_tree
+
+            expected_config_tree = [
+                "src/transformers/configuration_utils.py",
+                [("src/transformers/configuration_utils.py", "src/transformers/models/bert/configuration_bert.py")],
+                [
+                    ("src/transformers/models/bert/configuration_bert.py", "tests/models/bert/test_modeling_bert.py"),
+                    (
+                        "src/transformers/models/bert/configuration_bert.py",
+                        "src/transformers/models/bert/modeling_bert.py",
+                    ),
+                ],
+            ]
+            # Order of the edges is random
+            assert [set(v) for v in config_utils_tree] == [set(v) for v in expected_config_tree]
+
+    def test_print_tree_deps_of(self):
+        with tempfile.TemporaryDirectory() as tmp_folder:
+            tmp_folder = Path(tmp_folder)
+            create_tmp_repo(tmp_folder)
+
+            # There are two possible outputs since the order of the last two lines is non-deterministic.
+            expected_std_out = """src/transformers/models/bert/modeling_bert.py
+  tests/models/bert/test_modeling_bert.py
+src/transformers/configuration_utils.py
+  src/transformers/models/bert/configuration_bert.py
+    src/transformers/models/bert/modeling_bert.py
+    tests/models/bert/test_modeling_bert.py"""
+
+            expected_std_out_2 = """src/transformers/models/bert/modeling_bert.py
+  tests/models/bert/test_modeling_bert.py
+src/transformers/configuration_utils.py
+  src/transformers/models/bert/configuration_bert.py
+    tests/models/bert/test_modeling_bert.py
+    src/transformers/models/bert/modeling_bert.py"""
+
+            with patch_transformer_repo_path(tmp_folder), CaptureStdout() as cs:
+                print_tree_deps_of("src/transformers/models/bert/modeling_bert.py")
+                print_tree_deps_of("src/transformers/configuration_utils.py")
+
+            assert cs.out.strip() in [expected_std_out, expected_std_out_2]
+
+    def test_init_test_examples_dependencies(self):
+        with tempfile.TemporaryDirectory() as tmp_folder:
+            tmp_folder = Path(tmp_folder).resolve()
+            create_tmp_repo(tmp_folder)
+
+            expected_example_deps = {
+                "examples/pytorch/test_pytorch_examples.py": [
+                    "examples/pytorch/text-classification/run_glue.py",
+                    "examples/pytorch/test_pytorch_examples.py",
+                ],
+            }
+
+            expected_examples = {
+                "examples/pytorch/test_pytorch_examples.py",
+                "examples/pytorch/text-classification/run_glue.py",
+            }
+
+            with patch_transformer_repo_path(tmp_folder):
+                example_deps, all_examples = init_test_examples_dependencies()
+                assert example_deps == expected_example_deps
+                assert {str(f.relative_to(tmp_folder)) for f in all_examples} == expected_examples
+
+    def test_create_reverse_dependency_map(self):
+        with tempfile.TemporaryDirectory() as tmp_folder:
+            tmp_folder = Path(tmp_folder)
+            create_tmp_repo(tmp_folder)
+            with patch_transformer_repo_path(tmp_folder):
+                reverse_map = create_reverse_dependency_map()
+
+            # impact of BERT modeling file (note that we stop at the inits and don't go down further)
+            expected_bert_deps = {
+                "src/transformers/__init__.py",
+                "src/transformers/models/bert/__init__.py",
+                "tests/models/bert/test_modeling_bert.py",
+                "examples/pytorch/test_pytorch_examples.py",
+                "examples/pytorch/text-classification/run_glue.py",
+            }
+            assert set(reverse_map["src/transformers/models/bert/modeling_bert.py"]) == expected_bert_deps
+
+            # init gets the direct deps (and their recursive deps)
+            expected_init_deps = {
+                "src/transformers/utils/__init__.py",
+                "src/transformers/utils/hub.py",
+                "src/transformers/utils/imports.py",
+                "src/transformers/models/bert/__init__.py",
+                "src/transformers/models/bert/configuration_bert.py",
+                "src/transformers/models/bert/modeling_bert.py",
+                "src/transformers/configuration_utils.py",
+                "src/transformers/modeling_utils.py",
+                "tests/test_modeling_common.py",
+                "tests/models/bert/test_modeling_bert.py",
+                "examples/pytorch/test_pytorch_examples.py",
+                "examples/pytorch/text-classification/run_glue.py",
+            }
+            assert set(reverse_map["src/transformers/__init__.py"]) == expected_init_deps
+
+            expected_init_deps = {
+                "src/transformers/__init__.py",
+                "src/transformers/models/bert/configuration_bert.py",
+                "src/transformers/models/bert/modeling_bert.py",
+                "tests/models/bert/test_modeling_bert.py",
+                "examples/pytorch/test_pytorch_examples.py",
+                "examples/pytorch/text-classification/run_glue.py",
+            }
+            assert set(reverse_map["src/transformers/models/bert/__init__.py"]) == expected_init_deps
+
+            # Test that with more models init of bert only gets deps to bert.
+            create_tmp_repo(tmp_folder, models=["bert", "gpt2"])
+            with patch_transformer_repo_path(tmp_folder):
+                reverse_map = create_reverse_dependency_map()
+
+            # init gets the direct deps (and their recursive deps)
+            expected_init_deps = {
+                "src/transformers/__init__.py",
+                "src/transformers/models/bert/configuration_bert.py",
+                "src/transformers/models/bert/modeling_bert.py",
+                "tests/models/bert/test_modeling_bert.py",
+                "examples/pytorch/test_pytorch_examples.py",
+                "examples/pytorch/text-classification/run_glue.py",
+            }
+            assert set(reverse_map["src/transformers/models/bert/__init__.py"]) == expected_init_deps
+
+    @unittest.skip("Broken for now TODO @ArthurZucker")
+    def test_infer_tests_to_run(self):
+        with tempfile.TemporaryDirectory() as tmp_folder:
+            tmp_folder = Path(tmp_folder)
+            models = ["bert", "gpt2"] + [f"bert{i}" for i in range(10)]
+            repo = create_tmp_repo(tmp_folder, models=models)
+
+            commit_changes("src/transformers/models/bert/modeling_bert.py", BERT_MODEL_FILE_NEW_CODE, repo)
+
+            example_tests = {
+                "examples/pytorch/test_pytorch_examples.py",
+            }
+
+            with patch_transformer_repo_path(tmp_folder):
+                infer_tests_to_run(tmp_folder / "test-output.txt", diff_with_last_commit=True)
+                with open(tmp_folder / "test-output.txt") as f:
+                    tests_to_run = f.read()
+                with open(tmp_folder / "examples_test_list.txt") as f:
+                    example_tests_to_run = f.read()
+
+            assert tests_to_run == "tests/models/bert/test_modeling_bert.py"
+            assert set(example_tests_to_run.split(" ")) == example_tests
+
+            # Fake a new model addition
+            repo = create_tmp_repo(tmp_folder, models=models)
+
+            branch = repo.create_head("new_model")
+            branch.checkout()
+
+            with open(tmp_folder / "src/transformers/__init__.py", "a") as f:
+                f.write("from .models.t5 import T5Config, T5Model\n")
+
+            model_dir = tmp_folder / "src/transformers/models/t5"
+            model_dir.mkdir(exist_ok=True)
+
+            with open(model_dir / "__init__.py", "w") as f:
+                f.write("from .configuration_t5 import T5Config\nfrom .modeling_t5 import T5Model\n")
+            with open(model_dir / "configuration_t5.py", "w") as f:
+                f.write("from ...configuration_utils import PreTrainedConfig\ncode")
+            with open(model_dir / "modeling_t5.py", "w") as f:
+                modeling_code = BERT_MODEL_FILE.replace("bert", "t5").replace("Bert", "T5")
+                f.write(modeling_code)
+
+            test_dir = tmp_folder / "tests/models/t5"
+            test_dir.mkdir(exist_ok=True)
+            (test_dir / "__init__.py").touch()
+            with open(test_dir / "test_modeling_t5.py", "w") as f:
+                f.write(
+                    "from transformers import T5Config, T5Model\nfrom ...test_modeling_common import ModelTesterMixin\n\ncode"
+                )
+
+            repo.index.add(["src", "tests"])
+            repo.index.commit("Add T5 model")
+
+            with patch_transformer_repo_path(tmp_folder):
+                infer_tests_to_run(tmp_folder / "test-output.txt")
+                with open(tmp_folder / "test-output.txt") as f:
+                    tests_to_run = f.read()
+                with open(tmp_folder / "examples_test_list.txt") as f:
+                    example_tests_to_run = f.read()
+
+            expected_tests = {
+                "tests/models/bert/test_modeling_bert.py",
+                "tests/models/gpt2/test_modeling_gpt2.py",
+                "tests/models/t5/test_modeling_t5.py",
+                "tests/test_modeling_common.py",
+            }
+            assert set(tests_to_run.split(" ")) == expected_tests
+            assert set(example_tests_to_run.split(" ")) == example_tests
+
+            with patch_transformer_repo_path(tmp_folder):
+                infer_tests_to_run(tmp_folder / "test-output.txt")
+                with open(tmp_folder / "test-output.txt") as f:
+                    tests_to_run = f.read()
+                with open(tmp_folder / "examples_test_list.txt") as f:
+                    example_tests_to_run = f.read()
+
+            expected_tests = [f"tests/models/{name}/test_modeling_{name}.py" for name in models + ["t5"]]
+            expected_tests = set(expected_tests + ["tests/test_modeling_common.py"])
+            assert set(tests_to_run.split(" ")) == expected_tests
+            assert set(example_tests_to_run.split(" ")) == example_tests
+
+    @unittest.skip("Broken for now TODO @ArthurZucker")
+    def test_infer_tests_to_run_with_test_modifs(self):
+        with tempfile.TemporaryDirectory() as tmp_folder:
+            tmp_folder = Path(tmp_folder)
+            models = ["bert", "gpt2"] + [f"bert{i}" for i in range(10)]
+            repo = create_tmp_repo(tmp_folder, models=models)
+
+            commit_changes(
+                "tests/models/bert/test_modeling_bert.py",
+                "from transformers import BertConfig, BertModel\nfrom ...test_modeling_common import ModelTesterMixin\n\ncode1",
+                repo,
+            )
+
+            with patch_transformer_repo_path(tmp_folder):
+                infer_tests_to_run(tmp_folder / "test-output.txt", diff_with_last_commit=True)
+                with open(tmp_folder / "test-output.txt") as f:
+                    tests_to_run = f.read()
+
+            assert tests_to_run == "tests/models/bert/test_modeling_bert.py"
+
+    @unittest.skip("Broken for now TODO @ArthurZucker")
+    def test_infer_tests_to_run_with_examples_modifs(self):
+        with tempfile.TemporaryDirectory() as tmp_folder:
+            tmp_folder = Path(tmp_folder)
+            models = ["bert", "gpt2"]
+            repo = create_tmp_repo(tmp_folder, models=models)
+
+            # Modification in one example trigger the corresponding test
+            commit_changes(
+                "examples/pytorch/text-classification/run_glue.py",
+                "from transformers import BertModeln\n\ncode1",
+                repo,
+            )
+
+            with patch_transformer_repo_path(tmp_folder):
+                infer_tests_to_run(tmp_folder / "test-output.txt", diff_with_last_commit=True)
+                with open(tmp_folder / "examples_test_list.txt") as f:
+                    example_tests_to_run = f.read()
+
+            assert example_tests_to_run == "examples/pytorch/test_pytorch_examples.py"
+
+            # Modification in one test example file trigger that test
+            repo = create_tmp_repo(tmp_folder, models=models)
+            commit_changes(
+                "examples/pytorch/test_pytorch_examples.py",
+                """test_args = "run_glue.py"\nmore_code""",
+                repo,
+            )
+
+            with patch_transformer_repo_path(tmp_folder):
+                infer_tests_to_run(tmp_folder / "test-output.txt", diff_with_last_commit=True)
+                with open(tmp_folder / "examples_test_list.txt") as f:
+                    example_tests_to_run = f.read()
+
+            assert example_tests_to_run == "examples/pytorch/test_pytorch_examples.py"
+
+    def test_parse_commit_message(self):
+        assert parse_commit_message("Normal commit") == {"skip": False, "no_filter": False, "test_all": False}
+
+        assert parse_commit_message("[skip ci] commit") == {"skip": True, "no_filter": False, "test_all": False}
+        assert parse_commit_message("[ci skip] commit") == {"skip": True, "no_filter": False, "test_all": False}
+        assert parse_commit_message("[skip-ci] commit") == {"skip": True, "no_filter": False, "test_all": False}
+        assert parse_commit_message("[skip_ci] commit") == {"skip": True, "no_filter": False, "test_all": False}
+
+        assert parse_commit_message("[no filter] commit") == {"skip": False, "no_filter": True, "test_all": False}
+        assert parse_commit_message("[no-filter] commit") == {"skip": False, "no_filter": True, "test_all": False}
+        assert parse_commit_message("[no_filter] commit") == {"skip": False, "no_filter": True, "test_all": False}
+        assert parse_commit_message("[filter-no] commit") == {"skip": False, "no_filter": True, "test_all": False}
+
+        assert parse_commit_message("[test all] commit") == {"skip": False, "no_filter": False, "test_all": True}
+        assert parse_commit_message("[all test] commit") == {"skip": False, "no_filter": False, "test_all": True}
+        assert parse_commit_message("[test-all] commit") == {"skip": False, "no_filter": False, "test_all": True}
+        assert parse_commit_message("[all_test] commit") == {"skip": False, "no_filter": False, "test_all": True}