Use syntax updated with Python 3.7 and avoid unnecessary list creatio…

…ns (facebookresearch#2689)

* Use syntax updated with Python 3.7

* Avoid innecessary list creations

* Fix a PR comment

* Fix a `zip`

* Update future_annotations.py

build_helpers/test_helpers.py

@@ -112,8 +112,8 @@ def test_find(
         scan_exclude=scan_exclude,
     )
 
-    ret_set = set([str(Path(x)) for x in ret])
-    expected_set = set([str(Path(x)) for x in expected])
+    ret_set = {str(Path(x)) for x in ret}
+    expected_set = {str(Path(x)) for x in expected}
     assert ret_set == expected_set
 
 

examples/plugins/example_configsource_plugin/setup.py

@@ -2,7 +2,7 @@
 # type: ignore
 from setuptools import find_namespace_packages, setup
 
-with open("README.md", "r") as fh:
+with open("README.md") as fh:
     LONG_DESC = fh.read()
     setup(
         name="hydra-example-configsource",

examples/plugins/example_generic_plugin/setup.py

@@ -2,7 +2,7 @@
 # type: ignore
 from setuptools import find_namespace_packages, setup
 
-with open("README.md", "r") as fh:
+with open("README.md") as fh:
     LONG_DESC = fh.read()
     setup(
         name="hydra-example-generic-plugin",

examples/plugins/example_launcher_plugin/setup.py

@@ -2,7 +2,7 @@
 # type: ignore
 from setuptools import find_namespace_packages, setup
 
-with open("README.md", "r") as fh:
+with open("README.md") as fh:
     LONG_DESC = fh.read()
     setup(
         name="hydra-example-launcher",

examples/plugins/example_registered_plugin/setup.py

@@ -2,7 +2,7 @@
 # type: ignore
 from setuptools import setup
 
-with open("README.md", "r") as fh:
+with open("README.md") as fh:
     LONG_DESC = fh.read()
 setup(
     name="hydra-example-registered-plugin",

examples/plugins/example_searchpath_plugin/setup.py

@@ -3,7 +3,7 @@
 from setuptools import find_namespace_packages, setup, find_packages
 
 
-with open("README.md", "r") as fh:
+with open("README.md") as fh:
     LONG_DESC = fh.read()
     setup(
         name="hydra-example-searchpath-plugin",

examples/plugins/example_sweeper_plugin/setup.py

@@ -2,7 +2,7 @@
 # type: ignore
 from setuptools import find_namespace_packages, setup
 
-with open("README.md", "r") as fh:
+with open("README.md") as fh:
     LONG_DESC = fh.read()
     setup(
         name="hydra-example-sweeper",

hydra/_internal/core_plugins/basic_sweeper.py

@@ -62,7 +62,7 @@ def __init__(
         """
         Instantiates
         """
-        super(BasicSweeper, self).__init__()
+        super().__init__()
 
         if params is None:
             params = {}

hydra/_internal/core_plugins/zsh_completion.py

@@ -10,7 +10,7 @@
 
 class ZshCompletion(CompletionPlugin):
     def __init__(self, config_loader: ConfigLoader):
-        super(ZshCompletion, self).__init__(config_loader)
+        super().__init__(config_loader)
         from hydra._internal.core_plugins.bash_completion import BashCompletion
 
         self.delegate = BashCompletion(config_loader)

hydra/_internal/defaults_list.py

@@ -312,7 +312,7 @@ def _check_not_missing(
                 group_path,
                 results_filter=ObjectType.CONFIG,
             )
-            opt_list = "\n".join(["\t" + x for x in options])
+            opt_list = "\n".join("\t" + x for x in options)
             msg = dedent(
                 f"""\
                 You must specify '{override_key}', e.g, {override_key}=<OPTION>
@@ -778,7 +778,7 @@ def config_not_found_error(repo: IConfigRepository, tree: DefaultsTreeNode) -> N
         if isinstance(element, GroupDefault):
             msg = f"Could not find '{element.get_config_path()}'\n"
             if options is not None and len(options) > 0:
-                opt_list = "\n".join(["\t" + x for x in options])
+                opt_list = "\n".join("\t" + x for x in options)
                 msg = f"{msg}\nAvailable options in '{group}':\n" + opt_list
         else:
             msg = dedent(

hydra/_internal/hydra.py

@@ -233,7 +233,7 @@ def get_shell_to_plugin_map(
 
         for shell, plugins in shell_to_plugin.items():
             if len(plugins) > 1:
-                lst = ",".join([type(plugin).__name__ for plugin in plugins])
+                lst = ",".join(type(plugin).__name__ for plugin in plugins)
                 raise ValueError(f"Multiple plugins installed for {shell} : {lst}")
 
         return shell_to_plugin
@@ -251,7 +251,7 @@ def shell_completion(
 
         def find_plugin(cmd: str) -> CompletionPlugin:
             if cmd not in shell_to_plugin:
-                lst = "\n".join(["\t" + x for x in shell_to_plugin.keys()])
+                lst = "\n".join("\t" + x for x in shell_to_plugin.keys())
                 raise ValueError(
                     f"No completion plugin for '{cmd}' found, available : \n{lst}"
                 )
@@ -287,7 +287,7 @@ def list_all_config_groups(self, parent: str = "") -> Sequence[str]:
             if parent == "":
                 group_name = group
             else:
-                group_name = "{}/{}".format(parent, group)
+                group_name = f"{parent}/{group}"
             files = self.config_loader.get_group_options(group_name, ObjectType.CONFIG)
             dirs = self.config_loader.get_group_options(group_name, ObjectType.GROUP)
             if len(files) > 0:
@@ -305,10 +305,10 @@ def format_config_groups(
             options = sorted(self.config_loader.get_group_options(group))
             if compact:
                 items = ", ".join(options)
-                line = "{}: {}".format(group, items)
+                line = f"{group}: {items}"
             else:
-                items = "\n".join(["  " + o for o in options])
-                line = "{}:\n{}".format(group, items)
+                items = "\n".join("  " + o for o in options)
+                line = f"{group}:\n{items}"
             s += line + "\n"
 
         return s
@@ -396,14 +396,14 @@ def _print_plugins(self) -> None:
             if len(plugins) > 0:
                 Hydra._log_header(header=f"{plugin_type.__name__}:", prefix="\t")
                 for plugin in plugins:
-                    log.debug("\t\t{}".format(plugin.__name__))
+                    log.debug(f"\t\t{plugin.__name__}")
                     if plugin.__name__ in all_plugins:
                         all_plugins.remove(plugin.__name__)
 
         if len(all_plugins) > 0:
             Hydra._log_header(header="Generic plugins: ", prefix="\t")
             for plugin_name in all_plugins:
-                log.debug("\t\t{}".format(plugin_name))
+                log.debug(f"\t\t{plugin_name}")
 
     def _print_search_path(
         self,

hydra/_internal/utils.py

@@ -494,7 +494,7 @@ def _get_completion_help() -> str:
             completion_info.append(plugin_cls.help(cmd).format(_get_exec_command()))
         completion_info.append("")
 
-    completion_help = "\n".join([f"    {x}" if x else x for x in completion_info])
+    completion_help = "\n".join(f"    {x}" if x else x for x in completion_info)
     return completion_help
 
 

hydra/core/config_store.py

@@ -133,10 +133,10 @@ def get_type(self, path: str) -> ObjectType:
     def list(self, path: str) -> List[str]:
         d = self._open(path)
         if d is None:
-            raise IOError(f"Path not found {path}")
+            raise OSError(f"Path not found {path}")
 
         if not isinstance(d, dict):
-            raise IOError(f"Path points to a file : {path}")
+            raise OSError(f"Path points to a file : {path}")
 
         return sorted(d.keys())
 

hydra/core/singleton.py

@@ -10,7 +10,7 @@ class Singleton(type):
 
     def __call__(cls, *args: Any, **kwargs: Any) -> Any:
         if cls not in cls._instances:
-            cls._instances[cls] = super(Singleton, cls).__call__(*args, **kwargs)
+            cls._instances[cls] = super().__call__(*args, **kwargs)
         return cls._instances[cls]
 
     def instance(cls: Any, *args: Any, **kwargs: Any) -> Any:

hydra/errors.py

@@ -12,7 +12,7 @@ class CompactHydraException(HydraException):
 
 class OverrideParseException(CompactHydraException):
     def __init__(self, override: str, message: str) -> None:
-        super(OverrideParseException, self).__init__(message)
+        super().__init__(message)
         self.override = override
         self.message = message
 
@@ -36,7 +36,7 @@ def __init__(
         missing_cfg_file: Optional[str] = None,
         options: Optional[Sequence[str]] = None,
     ) -> None:
-        super(MissingConfigException, self).__init__(message)
+        super().__init__(message)
         self.missing_cfg_file = missing_cfg_file
         self.options = options
 

hydra/plugins/completion_plugin.py

@@ -127,15 +127,15 @@ def str_rep(in_key: Any, in_value: Any) -> str:
                 else:
                     key_matches = []
 
-                matches.extend([f"{word}{match}" for match in key_matches])
+                matches.extend(f"{word}{match}" for match in key_matches)
             else:
                 last_dot = word.rfind(".")
                 if last_dot != -1:
                     base_key = word[0:last_dot]
                     partial_key = word[last_dot + 1 :]
                     conf_node = OmegaConf.select(config, base_key)
                     key_matches = CompletionPlugin._get_matches(conf_node, partial_key)
-                    matches.extend([f"{base_key}.{match}" for match in key_matches])
+                    matches.extend(f"{base_key}.{match}" for match in key_matches)
                 else:
                     if isinstance(config, DictConfig):
                         for key, value in config.items_ex(resolve=False):

hydra/test_utils/test_utils.py

@@ -221,7 +221,7 @@ def find_parent_dir_containing(
         cur = os.path.relpath(os.path.join(cur, ".."))
         max_up = max_up - 1
     if max_up == 0:
-        raise IOError(f"Could not find {target} in parents of {os.getcwd()}")
+        raise OSError(f"Could not find {target} in parents of {os.getcwd()}")
     return cur
 
 
@@ -333,7 +333,7 @@ def experiment(cfg):
                 modified_env.update(env_override)
         subprocess.check_call(cmd, env=modified_env)
 
-        with open(output_file, "r") as f:
+        with open(output_file) as f:
             file_str = f.read()
             output = str.splitlines(file_str)
 

noxfile.py

@@ -210,7 +210,7 @@ def list_plugins(directory: str) -> List[Plugin]:
         setup_py = os.path.join(abspath, "setup.py")
         name_and_classifiers: List[str] = subprocess.check_output(
             [sys.executable, setup_py, "--name", "--classifiers"],
-            universal_newlines=True,
+            text=True,
         ).splitlines()
         name, classifiers = name_and_classifiers[0], name_and_classifiers[1:]
 

plugins/hydra_ax_sweeper/hydra_plugins/hydra_ax_sweeper/_core.py

@@ -34,7 +34,7 @@ class TrialBatch:
 
 def encoder_parameters_into_string(parameters: List[Dict[str, Any]]) -> str:
     """Convert a list of params into a string"""
-    mandatory_keys = set(["name", "type", "bounds", "values", "value"])
+    mandatory_keys = {"name", "type", "bounds", "values", "value"}
     parameter_log_string = ""
     for parameter in parameters:
         parameter_log_string += "\n"

plugins/hydra_joblib_launcher/hydra_plugins/hydra_joblib_launcher/_core.py

@@ -37,7 +37,7 @@ def execute_job(
         config, list(overrides)
     )
     with open_dict(sweep_config):
-        sweep_config.hydra.job.id = "{}_{}".format(sweep_config.hydra.job.name, idx)
+        sweep_config.hydra.job.id = f"{sweep_config.hydra.job.name}_{idx}"
         sweep_config.hydra.job.num = idx
     HydraConfig.instance().set_config(sweep_config)
 
@@ -90,11 +90,11 @@ def launch(
 
     log.info(
         "Joblib.Parallel({}) is launching {} jobs".format(
-            ",".join([f"{k}={v}" for k, v in joblib_cfg.items()]),
+            ",".join(f"{k}={v}" for k, v in joblib_cfg.items()),
             len(job_overrides),
         )
     )
-    log.info("Launching jobs, sweep output dir : {}".format(sweep_dir))
+    log.info(f"Launching jobs, sweep output dir : {sweep_dir}")
     for idx, overrides in enumerate(job_overrides):
         log.info("\t#{} : {}".format(idx, " ".join(filter_overrides(overrides))))
 

plugins/hydra_rq_launcher/hydra_plugins/hydra_rq_launcher/_core.py

@@ -95,7 +95,7 @@ def launch(
     log.info(
         f"RQ Launcher is enqueuing {len(job_overrides)} job(s) in queue : {rq_cfg.queue}"
     )
-    log.info("Sweep output dir : {}".format(sweep_dir))
+    log.info(f"Sweep output dir : {sweep_dir}")
     if not sweep_dir.is_absolute():
         log.warn(
             "Using relative sweep dir: Please be aware that dir will be relative to where workers are started from."

setup.py

@@ -21,7 +21,7 @@
     ]
 
 
-with open("README.md", "r") as fh:
+with open("README.md") as fh:
     LONG_DESC = fh.read()
     setup(
         cmdclass={

tests/instantiate/__init__.py

@@ -39,7 +39,7 @@ def partial_equal(obj1: Any, obj2: Any) -> bool:
     if isinstance(obj1, list):
         if len(obj1) != len(obj2):
             return False
-        return all([partial_equal(obj1[i], obj2[i]) for i in range(len(obj1))])
+        return all(partial_equal(o1, o2) for o1, o2 in zip(obj1, obj2))
     if not (isinstance(obj1, partial) and isinstance(obj2, partial)):
         return False
     return all(

tests/test_callbacks.py

@@ -183,7 +183,7 @@ def test_save_job_return_callback(tmpdir: Path, multirun: bool) -> None:
         log_paths = [tmpdir / "my_app.log"]
 
     for p in log_paths:
-        with open(p, "r") as file:
+        with open(p) as file:
             logs = file.readlines()
             assert log_msg in logs
 
@@ -215,7 +215,7 @@ def test_experimental_rerun(
     assert config_file.exists()
     assert log_file.exists()
 
-    with open(log_file, "r") as file:
+    with open(log_file) as file:
         logs = file.read().splitlines()
         assert "[JOB] Running my_app" in logs
 
@@ -232,6 +232,6 @@ def test_experimental_rerun(
     result, err = run_python_script(cmd, allow_warnings=True)
     assert warning_msg in err
 
-    with open(log_file, "r") as file:
+    with open(log_file) as file:
         logs = file.read().splitlines()
         assert "[JOB] Running my_app" in logs

tests/test_completion.py

@@ -269,7 +269,7 @@ def test_shell_integration(
 
         cmd.extend(expected)
         if verbose:
-            print("\nCOMMAND:\n" + " ".join([f"'{x}'" for x in cmd]))
+            print("\nCOMMAND:\n" + " ".join(f"'{x}'" for x in cmd))
 
         subprocess.check_call(cmd)
 

tests/test_hydra.py

@@ -1090,7 +1090,7 @@ def test_hydra_output_dir(
     ) as task:
         assert task.temp_dir is not None
         path = Path(task.temp_dir)
-        files = set([str(x)[len(task.temp_dir) + 1 :] for x in path.iterdir()])
+        files = {str(x)[len(task.temp_dir) + 1 :] for x in path.iterdir()}
         assert files == expected_files
 
 

tests/test_overrides_parser.py

@@ -2,7 +2,6 @@
 import builtins
 import math
 import re
-from builtins import isinstance
 from dataclasses import dataclass
 from typing import Any, Callable, Dict, List, Union
 

tools/configen/configen/utils.py

@@ -47,7 +47,7 @@ def type_str(t: Any) -> str:
 
     args = getattr(t, "__args__", None)
     if args is not None:
-        args = ", ".join([type_str(t) for t in t.__args__])
+        args = ", ".join(type_str(t) for t in t.__args__)
         ret = f"{name}[{args}]"
     else:
         ret = name
@@ -59,10 +59,7 @@ def type_str(t: Any) -> str:
 
 def is_tuple_annotation(type_: Any) -> bool:
     origin = getattr(type_, "__origin__", None)
-    if sys.version_info < (3, 7, 0):
-        return origin is Tuple or type_ is Tuple  # pragma: no cover
-    else:
-        return origin is tuple  # pragma: no cover
+    return origin is tuple
 
 
 def convert_imports(imports: Set[Any], string_imports: Iterable[str]) -> List[str]:

tools/configen/tests/test_modules/__init__.py

@@ -180,7 +180,7 @@ def __eq__(self, other):
         )
 
 
-class PeskySentinel(object):
+class PeskySentinel:
     def __repr__(self):
         return "<I am a pesky sentinel>"