Centralize logging and remove old logging dependencies

Signed-off-by: m09 <142691+m09@users.noreply.github.com>

Author: m09

formatml/pipelines/codrep/index.py

@@ -4,7 +4,6 @@
 from typing import List
 
 from asdf import open as asdf_open
-from coloredlogs import install as coloredlogs_install
 
 from formatml.data.fields.binary_label_field import BinaryLabelsField
 from formatml.data.fields.graph_fields.internal_type_field import InternalTypeField
@@ -15,6 +14,7 @@
 from formatml.data.types.codrep_label import CodRepLabel
 from formatml.parsing.parser import Nodes
 from formatml.pipelines.pipeline import register_step
+from formatml.utils.helpers import setup_logging
 
 
 def add_arguments_to_parser(parser: ArgumentParser) -> None:
@@ -52,7 +52,7 @@ def index(
     log_level: str,
 ) -> None:
     """Index UASTs with respect to some fields."""
-    coloredlogs_install(level=log_level, fmt="%(name)27s %(levelname)8s %(message)s")
+    setup_logging(log_level)
     logger = getLogger(__name__)
 
     input_dir_path = Path(input_dir).expanduser().resolve()

formatml/pipelines/codrep/parse.py

@@ -4,12 +4,12 @@
 from time import time
 
 from asdf import AsdfFile
-from coloredlogs import install as coloredlogs_install
 
 from formatml.data.types.codrep_label import CodRepLabel
 from formatml.parsing.java_parser import JavaParser
 from formatml.parsing.parser import ParsingException
 from formatml.pipelines.pipeline import register_step
+from formatml.utils.helpers import setup_logging
 
 
 def add_arguments_to_parser(parser: ArgumentParser) -> None:
@@ -29,7 +29,7 @@ def add_arguments_to_parser(parser: ArgumentParser) -> None:
 )
 def codrep_train(*, input_dir: str, output_dir: str, log_level: str) -> None:
     """Parse a CodRep 2019 dataset into UASTs."""
-    coloredlogs_install(level=log_level, fmt="%(name)27s %(levelname)8s %(message)s")
+    setup_logging(log_level)
     logger = getLogger(__name__)
     input_dir_path = Path(input_dir).expanduser().resolve()
     output_dir_path = Path(output_dir).expanduser().resolve()

formatml/pipelines/codrep/tensorize.py

@@ -8,12 +8,12 @@
 from pickle import dump as pickle_dump, load as pickle_load
 
 from asdf import open as asdf_open
-from coloredlogs import install as coloredlogs_install
 
 from formatml.data.instance import Instance
 from formatml.data.types.codrep_label import CodRepLabel
 from formatml.parsing.parser import Nodes
 from formatml.pipelines.pipeline import register_step
+from formatml.utils.helpers import setup_logging
 
 
 def add_arguments_to_parser(parser: ArgumentParser) -> None:
@@ -53,7 +53,7 @@ def tensorize(
     log_level: str,
 ) -> None:
     """Tensorize the UASTs."""
-    coloredlogs_install(level=log_level, fmt="%(name)27s %(levelname)8s %(message)s")
+    setup_logging(log_level)
     logger = getLogger(__name__)
 
     uasts_dir_path = Path(uasts_dir).expanduser().resolve()

formatml/pipelines/codrep/train.py

@@ -5,7 +5,6 @@
 from pickle import load as pickle_load
 from typing import List, Optional
 
-from coloredlogs import install as coloredlogs_install
 from torch.nn import Linear
 from torch.optim import Adam
 from torch.optim.lr_scheduler import StepLR
@@ -16,6 +15,7 @@
 from formatml.modules.misc.graph_embedding import GraphEmbedding
 from formatml.pipelines.pipeline import register_step
 from formatml.training.trainer import Trainer
+from formatml.utils.helpers import setup_logging
 
 
 def add_arguments_to_parser(parser: ArgumentParser) -> None:
@@ -128,7 +128,7 @@ def train(
     log_level: str,
 ) -> None:
     """Run the training."""
-    coloredlogs_install(level=log_level, fmt="%(name)27s %(levelname)8s %(message)s")
+    setup_logging(log_level)
     logger = getLogger(__name__)
 
     tensors_dir_path = Path(tensors_dir).expanduser().resolve()

formatml/utils/helpers.py

@@ -1,14 +1,31 @@
 from datetime import datetime
 from itertools import chain
+from logging import Filter, getLogger, LogRecord
 from pathlib import Path
 from sys import exit, stderr
 from typing import Any, Iterable, List, Optional, Tuple, Type, TypeVar, Union
 
+from coloredlogs import install as coloredlogs_install
 from dulwich.errors import NotGitRepository
 from dulwich.porcelain import status
 from dulwich.repo import Repo
 
 
+class ShortNameFilter(Filter):
+    def filter(self, record: LogRecord) -> int:
+        record.shortname = record.name.split(".")[-1]  # type: ignore
+        return 1
+
+
+def setup_logging(log_level: str) -> None:
+    coloredlogs_install(
+        level=log_level,
+        fmt="%(asctime)s %(shortname)10s %(message)s",
+        datefmt="%H:%M:%S",
+    )
+    getLogger().handlers[0].addFilter(ShortNameFilter())
+
+
 def get_sha_and_dirtiness(prompt_on_dirty: bool = True) -> Optional[Tuple[str, bool]]:
     try:
         git_status = status()

requirements.txt

@@ -4,6 +4,4 @@ dgl == 0.3
 bblfsh == 2.12.7
 asdf == 2.3.3
 dulwich == 0.19.11
-tf-nightly == 1.15.0.dev20190710
-future == 0.17.1
-Pillow == 6.1.0
+tensorboard == 1.14.0

setup.py

@@ -31,16 +31,7 @@
     url="https://github.com/src-d/formatml",
     packages=find_packages(exclude=["tests"]),
     entry_points={"console_scripts": ["formatml=formatml.__main__:main"]},
-    install_requires=[
-        "coloredlogs",
-        "bblfsh <3.0",
-        "asdf",
-        "dulwich",
-        "tf-nightly",
-        # Below are extra deps for TF
-        "Pillow",
-        "future",
-    ],
+    install_requires=["coloredlogs", "bblfsh <3.0", "asdf", "dulwich", "tensorboard"],
     include_package_data=True,
     license="Apache-2.0",
     classifiers=[