cpdbench.control.ExecutionController

 1from abc import ABC, abstractmethod
 2
 3from cpdbench.control.CPDResult import CPDResult
 4from cpdbench.exception.UserParameterDoesNotExistException import UserParameterDoesNotExistException
 5from cpdbench.exception.ValidationException import ValidationException
 6from cpdbench.task.Task import TaskType
 7from cpdbench.task.TaskFactory import TaskFactory
 8from cpdbench.utils import Utils
 9
10
11class ExecutionController(ABC):
12    """Abstract base class for testbench run configurations.
13    Each subclass has to give an execute_run() implementation with the actual run logic.
14    """
15
16    def __init__(self, logger):
17        self._logger = logger
18
19    @abstractmethod
20    def execute_run(self, methods: dict) -> CPDResult:
21        """Executes the run implemented by this class.
22        :param methods: dictionary with all given input functions, grouped by function type.
23        :return: A result object which can be handed to the user
24        """
25        pass
26
27    def _create_tasks(self, methods: dict) -> dict:
28        """Creates task objects from the entered function handles
29        :param methods: A dictionary containing the function handles, grouped by function type
30        :return: A dictionary of task objects, which were converted from the given handles
31        """
32        task_objects = {
33            "datasets": [],
34            "algorithms": [],
35            "metrics": []
36        }
37        task_factory = TaskFactory()
38        for dataset_function in methods["datasets"]:
39            self._logger.debug(f"Creating and validating dataset task "
40                               f"for {Utils.get_name_of_function(dataset_function)}")
41            try:
42                tasks = task_factory.create_tasks_with_parameters(dataset_function, TaskType.DATASET_FETCH)
43            except UserParameterDoesNotExistException as e:
44                self._logger.exception(e)
45                self._logger.debug(f"Creating {Utils.get_name_of_function(dataset_function)} has failed")
46                continue
47            try:
48                for task in tasks:
49                    task.validate_task()
50            except ValidationException as e:
51                self._logger.exception(e)
52            else:
53                self._logger.debug(f'Validating and creating {Utils.get_name_of_function(dataset_function)} has succeeded')
54                task_objects["datasets"] += tasks
55
56        for algorithm_function in methods["algorithms"]:
57            self._logger.debug(f"Creating and validating algorithm task "
58                               f"for {Utils.get_name_of_function(algorithm_function)}")
59            try:
60                tasks = task_factory.create_tasks_with_parameters(algorithm_function, TaskType.ALGORITHM_EXECUTION)
61            except UserParameterDoesNotExistException as e:
62                self._logger.exception(e)
63                self._logger.debug(f"Creating {Utils.get_name_of_function(algorithm_function)} has failed")
64                continue
65            try:
66                for task in tasks:
67                    task.validate_task()
68            except ValidationException as e:
69                self._logger.exception(e)
70            else:
71                self._logger.debug(f'Validating {Utils.get_name_of_function(algorithm_function)} has succeeded')
72                task_objects["algorithms"] += tasks
73
74        for metric_function in methods["metrics"]:
75            self._logger.debug(f"Creating and validating metric task "
76                               f"for {Utils.get_name_of_function(metric_function)}")
77            try:
78                tasks = task_factory.create_tasks_with_parameters(metric_function, TaskType.METRIC_EXECUTION)
79            except UserParameterDoesNotExistException as e:
80                self._logger.exception(e)
81                self._logger.debug(f"Creating {Utils.get_name_of_function(metric_function)} has failed")
82                continue
83            try:
84                for task in tasks:
85                    task.validate_task()
86            except ValidationException as e:
87                self._logger.exception(e)
88            else:
89                self._logger.debug(f'Validating {Utils.get_name_of_function(metric_function)} has succeeded')
90                task_objects["metrics"] += tasks
91        return task_objects