Skip to content
This repository has been archived by the owner on Oct 31, 2023. It is now read-only.

Commit

Permalink
Merge pull request #4662 from golemfactory/fix_subtasks_count
Browse files Browse the repository at this point in the history 
fix subtasks_count calculation
  • Loading branch information
@etam
etam authored Aug 30, 2019
2 parents 66ef664 + 3d6e8c8 commit 7a50aff
Show file tree
Hide file tree
Showing 5 changed files with 180 additions and 130 deletions.
2 changes: 2 additions & 0 deletions apps/core/task/coretask.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -551,6 +551,8 @@ def build_minimal_definition(cls, task_type: CoreTaskTypeInfo, dictionary) \
definition.compute_on = dictionary.get('compute_on', 'cpu')
definition.resources = set(dictionary['resources'])
definition.subtasks_count = int(dictionary['subtasks_count'])
if 'optimize_total' in dictionary:
definition.optimize_total = bool(dictionary['optimize_total'])
return definition

@classmethod
Expand Down
17 changes: 7 additions & 10 deletions apps/rendering/task/framerenderingtask.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -510,16 +510,6 @@ def __init__(self, owner, task_definition, dir_manager):
task_definition,
dir_manager)

@classmethod
def _calculate_total(cls, task_definition):
task_definition.subtasks_count = _calculate_subtasks_count(
subtasks_count=task_definition.subtasks_count,
optimize_total=task_definition.optimize_total,
use_frames=task_definition.options.use_frames,
frames=task_definition.options.frames,
defaults=cls.DEFAULTS(),
)

@classmethod
def build_dictionary(cls, definition):
parent = super(FrameRenderingTaskBuilder, cls)
Expand All @@ -540,6 +530,13 @@ def build_minimal_definition(cls, task_type, dictionary):
definition.options.frames_string = frames_string
definition.options.frames = frames
definition.options.use_frames = use_frames
definition.subtasks_count = _calculate_subtasks_count(
subtasks_count=int(dictionary['subtasks_count']),
optimize_total=definition.optimize_total,
use_frames=definition.options.use_frames,
frames=definition.options.frames,
defaults=cls.DEFAULTS(),
)

return definition

Expand Down
14 changes: 5 additions & 9 deletions apps/rendering/task/renderingtask.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -287,14 +287,6 @@ class RenderingTaskBuilder(CoreTaskBuilder):
TASK_CLASS = RenderingTask
DEFAULTS = RendererDefaults

@classmethod
def _calculate_total(cls, task_definition: 'RenderingTaskDefinition'):
task_definition.subtasks_count = _calculate_subtasks_count(
subtasks_count=task_definition.subtasks_count,
optimize_total=task_definition.optimize_total,
defaults=cls.DEFAULTS(),
)

@staticmethod
def _scene_file(type, resources):
extensions = type.output_file_ext
Expand Down Expand Up @@ -323,7 +315,11 @@ def build_minimal_definition(cls, task_type, dictionary) \
resources = dictionary['resources']

definition = parent.build_minimal_definition(task_type, dictionary)
cls._calculate_total(definition)
definition.subtasks_count = _calculate_subtasks_count(
subtasks_count=int(dictionary['subtasks_count']),
optimize_total=definition.optimize_total,
defaults=cls.DEFAULTS(),
)

if 'main_scene_file' in dictionary:
main_scene_file = dictionary['main_scene_file']
Expand Down
237 changes: 166 additions & 71 deletions tests/apps/rendering/task/test_framerenderingtask.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,10 @@
from pathlib import Path

from golem_messages.factories.datastructures import p2p as dt_p2p_factory
from apps.rendering.resources.imgrepr import load_img, EXRImgRepr, OpenCVImgRepr

from apps.core.task.coretask import CoreTaskTypeInfo
from apps.core.task.coretaskstate import Options
from apps.rendering.resources.imgrepr import load_img, EXRImgRepr, OpenCVImgRepr
from apps.rendering.task.framerenderingtask import get_frame_name, \
FrameRenderingTask, FrameRenderingTaskBuilder, FrameRendererOptions, logger
from apps.rendering.task.renderingtaskstate import RenderingTaskDefinition
Expand Down Expand Up @@ -302,77 +304,170 @@ def test_get_subtask_for_multiple_subtask_per_frame(self):
assert len(states) == 2


class TestFrameRenderingTaskBuilder(unittest.TestCase):
class TestBuildDefinition(unittest.TestCase):
def setUp(self):
self.tti = CoreTaskTypeInfo("TESTTASK", RenderingTaskDefinition,
Options,
FrameRenderingTaskBuilder)
self.tti.output_file_ext = 'txt'

@staticmethod
def _create_definition(*,
optimize_total=False,
subtasks_count=12,
use_frames=True):
definition = RenderingTaskDefinition()
definition.optimize_total = optimize_total
definition.subtasks_count = subtasks_count
definition.options = FrameRendererOptions()
definition.options.use_frames = use_frames
definition.options.frames = list(range(1, 7))
return definition

def _assert_calculated_subtasks_count_is_equal(
self, definition, expected_count):
FrameRenderingTaskBuilder._calculate_total(definition)
assert definition.subtasks_count == expected_count

def test_more_subtasks_than_frames(self):
# use frames count
self._assert_calculated_subtasks_count_is_equal(
self._create_definition(optimize_total=True), 6)

def test_optimize_to_default(self):
self._assert_calculated_subtasks_count_is_equal(
self._create_definition(optimize_total=True, use_frames=False), 20)

def test_calculate_total(self):
self._assert_calculated_subtasks_count_is_equal(
self._create_definition(use_frames=False), 12)

def test_subtasks_count_unknown(self):
# use default
self._assert_calculated_subtasks_count_is_equal(
self._create_definition(subtasks_count=None, use_frames=False), 20)

def test_subtasks_count_below_min(self):
# use default
self._assert_calculated_subtasks_count_is_equal(
self._create_definition(subtasks_count=0, use_frames=False), 20)

def test_subtasks_count_min(self):
self._assert_calculated_subtasks_count_is_equal(
self._create_definition(subtasks_count=1, use_frames=False), 1)

def test_subtasks_count_over_max(self):
# use default
self._assert_calculated_subtasks_count_is_equal(
self._create_definition(subtasks_count=51, use_frames=False), 20)

def test_subtasks_count_max(self):
self._assert_calculated_subtasks_count_is_equal(
self._create_definition(subtasks_count=50, use_frames=False), 50)

def test_subtasks_count_frames(self):
for i in [
(None, 6),
(0, 6),
(1, 1),
(2, 2),
(3, 3),
(4, 3),
(6, 6),
(12, 12),
(13, 12),
(17, 12),
(18, 18)
]:
self._assert_calculated_subtasks_count_is_equal(
self._create_definition(subtasks_count=i[0]), i[1])
def _make_dict(
*,
frame_count: int = 1,
subtasks_count: int = 1,
optimize_total: bool = False,
):
assert frame_count > 0
if frame_count == 1:
frames = "1"
else:
frames = f"1-{frame_count}"

return {
"bid": 0,
"name": "foo",
"optimize_total": optimize_total,
"options": {
"format": "PNG",
"frame_count": frame_count,
"frames": frames,
"output_path": "/tmp/foo",
"resolution": [1, 1],
},
"resources": ["foo.txt"],
"subtask_timeout": "0:00:01",
"subtasks_count": subtasks_count,
"timeout": "0:00:01",
}

def test_subtasks_count(self):
# sort tests by (optimize_total, frame_count, subtasks_count)
tests = [
{
"make_dict_kwargs": {
"subtasks_count": 0,
},
"expected_subtasks_count": 20,
},
{
"make_dict_kwargs": {},
"expected_subtasks_count": 1,
},
{
"make_dict_kwargs": {
"subtasks_count": 50,
},
"expected_subtasks_count": 50,
},
{
"make_dict_kwargs": {
"subtasks_count": 51,
},
"expected_subtasks_count": 20,
},
{
"make_dict_kwargs": {
"frame_count": 6,
"subtasks_count": 0,
},
"expected_subtasks_count": 6,
},
{
"make_dict_kwargs": {
"frame_count": 6,
"subtasks_count": 1,
},
"expected_subtasks_count": 1,
},
{
"make_dict_kwargs": {
"frame_count": 6,
"subtasks_count": 2,
},
"expected_subtasks_count": 2,
},
{
"make_dict_kwargs": {
"frame_count": 6,
"subtasks_count": 3,
},
"expected_subtasks_count": 3,
},
{
"make_dict_kwargs": {
"frame_count": 6,
"subtasks_count": 4,
},
"expected_subtasks_count": 3,
},
{
"make_dict_kwargs": {
"frame_count": 6,
"subtasks_count": 6,
},
"expected_subtasks_count": 6,
},
{
"make_dict_kwargs": {
"frame_count": 6,
"subtasks_count": 12,
},
"expected_subtasks_count": 12,
},
{
"make_dict_kwargs": {
"frame_count": 6,
"subtasks_count": 13,
},
"expected_subtasks_count": 12,
},
{
"make_dict_kwargs": {
"frame_count": 6,
"subtasks_count": 17,
},
"expected_subtasks_count": 12,
},
{
"make_dict_kwargs": {
"frame_count": 6,
"subtasks_count": 18,
},
"expected_subtasks_count": 18,
},
{
"make_dict_kwargs": {
"optimize_total": True,
},
"expected_subtasks_count": 20,
},
{
"make_dict_kwargs": {
"frame_count": 6,
"subtasks_count": 12,
"optimize_total": True,
},
"expected_subtasks_count": 6,
},
{
"make_dict_kwargs": {
"frame_count": 6,
"subtasks_count": 12,
"optimize_total": True,
},
"expected_subtasks_count": 6,
},
]
failures = []
for test in tests:
task_dict = self._make_dict(**test["make_dict_kwargs"])
definition = FrameRenderingTaskBuilder.build_definition(
self.tti, task_dict)
if definition.subtasks_count != test["expected_subtasks_count"]:
test["actual_subtasks_count"] = definition.subtasks_count
failures.append(test)
assert failures == []


class TestFramesConversion(unittest.TestCase):
Expand Down
40 changes: 0 additions & 40 deletions tests/apps/rendering/task/test_renderingtask.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -298,46 +298,6 @@ def test_update_task_preview_ioerror(self):
assert logger.exception.called


class TestRenderingTaskBuilder(TestCase):
@staticmethod
def _create_definition(*,
optimize_total=False,
subtasks_count=0):
definition = RenderingTaskDefinition()
definition.optimize_total = optimize_total
definition.subtasks_count = subtasks_count
return definition

def _assert_calculated_subtasks_count_is_equal(
self, definition, expected_count):
RenderingTaskBuilder._calculate_total(definition)
assert definition.subtasks_count == expected_count

def test_optimize_to_default(self):
self._assert_calculated_subtasks_count_is_equal(
self._create_definition(optimize_total=True), 20)

def test_calculate_total(self):
self._assert_calculated_subtasks_count_is_equal(
self._create_definition(subtasks_count=18), 18)

def test_subtasks_count_below_min(self):
self._assert_calculated_subtasks_count_is_equal(
self._create_definition(subtasks_count=0), 20)

def test_subtasks_count_min(self):
self._assert_calculated_subtasks_count_is_equal(
self._create_definition(subtasks_count=1), 1)

def test_subtasks_count_over_max(self):
self._assert_calculated_subtasks_count_is_equal(
self._create_definition(subtasks_count=51), 20)

def test_subtasks_count_max(self):
self._assert_calculated_subtasks_count_is_equal(
self._create_definition(subtasks_count=50), 50)


class TestBuildDefinition(LogTestCase):
def setUp(self):
super().setUp()
Expand Down

0 comments on commit 7a50aff

Please sign in to comment.