From fb3d975075402e9ea26be2ad1a6f77d46eb7b4ba Mon Sep 17 00:00:00 2001 From: Martha Cryan Date: Fri, 15 Jan 2021 17:02:39 -0600 Subject: [PATCH 01/16] Add initial support for usage properties --- .../src/PipelineEditorWidget.tsx | 7 + packages/pipeline-editor/src/properties.json | 183 ++++++++++++++++++ packages/pipeline-editor/style/index.css | 50 +++++ 3 files changed, 240 insertions(+) diff --git a/packages/pipeline-editor/src/PipelineEditorWidget.tsx b/packages/pipeline-editor/src/PipelineEditorWidget.tsx index 35663ef8c..6e44d2bf6 100644 --- a/packages/pipeline-editor/src/PipelineEditorWidget.tsx +++ b/packages/pipeline-editor/src/PipelineEditorWidget.tsx @@ -492,6 +492,9 @@ export class PipelineEditor extends React.Component< app_data.dependencies; node_props.parameterDef.current_parameters.include_subdirectories = app_data.include_subdirectories; + node_props.parameterDef.current_parameters.cpu = app_data.cpu; + node_props.parameterDef.current_parameters.memory = app_data.memory; + node_props.parameterDef.current_parameters.gpu = app_data.gpu; node_props.parameterDef.titleDefinition = { title: this.canvasController.getNode(source.id).label, editable: true @@ -540,6 +543,10 @@ export class PipelineEditor extends React.Component< app_data.env_vars = propertySet.env_vars; app_data.dependencies = propertySet.dependencies; app_data.include_subdirectories = propertySet.include_subdirectories; + app_data.cpu = propertySet.cpu; + app_data.memory_dbl = propertySet.memory_dbl; + app_data.memory_unit = propertySet.memory_unit; + app_data.gpu = propertySet.gpu; this.validateAllNodes(); this.updateModel(); } diff --git a/packages/pipeline-editor/src/properties.json b/packages/pipeline-editor/src/properties.json index 776f7edb8..470feddc0 100644 --- a/packages/pipeline-editor/src/properties.json +++ b/packages/pipeline-editor/src/properties.json @@ -18,6 +18,39 @@ "enum": [], "required": true }, + { + "id": "cpu", + "type": "double", + "required": "false" + }, + { + "id": "memory_dbl", + "type": "double", + "required": "false" + }, + { + "id": "memory_unit", + "enum": [ + "E", + "Ei", + "P", + "Pi", + "T", + "Ti", + "G", + "Gi", + "M", + "Mi", + "K", + "Ki" + ], + "required": "false" + }, + { + "id": "gpu", + "type": "double", + "required": "false" + }, { "id": "dependencies", "type": "array[string]", @@ -59,6 +92,40 @@ "default": "Container image used as execution environment" } }, + { + "parameter_ref": "cpu", + "label": { + "default": "CPU" + }, + "description": { + "default": "For CPU-intensive workloads, you can choose more than 1 CPU (e.g. 1.5)." + } + }, + { + "parameter_ref": "memory_dbl", + "label": { + "default": "Memory" + }, + "description": { + "default": "The total amount of RAM specified " + } + }, + { + "parameter_ref": "memory_unit", + "label": { + "default": " " + }, + "control": "oneofselect" + }, + { + "parameter_ref": "gpu", + "label": { + "default": "GPU" + }, + "description": { + "default": "" + } + }, { "parameter_ref": "dependencies", "label": { @@ -150,6 +217,30 @@ "type": "controls", "parameter_refs": ["runtime_image"] }, + { + "id": "nodeUsageControls", + "type": "panels", + "label": { + "default": "Usage" + }, + "group_info": [ + { + "id": "nodeCPUControls", + "type": "controls", + "parameter_refs": ["cpu"] + }, + { + "id": "nodeMemoryControls", + "type": "controls", + "parameter_refs": ["memory_dbl", "memory_unit"] + }, + { + "id": "nodeGPUControls", + "type": "controls", + "parameter_refs": ["gpu"] + } + ] + }, { "id": "nodeDependenciesControls", "type": "panels", @@ -170,5 +261,97 @@ } ] }, + "conditions": [ + { + "validation": { + "fail_message": { + "type": "error", + "focus_parameter_ref": "memory_dbl", + "message": { + "default": "Memory must be greater than 0." + } + }, + "evaluate": { + "condition": { + "parameter_ref": "memory_dbl", + "op": "greaterThan", + "value": 0 + } + } + } + }, + { + "validation": { + "fail_message": { + "type": "error", + "focus_parameter_ref": "cpu", + "message": { + "default": "CPU must be greater than 0." + } + }, + "evaluate": { + "condition": { + "parameter_ref": "cpu", + "op": "greaterThan", + "value": 0 + } + } + } + }, + { + "validation": { + "fail_message": { + "type": "error", + "focus_parameter_ref": "gpu", + "message": { + "default": "GPU must be greater than 0." + } + }, + "evaluate": { + "condition": { + "parameter_ref": "gpu", + "op": "greaterThan", + "value": 0 + } + } + } + }, + { + "validation": { + "fail_message": { + "type": "error", + "focus_parameter_ref": "cpu", + "message": { + "default": "CPU must be less than 99." + } + }, + "evaluate": { + "condition": { + "parameter_ref": "cpu", + "op": "lessThan", + "value": 99 + } + } + } + }, + { + "validation": { + "fail_message": { + "type": "error", + "focus_parameter_ref": "gpu", + "message": { + "default": "GPU must be less than 99." + } + }, + "evaluate": { + "condition": { + "parameter_ref": "gpu", + "op": "lessThan", + "value": 99 + } + } + } + } + ], "resources": {} } diff --git a/packages/pipeline-editor/style/index.css b/packages/pipeline-editor/style/index.css index 009112ea0..ee9229e79 100644 --- a/packages/pipeline-editor/style/index.css +++ b/packages/pipeline-editor/style/index.css @@ -56,6 +56,7 @@ margin-bottom: 14px; margin-top: 14px; border-bottom: 1px solid #cfd1d4; + min-width: 50%; } .properties-control-item { padding: 5px 0 5px 0; @@ -234,6 +235,20 @@ td { align-items: center; width: 100%; } +.properties-control-panel[data-id='properties-nodeMemoryControls'] { + display: flex; +} +.properties-control-panel[data-id='properties-nodeUsageControls'] { + display: flex; + flex-wrap: wrap; +} +.properties-control-panel[data-id='properties-nodeUsageControls'] + .properties-control-panel { + margin-bottom: 0px; +} +.properties-control-panel[data-id='properties-nodeGPUControls'] { + border-bottom: none; +} .properties-control-panel[data-id='properties-nodeFileControl'] .properties-control-item { width: calc(100% - 67px); @@ -250,6 +265,41 @@ td { .properties-control-panel > .properties-control-panel { padding: 0; } +.properties-control-item[data-id='properties-ci-memory_unit'] { + margin-top: 17px; +} +.properties-numberfield[data-id='properties-memory_dbl'] { + padding: 5px 0; +} +.properties-numberfield[data-id='properties-memory_dbl'] + .bx--number + input[type='number'] { + border-radius: 3px 0 0 3px; +} +#properties-memory_unit-dropdown .bx--list-box__field { + border-radius: 0 3px 3px 0; +} +#properties-memory_unit { + max-width: 60px; +} +.properties-numberfield .bx--number input[type='number'] { + border-radius: 3px; + border-bottom: 0; + box-shadow: inset 0 0 0 var(--jp-border-width) var(--jp-input-border-color); +} +.properties-numberfield .bx--number input[type='number']:focus { + box-shadow: inset 0 0 0 var(--jp-border-width) + var(--jp-input-active-box-shadow-color), + inset 0 0 0 3px var(--jp-input-active-box-shadow-color); + outline: none; +} + +.properties-numberfield .bx--number__controls .bx--number__control-btn:focus { + box-shadow: inset 0 0 0 var(--jp-border-width) + var(--jp-input-active-box-shadow-color), + inset 0 0 0 3px var(--jp-input-active-box-shadow-color); + outline: none; +} .elyra-Tooltip { top: 84px; From de8861eeaf9462423cf34082ec17b8a7a8b83cf2 Mon Sep 17 00:00:00 2001 From: Martha Cryan Date: Fri, 15 Jan 2021 17:20:52 -0600 Subject: [PATCH 02/16] Fix small css issues --- packages/pipeline-editor/style/index.css | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/pipeline-editor/style/index.css b/packages/pipeline-editor/style/index.css index ee9229e79..bd6fb9cbb 100644 --- a/packages/pipeline-editor/style/index.css +++ b/packages/pipeline-editor/style/index.css @@ -246,7 +246,8 @@ td { .properties-control-panel { margin-bottom: 0px; } -.properties-control-panel[data-id='properties-nodeGPUControls'] { +.properties-control-panel[data-id='properties-nodeUsageControls'] + .properties-control-panel[data-id='properties-nodeGPUControls'] { border-bottom: none; } .properties-control-panel[data-id='properties-nodeFileControl'] @@ -267,6 +268,8 @@ td { } .properties-control-item[data-id='properties-ci-memory_unit'] { margin-top: 17px; + margin-bottom: 7px; + max-width: 60px; } .properties-numberfield[data-id='properties-memory_dbl'] { padding: 5px 0; @@ -279,9 +282,6 @@ td { #properties-memory_unit-dropdown .bx--list-box__field { border-radius: 0 3px 3px 0; } -#properties-memory_unit { - max-width: 60px; -} .properties-numberfield .bx--number input[type='number'] { border-radius: 3px; border-bottom: 0; From cf06ba4cad6efcad5d915591eae6ec1a13284fd6 Mon Sep 17 00:00:00 2001 From: Martha Cryan Date: Tue, 19 Jan 2021 11:51:19 -0600 Subject: [PATCH 03/16] Fix test dropdown issue --- tests/integration/pipeline.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/integration/pipeline.ts b/tests/integration/pipeline.ts index 4311752d0..97025238c 100644 --- a/tests/integration/pipeline.ts +++ b/tests/integration/pipeline.ts @@ -200,7 +200,9 @@ describe('Pipeline Editor tests', () => { cy.get('.react-contextmenu-item:nth-child(9)') .contains('Properties') .click(); - cy.get('div.properties-dropdown').click(); + cy.get( + 'div.properties-dropdown[data-id="properties-runtime_image"]' + ).click(); // selects the first item of the runtimes dropdown cy.get('#downshift-0-item-0').click(); From 954ae293b47c4435ba518104433a9c9c50c9c2a1 Mon Sep 17 00:00:00 2001 From: Martha Cryan Date: Tue, 19 Jan 2021 12:59:18 -0600 Subject: [PATCH 04/16] Remove units, condense fields --- packages/pipeline-editor/src/properties.json | 160 ++++++++++--------- packages/pipeline-editor/style/index.css | 34 +--- 2 files changed, 91 insertions(+), 103 deletions(-) diff --git a/packages/pipeline-editor/src/properties.json b/packages/pipeline-editor/src/properties.json index 470feddc0..df151a185 100644 --- a/packages/pipeline-editor/src/properties.json +++ b/packages/pipeline-editor/src/properties.json @@ -24,28 +24,10 @@ "required": "false" }, { - "id": "memory_dbl", + "id": "memory", "type": "double", "required": "false" }, - { - "id": "memory_unit", - "enum": [ - "E", - "Ei", - "P", - "Pi", - "T", - "Ti", - "G", - "Gi", - "M", - "Mi", - "K", - "Ki" - ], - "required": "false" - }, { "id": "gpu", "type": "double", @@ -102,28 +84,21 @@ } }, { - "parameter_ref": "memory_dbl", + "parameter_ref": "memory", "label": { - "default": "Memory" + "default": "RAM(GB)" }, "description": { - "default": "The total amount of RAM specified " + "default": "The total amount of RAM specified." } }, - { - "parameter_ref": "memory_unit", - "label": { - "default": " " - }, - "control": "oneofselect" - }, { "parameter_ref": "gpu", "label": { "default": "GPU" }, "description": { - "default": "" + "default": "For GPU-intensive workloads, you can choose more than 1 GPU (e.g. 1.5)." } }, { @@ -219,27 +194,8 @@ }, { "id": "nodeUsageControls", - "type": "panels", - "label": { - "default": "Usage" - }, - "group_info": [ - { - "id": "nodeCPUControls", - "type": "controls", - "parameter_refs": ["cpu"] - }, - { - "id": "nodeMemoryControls", - "type": "controls", - "parameter_refs": ["memory_dbl", "memory_unit"] - }, - { - "id": "nodeGPUControls", - "type": "controls", - "parameter_refs": ["gpu"] - } - ] + "type": "controls", + "parameter_refs": ["cpu", "gpu", "memory"] }, { "id": "nodeDependenciesControls", @@ -266,17 +222,27 @@ "validation": { "fail_message": { "type": "error", - "focus_parameter_ref": "memory_dbl", + "focus_parameter_ref": "memory", "message": { "default": "Memory must be greater than 0." } }, "evaluate": { - "condition": { - "parameter_ref": "memory_dbl", - "op": "greaterThan", - "value": 0 - } + "or": [ + { + "condition": { + "parameter_ref": "memory", + "op": "greaterThan", + "value": 0 + } + }, + { + "condition": { + "parameter_ref": "memory", + "op": "isEmpty" + } + } + ] } } }, @@ -290,11 +256,21 @@ } }, "evaluate": { - "condition": { - "parameter_ref": "cpu", - "op": "greaterThan", - "value": 0 - } + "or": [ + { + "condition": { + "parameter_ref": "cpu", + "op": "greaterThan", + "value": 0 + } + }, + { + "condition": { + "parameter_ref": "cpu", + "op": "isEmpty" + } + } + ] } } }, @@ -308,11 +284,21 @@ } }, "evaluate": { - "condition": { - "parameter_ref": "gpu", - "op": "greaterThan", - "value": 0 - } + "or": [ + { + "condition": { + "parameter_ref": "gpu", + "op": "greaterThan", + "value": 0 + } + }, + { + "condition": { + "parameter_ref": "gpu", + "op": "isEmpty" + } + } + ] } } }, @@ -326,11 +312,21 @@ } }, "evaluate": { - "condition": { - "parameter_ref": "cpu", - "op": "lessThan", - "value": 99 - } + "or": [ + { + "condition": { + "parameter_ref": "cpu", + "op": "lessThan", + "value": 99 + } + }, + { + "condition": { + "parameter_ref": "cpu", + "op": "isEmpty" + } + } + ] } } }, @@ -344,11 +340,21 @@ } }, "evaluate": { - "condition": { - "parameter_ref": "gpu", - "op": "lessThan", - "value": 99 - } + "or": [ + { + "condition": { + "parameter_ref": "gpu", + "op": "lessThan", + "value": 99 + } + }, + { + "condition": { + "parameter_ref": "gpu", + "op": "isEmpty" + } + } + ] } } } diff --git a/packages/pipeline-editor/style/index.css b/packages/pipeline-editor/style/index.css index bd6fb9cbb..3a6988519 100644 --- a/packages/pipeline-editor/style/index.css +++ b/packages/pipeline-editor/style/index.css @@ -56,7 +56,6 @@ margin-bottom: 14px; margin-top: 14px; border-bottom: 1px solid #cfd1d4; - min-width: 50%; } .properties-control-item { padding: 5px 0 5px 0; @@ -235,19 +234,16 @@ td { align-items: center; width: 100%; } -.properties-control-panel[data-id='properties-nodeMemoryControls'] { - display: flex; -} .properties-control-panel[data-id='properties-nodeUsageControls'] { display: flex; flex-wrap: wrap; } -.properties-control-panel[data-id='properties-nodeUsageControls'] - .properties-control-panel { - margin-bottom: 0px; -} -.properties-control-panel[data-id='properties-nodeUsageControls'] - .properties-control-panel[data-id='properties-nodeGPUControls'] { +.properties-editor-form + .properties-control-panel[data-id='properties-nodeUsageControls'] + .properties-control-item { + margin: 0; + padding: 0px 3px 14px 3px; + max-width: 32%; border-bottom: none; } .properties-control-panel[data-id='properties-nodeFileControl'] @@ -257,6 +253,7 @@ td { .properties-control-panel[data-id='properties-nodePropertiesControls'] .properties-control-item[data-id='properties-ci-include_subdirectories'] { border-bottom: 1px solid #cfd1d4; + padding: 0 0 10px 0; } .properties-control-panel[data-id='properties-nodePropertiesControls'] .properties-control-item[data-id='properties-ci-env_vars'] { @@ -266,26 +263,11 @@ td { .properties-control-panel > .properties-control-panel { padding: 0; } -.properties-control-item[data-id='properties-ci-memory_unit'] { - margin-top: 17px; - margin-bottom: 7px; - max-width: 60px; -} -.properties-numberfield[data-id='properties-memory_dbl'] { - padding: 5px 0; -} -.properties-numberfield[data-id='properties-memory_dbl'] - .bx--number - input[type='number'] { - border-radius: 3px 0 0 3px; -} -#properties-memory_unit-dropdown .bx--list-box__field { - border-radius: 0 3px 3px 0; -} .properties-numberfield .bx--number input[type='number'] { border-radius: 3px; border-bottom: 0; box-shadow: inset 0 0 0 var(--jp-border-width) var(--jp-input-border-color); + min-width: auto; } .properties-numberfield .bx--number input[type='number']:focus { box-shadow: inset 0 0 0 var(--jp-border-width) From e86e94fc38adb864b3b4b22f587c42cbbff307d3 Mon Sep 17 00:00:00 2001 From: Martha Cryan Date: Tue, 19 Jan 2021 22:54:11 -0600 Subject: [PATCH 05/16] Add to gpu description and fix bug --- .../pipeline-editor/src/PipelineEditorWidget.tsx | 3 +-- packages/pipeline-editor/src/properties.json | 16 ++++++++-------- 2 files changed, 9 insertions(+), 10 deletions(-) diff --git a/packages/pipeline-editor/src/PipelineEditorWidget.tsx b/packages/pipeline-editor/src/PipelineEditorWidget.tsx index 6e44d2bf6..a88cd7bb2 100644 --- a/packages/pipeline-editor/src/PipelineEditorWidget.tsx +++ b/packages/pipeline-editor/src/PipelineEditorWidget.tsx @@ -544,8 +544,7 @@ export class PipelineEditor extends React.Component< app_data.dependencies = propertySet.dependencies; app_data.include_subdirectories = propertySet.include_subdirectories; app_data.cpu = propertySet.cpu; - app_data.memory_dbl = propertySet.memory_dbl; - app_data.memory_unit = propertySet.memory_unit; + app_data.memory = propertySet.memory; app_data.gpu = propertySet.gpu; this.validateAllNodes(); this.updateModel(); diff --git a/packages/pipeline-editor/src/properties.json b/packages/pipeline-editor/src/properties.json index df151a185..2143feb97 100644 --- a/packages/pipeline-editor/src/properties.json +++ b/packages/pipeline-editor/src/properties.json @@ -24,12 +24,12 @@ "required": "false" }, { - "id": "memory", + "id": "gpu", "type": "double", "required": "false" }, { - "id": "gpu", + "id": "memory", "type": "double", "required": "false" }, @@ -84,21 +84,21 @@ } }, { - "parameter_ref": "memory", + "parameter_ref": "gpu", "label": { - "default": "RAM(GB)" + "default": "GPU" }, "description": { - "default": "The total amount of RAM specified." + "default": "For GPU-intensive workloads, you can choose more than 1 GPU. Must be an integer." } }, { - "parameter_ref": "gpu", + "parameter_ref": "memory", "label": { - "default": "GPU" + "default": "RAM(GB)" }, "description": { - "default": "For GPU-intensive workloads, you can choose more than 1 GPU (e.g. 1.5)." + "default": "The total amount of RAM specified." } }, { From ebad3f051bfe300984be005076d82c72dfa40ec9 Mon Sep 17 00:00:00 2001 From: Alan Chin Date: Wed, 20 Jan 2021 10:09:39 -0800 Subject: [PATCH 06/16] Add backend changes to support node level resource property configuration --- elyra/pipeline/parser.py | 3 ++ elyra/pipeline/pipeline.py | 51 ++++++++++++++++++++++------- elyra/pipeline/processor_kfp.py | 3 ++ elyra/templates/kfp_template.jinja2 | 3 ++ 4 files changed, 48 insertions(+), 12 deletions(-) diff --git a/elyra/pipeline/parser.py b/elyra/pipeline/parser.py index 2ee32f1d3..318993c1b 100644 --- a/elyra/pipeline/parser.py +++ b/elyra/pipeline/parser.py @@ -133,6 +133,9 @@ def _create_pipeline_operation(node: Dict, super_node: Optional[Dict] = None): id=node_id, type=node.get('type'), classifier=node.get('op'), + cpu=PipelineParser._get_app_data_field(node, 'cpu'), + gpu=PipelineParser._get_app_data_field(node, 'gpu'), + memory=PipelineParser._get_app_data_field(node, 'memory'), filename=PipelineParser._get_app_data_field(node, 'filename'), runtime_image=PipelineParser._get_app_data_field(node, 'runtime_image'), dependencies=PipelineParser._get_app_data_field(node, 'dependencies', []), diff --git a/elyra/pipeline/pipeline.py b/elyra/pipeline/pipeline.py index a1216a055..b1fc1545f 100644 --- a/elyra/pipeline/pipeline.py +++ b/elyra/pipeline/pipeline.py @@ -22,8 +22,8 @@ class Operation(object): Represents a single operation in a pipeline """ - def __init__(self, id, type, classifier, filename, runtime_image, dependencies=None, - include_subdirectories: bool = False, env_vars=None, inputs=None, outputs=None, + def __init__(self, id, type, classifier, filename, runtime_image, memory=None, cpu=None, gpu=None, + dependencies=None, include_subdirectories: bool = False, env_vars=None, inputs=None, outputs=None, parent_operations=None): """ :param id: Generated UUID, 128 bit number used as a unique identifier @@ -42,6 +42,9 @@ def __init__(self, id, type, classifier, filename, runtime_image, dependencies=N :param inputs: List of files to be consumed by this operation, produced by parent operation(s) :param outputs: List of files produced by this operation to be included in a child operation(s) :param parent_operations: List of parent operation 'ids' required to execute prior to this operation + :param cpu: number of cpus requested to run the operation + :param memory: amount of memory requested to run the operation (in Gi) + :param gpu: number of gpus requested to run the operation """ # validate that the operation has all required properties @@ -67,6 +70,9 @@ def __init__(self, id, type, classifier, filename, runtime_image, dependencies=N self._inputs = inputs or [] self._outputs = outputs or [] self._parent_operations = parent_operations or [] + self._cpu = cpu + self._gpu = gpu + self._memory = memory @property def id(self): @@ -104,6 +110,18 @@ def include_subdirectories(self): def env_vars(self): return self._env_vars + @property + def cpu(self): + return self._cpu + + @property + def memory(self): + return self._memory + + @property + def gpu(self): + return self._gpu + def env_vars_as_dict(self, logger: Optional[object] = None) -> Dict: """Operation stores environment variables in a list of name=value pairs, while subprocess.run() requires a dictionary - so we must convert. If no envs are @@ -155,7 +173,10 @@ def __eq__(self, other: object) -> bool: self.include_subdirectories == other.include_subdirectories and \ self.outputs == other.outputs and \ self.inputs == other.inputs and \ - self.parent_operations == other.parent_operations + self.parent_operations == other.parent_operations and \ + self.cpu == other.cpu and \ + self.gpu == other.gpu and \ + self.memory == other.memory def __str__(self) -> str: return "componentID : {id} \n " \ @@ -166,15 +187,21 @@ def __str__(self) -> str: "filename : {filename} \n " \ "inputs : {inputs} \n " \ "outputs : {outputs} \n " \ - "runtime image : {image} \n ".format(id=self.id, - name=self.name, - parent_op=self.parent_operations, - depends=self.dependencies, - inc_subdirs=self.include_subdirectories, - filename=self.filename, - inputs=self.inputs, - outputs=self.outputs, - image=self.runtime_image) + "image : {image} \n " \ + "gpu: {gpu} \n " \ + "memory: {memory} \n " \ + "cpu : {cpu} \n ".format(id=self.id, + name=self.name, + parent_op=self.parent_operations, + depends=self.dependencies, + inc_subdirs=self.include_subdirectories, + filename=self.filename, + inputs=self.inputs, + outputs=self.outputs, + image=self.runtime_image, + gpu=self.gpu, + cpu=self.cpu, + memory=self.memory) class Pipeline(object): diff --git a/elyra/pipeline/processor_kfp.py b/elyra/pipeline/processor_kfp.py index 6ef4771f9..a3b388e1d 100644 --- a/elyra/pipeline/processor_kfp.py +++ b/elyra/pipeline/processor_kfp.py @@ -270,6 +270,9 @@ def _cc_pipeline(self, pipeline, pipeline_name): pipeline_outputs=operation.outputs, pipeline_envs=pipeline_envs, emptydir_volume_size=emptydir_volume_size, + cpu_request=operation.cpu, + mem_request=operation.memory, + gpu_request=operation.gpu, image=operation.runtime_image, file_outputs={ 'mlpipeline-metrics': diff --git a/elyra/templates/kfp_template.jinja2 b/elyra/templates/kfp_template.jinja2 index 4adf4b0de..890648096 100644 --- a/elyra/templates/kfp_template.jinja2 +++ b/elyra/templates/kfp_template.jinja2 @@ -22,6 +22,9 @@ def create_pipeline(): cos_dependencies_archive='{{ operation.cos_dependencies_archive }}', pipeline_inputs={{ operation.pipeline_inputs }}, pipeline_outputs={{ operation.pipeline_outputs }}, + cpu_request='{{ operation.cpu_request }}', + mem_request='{{ operation.memory_request }}', + gpu_request='{{ operation.gpu_request }}', image='{{ operation.image }}', file_outputs={ 'mlpipeline-metrics': '{{ metrics_file }}', From d8b8cb1439266707ccb5960e1687b5559e13c053 Mon Sep 17 00:00:00 2001 From: Alan Chin Date: Wed, 20 Jan 2021 10:46:09 -0800 Subject: [PATCH 07/16] fix typo in kfp jinja template --- elyra/templates/kfp_template.jinja2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elyra/templates/kfp_template.jinja2 b/elyra/templates/kfp_template.jinja2 index 890648096..a64d189af 100644 --- a/elyra/templates/kfp_template.jinja2 +++ b/elyra/templates/kfp_template.jinja2 @@ -23,7 +23,7 @@ def create_pipeline(): pipeline_inputs={{ operation.pipeline_inputs }}, pipeline_outputs={{ operation.pipeline_outputs }}, cpu_request='{{ operation.cpu_request }}', - mem_request='{{ operation.memory_request }}', + mem_request='{{ operation.mem_request }}', gpu_request='{{ operation.gpu_request }}', image='{{ operation.image }}', file_outputs={ From 9b870bde5b1c1445f68e9eaa65874332a86b25d9 Mon Sep 17 00:00:00 2001 From: Alan Chin Date: Wed, 20 Jan 2021 13:53:04 -0800 Subject: [PATCH 08/16] Add memory unit to kfp template --- elyra/templates/kfp_template.jinja2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elyra/templates/kfp_template.jinja2 b/elyra/templates/kfp_template.jinja2 index a64d189af..c8bc429be 100644 --- a/elyra/templates/kfp_template.jinja2 +++ b/elyra/templates/kfp_template.jinja2 @@ -23,7 +23,7 @@ def create_pipeline(): pipeline_inputs={{ operation.pipeline_inputs }}, pipeline_outputs={{ operation.pipeline_outputs }}, cpu_request='{{ operation.cpu_request }}', - mem_request='{{ operation.mem_request }}', + mem_request='{{ operation.mem_request }}G', gpu_request='{{ operation.gpu_request }}', image='{{ operation.image }}', file_outputs={ From 0e0ad82debae61b7f37911fec7f8a6afedfa974b Mon Sep 17 00:00:00 2001 From: Alan Chin Date: Wed, 20 Jan 2021 17:36:58 -0800 Subject: [PATCH 09/16] Rename gpu variable to be more consistent --- elyra/pipeline/processor_kfp.py | 2 +- elyra/templates/kfp_template.jinja2 | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/elyra/pipeline/processor_kfp.py b/elyra/pipeline/processor_kfp.py index a3b388e1d..2b4a6f962 100644 --- a/elyra/pipeline/processor_kfp.py +++ b/elyra/pipeline/processor_kfp.py @@ -272,7 +272,7 @@ def _cc_pipeline(self, pipeline, pipeline_name): emptydir_volume_size=emptydir_volume_size, cpu_request=operation.cpu, mem_request=operation.memory, - gpu_request=operation.gpu, + gpu_limit=operation.gpu, image=operation.runtime_image, file_outputs={ 'mlpipeline-metrics': diff --git a/elyra/templates/kfp_template.jinja2 b/elyra/templates/kfp_template.jinja2 index c8bc429be..af7c7eee2 100644 --- a/elyra/templates/kfp_template.jinja2 +++ b/elyra/templates/kfp_template.jinja2 @@ -24,7 +24,7 @@ def create_pipeline(): pipeline_outputs={{ operation.pipeline_outputs }}, cpu_request='{{ operation.cpu_request }}', mem_request='{{ operation.mem_request }}G', - gpu_request='{{ operation.gpu_request }}', + gpu_limit='{{ operation.gpu_limit }}', image='{{ operation.image }}', file_outputs={ 'mlpipeline-metrics': '{{ metrics_file }}', From 85f1ee8c0c19cf9ddb9d18d370a14342450c7035 Mon Sep 17 00:00:00 2001 From: Martha Cryan Date: Fri, 22 Jan 2021 17:07:02 -0600 Subject: [PATCH 10/16] Delete properties that have null value --- .../src/PipelineEditorWidget.tsx | 20 +++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/packages/pipeline-editor/src/PipelineEditorWidget.tsx b/packages/pipeline-editor/src/PipelineEditorWidget.tsx index a88cd7bb2..4f9d93a1a 100644 --- a/packages/pipeline-editor/src/PipelineEditorWidget.tsx +++ b/packages/pipeline-editor/src/PipelineEditorWidget.tsx @@ -854,6 +854,22 @@ export class PipelineEditor extends React.Component< } } + cleanNullProperties(): void { + // Delete optional fields that have null value + for (const node of this.canvasController.getPipelineFlow().pipelines[0] + .nodes) { + if (node.app_data.cpu === null) { + delete node.app_data.cpu; + } + if (node.app_data.memory === null) { + delete node.app_data.memory; + } + if (node.app_data.gpu === null) { + delete node.app_data.gpu; + } + } + } + async handleExportPipeline(): Promise { // Warn user if the pipeline has invalid nodes const errorMessage = await this.validatePipeline(); @@ -912,6 +928,8 @@ export class PipelineEditor extends React.Component< this.widgetContext.path ); + this.cleanNullProperties(); + pipelineFlow.pipelines[0]['app_data']['name'] = pipeline_name; pipelineFlow.pipelines[0]['app_data']['runtime'] = runtime; pipelineFlow.pipelines[0]['app_data']['runtime-config'] = runtime_config; @@ -1265,6 +1283,8 @@ export class PipelineEditor extends React.Component< this.widgetContext.path ); + this.cleanNullProperties(); + pipelineFlow.pipelines[0]['app_data']['name'] = dialogResult.value.pipeline_name; pipelineFlow.pipelines[0]['app_data']['runtime'] = runtime; From ee6a7c923cbd92e68acf2ff6727628b9ff68d5d7 Mon Sep 17 00:00:00 2001 From: Alex Bozarth Date: Tue, 19 Jan 2021 21:34:48 -0800 Subject: [PATCH 11/16] Re-add support for showing resource utilization on status bar (#1204) nbresuse UI code was moved from core JupyterLab into an extension within the resource utilization APIs and was also renamed and moved moved to the jupyter-server GitHub organization. --- Makefile | 8 ++++++-- .../jupyter_server_config.d/jupyter_resource_usage.json | 7 +++++++ setup.py | 4 ++-- 3 files changed, 15 insertions(+), 4 deletions(-) create mode 100644 etc/config/jupyter_server_config.d/jupyter_resource_usage.json diff --git a/Makefile b/Makefile index 1305f616b..6f38db787 100644 --- a/Makefile +++ b/Makefile @@ -22,6 +22,7 @@ SHELL:=/bin/bash GIT_VERSION:=0.23.1 LSP_VERSION:=3.0.0 +RESUSE_VERSION:=0.5.1 TOC_VERSION:=4.0.0 TAG:=dev @@ -59,12 +60,13 @@ uninstall: $(call UNINSTALL_LAB_EXTENSION,@elyra/metadata-extension) $(call UNINSTALL_LAB_EXTENSION,@elyra/pipeline-editor-extension) $(call UNINSTALL_LAB_EXTENSION,@elyra/python-editor-extension) - $(call UNINSTALL_LAB_EXTENSION,@jupyterlab/toc) pip uninstall -y jupyterlab-git pip uninstall -y jupyter-lsp - jupyter labextension uninstall @krassowski/jupyterlab-lsp pip uninstall -y jupyterlab-lsp pip uninstall -y python-language-server + pip uninstall -y jupyter-resource-usage + - jupyter labextension uninstall @jupyter-server/resource-usage pip uninstall -y elyra - jupyter lab clean @@ -109,6 +111,7 @@ install-ui: build-ui install-external-extensions: # pip install --upgrade jupyterlab-git==$(GIT_VERSION) pip install jupyterlab-lsp==$(LSP_VERSION) + pip install jupyter-resource-usage==$(RESUSE_VERSION) pip install python-language-server[all] install: install-server install-ui install-external-extensions ## Build and install @@ -150,7 +153,8 @@ dist-ui: build-ui $(call PACKAGE_LAB_EXTENSION,pipeline-editor) $(call PACKAGE_LAB_EXTENSION,python-editor) # cd dist && curl -o jupyterlab-git-$(GIT_VERSION).tgz $$(npm view @jupyterlab/git@$(GIT_VERSION) dist.tarball) && cd - -# cd dist && curl -o jupyterlab-lsp-$(LSP_VERSION).tgz $$(npm view @jupyterlab/git@$(LSP_VERSION) dist.tarball) && cd - + cd dist && curl -o jupyterlab-lsp-$(LSP_VERSION).tgz $$(npm view @krassowski/jupyterlab-lsp@$(LSP_VERSION) dist.tarball) && cd - + cd dist && curl -o jupyter-resource-usage-$(REUSE_VERSION).tgz $$(npm view @jupyter-server/resource-usage@$(RESUSE_VERSION) dist.tarball) && cd - release: dist-ui build-server ## Build wheel file for release diff --git a/etc/config/jupyter_server_config.d/jupyter_resource_usage.json b/etc/config/jupyter_server_config.d/jupyter_resource_usage.json new file mode 100644 index 000000000..872cc9f53 --- /dev/null +++ b/etc/config/jupyter_server_config.d/jupyter_resource_usage.json @@ -0,0 +1,7 @@ +{ + "ServerApp": { + "jpserver_extensions": { + "jupyter_resource_usage": true + } + } +} diff --git a/setup.py b/setup.py index 1b63ebf17..5c08a1d00 100644 --- a/setup.py +++ b/setup.py @@ -57,14 +57,14 @@ 'jupyter_server>=1.2.0', 'jupyterlab>=3.0.0', # 'jupyterlab-git==0.21.1', - 'jupyterlab-lsp>==3.0.0', + 'jupyterlab-lsp>=3.0.0', + 'jupyter-resource-usage>=0.5.1', 'kfp-notebook>=0.17.0,<0.18.0', 'kfp==1.1.2', 'minio>=5.0.7,<7.0.0', 'nbclient>=0.5.1', 'nbconvert>=5.6.1,<6.0', # 'nbdime>=2.0.0', - 'nbresuse>=0.3.6', 'papermill>=2.1.3', 'python-language-server[all]', 'requests>=2.9.1,<3.0', From cea1dcb016aa2e0ce75f446af685d19757ebfbc8 Mon Sep 17 00:00:00 2001 From: Luciano Resende Date: Tue, 19 Jan 2021 21:36:26 -0800 Subject: [PATCH 12/16] Update jest related dependencies to avoid CI warnings (#1210) --- package.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index e469cd92f..16395fd9e 100644 --- a/package.json +++ b/package.json @@ -40,7 +40,7 @@ "@4tw/cypress-drag-drop": "^1.3.1", "@cypress/webpack-preprocessor": "^5.5.0", "@jupyterlab/testutils": "^3.0.0", - "@types/jest": "^26.0.9", + "@types/jest": "^26.0.20", "@types/react": "^17.0.0", "@types/react-dom": "^17.0.0", "@types/react-intl": "^3.0.0", @@ -56,15 +56,15 @@ "eslint-plugin-react-hooks": "^4.1.2", "husky": "^2.3.0", "install": "^0.13.0", - "jest": "^24.7.1", + "jest": "^26.6.3", "jest-raw-loader": "^1.0.1", "lerna": "^3.16.4", "lint-staged": "^9.5.0", "prettier": "^1.19.1", "rimraf": "~3.0.2", "start-server-and-test": "1.7.9", - "ts-jest": "^24.0.2", - "ts-loader": "^6.2.1", + "ts-jest": "^26.4.4", + "ts-loader": "^6.2.2", "typescript": "~4.1.3", "webpack": "^5.0.0" } From 2befd98cfee2d25e2a50dcbf4be23fab2e15b3b0 Mon Sep 17 00:00:00 2001 From: Patrick Titzler Date: Wed, 20 Jan 2021 09:25:32 -0800 Subject: [PATCH 13/16] Update python language server dpeendencies (#1211) --- Makefile | 5 +++-- setup.py | 2 +- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 6f38db787..9bfbd249c 100644 --- a/Makefile +++ b/Makefile @@ -110,9 +110,10 @@ install-ui: build-ui install-external-extensions: # pip install --upgrade jupyterlab-git==$(GIT_VERSION) - pip install jupyterlab-lsp==$(LSP_VERSION) +# pip install jupyterlab-lsp==$(LSP_VERSION) pip install jupyter-resource-usage==$(RESUSE_VERSION) - pip install python-language-server[all] +# pip install python-language-server[all] + install: install-server install-ui install-external-extensions ## Build and install jupyter lab build diff --git a/setup.py b/setup.py index 5c08a1d00..b225f0b6b 100644 --- a/setup.py +++ b/setup.py @@ -66,7 +66,7 @@ 'nbconvert>=5.6.1,<6.0', # 'nbdime>=2.0.0', 'papermill>=2.1.3', - 'python-language-server[all]', + 'python-language-server[all]>=0.36.2', 'requests>=2.9.1,<3.0', 'rfc3986-validator>=0.1.1', 'traitlets>=4.3.2', From 7d28cb7513b8012d27dc2b141223108137d30423 Mon Sep 17 00:00:00 2001 From: Luciano Resende Date: Wed, 20 Jan 2021 10:12:35 -0800 Subject: [PATCH 14/16] Schedule GitHub actions to run daily (#1209) --- .github/workflows/{pythonapp.yml => build.yml} | 3 +++ 1 file changed, 3 insertions(+) rename .github/workflows/{pythonapp.yml => build.yml} (97%) diff --git a/.github/workflows/pythonapp.yml b/.github/workflows/build.yml similarity index 97% rename from .github/workflows/pythonapp.yml rename to .github/workflows/build.yml index 74607798f..abb4e81ad 100644 --- a/.github/workflows/pythonapp.yml +++ b/.github/workflows/build.yml @@ -19,6 +19,9 @@ on: branches: '*' pull_request: branches: '*' + schedule: + # once a day at 3 am (UTC) (7 pm (PST)) + - cron: '0 3 * * *' jobs: build: From 2d82e096d710773e5d1b252caa3be7935d95c521 Mon Sep 17 00:00:00 2001 From: Alex Bozarth Date: Wed, 20 Jan 2021 18:55:15 -0800 Subject: [PATCH 15/16] Reenable git extension support (#1202) --- Makefile | 21 +++------------------ setup.py | 6 +++--- tests/integration/git.ts | 20 +++++++++----------- 3 files changed, 15 insertions(+), 32 deletions(-) diff --git a/Makefile b/Makefile index 9bfbd249c..fcf586be3 100644 --- a/Makefile +++ b/Makefile @@ -15,16 +15,11 @@ # .PHONY: help purge uninstall clean test-dependencies lint-server lint-ui lint yarn-install build-ui build-server install-server -.PHONY: install-external-extensions install watch test-server test-ui test-ui-debug test docs-dependencies docs dist-ui release +.PHONY: install watch test-server test-ui test-ui-debug test docs-dependencies docs dist-ui release .PHONY: docker-image, validate-runtime-images SHELL:=/bin/bash -GIT_VERSION:=0.23.1 -LSP_VERSION:=3.0.0 -RESUSE_VERSION:=0.5.1 -TOC_VERSION:=4.0.0 - TAG:=dev IMAGE=elyra/elyra:$(TAG) @@ -96,7 +91,7 @@ build-server: lint-server ## Build backend build: build-server build-ui install-server: build-server ## Install backend - pip install --upgrade --upgrade-strategy $(UPGRADE_STRATEGY) dist/elyra-*-py3-none-any.whl + pip install --upgrade --upgrade-strategy $(UPGRADE_STRATEGY) --use-deprecated=legacy-resolver dist/elyra-*-py3-none-any.whl install-ui: build-ui $(call LINK_LAB_EXTENSION,application) @@ -108,14 +103,7 @@ install-ui: build-ui $(call INSTALL_LAB_EXTENSION,pipeline-editor) $(call INSTALL_LAB_EXTENSION,python-editor) -install-external-extensions: -# pip install --upgrade jupyterlab-git==$(GIT_VERSION) -# pip install jupyterlab-lsp==$(LSP_VERSION) - pip install jupyter-resource-usage==$(RESUSE_VERSION) -# pip install python-language-server[all] - - -install: install-server install-ui install-external-extensions ## Build and install +install: install-server install-ui ## Build and install jupyter lab build jupyter serverextension list jupyter server extension list @@ -153,9 +141,6 @@ dist-ui: build-ui $(call PACKAGE_LAB_EXTENSION,metadata) $(call PACKAGE_LAB_EXTENSION,pipeline-editor) $(call PACKAGE_LAB_EXTENSION,python-editor) -# cd dist && curl -o jupyterlab-git-$(GIT_VERSION).tgz $$(npm view @jupyterlab/git@$(GIT_VERSION) dist.tarball) && cd - - cd dist && curl -o jupyterlab-lsp-$(LSP_VERSION).tgz $$(npm view @krassowski/jupyterlab-lsp@$(LSP_VERSION) dist.tarball) && cd - - cd dist && curl -o jupyter-resource-usage-$(REUSE_VERSION).tgz $$(npm view @jupyter-server/resource-usage@$(RESUSE_VERSION) dist.tarball) && cd - release: dist-ui build-server ## Build wheel file for release diff --git a/setup.py b/setup.py index b225f0b6b..45c686ca6 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ exec(f.read(), {}, version_ns) npm_packages_path = "./dist/*.tgz" -auto_extension_path = "./etc/config/jupyter_server_config.d/elyra.json" # TODO change back to include git +auto_extension_path = "./etc/config/jupyter_server_config.d/*.json" settings_path = './etc/config/settings/*.json' metadata_path = './etc/config/metadata/runtime-images/*.json' @@ -56,7 +56,7 @@ 'jupyter_client>=6.1.7', 'jupyter_server>=1.2.0', 'jupyterlab>=3.0.0', - # 'jupyterlab-git==0.21.1', + 'jupyterlab-git==0.30.0b1', 'jupyterlab-lsp>=3.0.0', 'jupyter-resource-usage>=0.5.1', 'kfp-notebook>=0.17.0,<0.18.0', @@ -64,7 +64,7 @@ 'minio>=5.0.7,<7.0.0', 'nbclient>=0.5.1', 'nbconvert>=5.6.1,<6.0', - # 'nbdime>=2.0.0', + 'nbdime>=3.0.0.b1', 'papermill>=2.1.3', 'python-language-server[all]>=0.36.2', 'requests>=2.9.1,<3.0', diff --git a/tests/integration/git.ts b/tests/integration/git.ts index 620ba7c8d..e90febf31 100644 --- a/tests/integration/git.ts +++ b/tests/integration/git.ts @@ -14,14 +14,12 @@ * limitations under the License. */ -// TODO: Enable once git is supporting lab 3 -// -// describe('Git', () => { -// it('opens jupyterlab', () => { -// cy.visit('?token=test&reset'); -// }); -// -// it('opens git extension', () => { -// cy.get('[title="Git"]').click(); -// }); -// }); +describe('Git', () => { + it('opens jupyterlab', () => { + cy.visit('?token=test&reset'); + }); + + it('opens git extension', () => { + cy.get('[title="Git"]').click(); + }); +}); From 94d529591c5e85d3c0a6bc095e882eb6a75b671b Mon Sep 17 00:00:00 2001 From: Patrick Titzler Date: Wed, 20 Jan 2021 19:03:07 -0800 Subject: [PATCH 16/16] Create pipeline issue template (#1214) Co-authored-by: Karla Spuldaro --- .../ISSUE_TEMPLATE/pipeline-issue-report.md | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/pipeline-issue-report.md diff --git a/.github/ISSUE_TEMPLATE/pipeline-issue-report.md b/.github/ISSUE_TEMPLATE/pipeline-issue-report.md new file mode 100644 index 000000000..7814ff706 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/pipeline-issue-report.md @@ -0,0 +1,45 @@ +--- +name: Pipeline issue report +about: Report a potential issue related to the pipeline editor or pipeline execution +title: '' +labels: component:pipeline-editor, component:pipeline-runtime, status:Needs Triage, kind:user-feedback +assignees: '' + +--- + +**Describe the issue** +A clear and concise description of what the issue is. + +**To Reproduce** +Steps to reproduce the behavior: +1. Go to '...' +2. Click on '....' +3. Scroll down to '....' +4. See error + +**Screenshots or log output** +If applicable, add screenshots or log output to help explain your problem. +
Log Output +
+Paste the log output here.
+
+
+ +**Expected behavior** +A clear and concise description of what you expected to happen. + +**Deployment information** +Describe what you've deployed and how: + - Elyra version: [e.g. 1.5.3] + - Operating system: [e.g. macos, linux] + - Installation source: [e.g. PyPI, conda, from source, official container image, custom container image] + - Deployment type: [e.g. local installation, Docker, Kubernetes, Kubeflow [notebook server] , Open Data Hub] + +**Pipeline runtime environment** +If the issue is related to pipeline execution, identify the environment where the pipeline is executed + - Local execution + - Kubeflow Pipelines (provide version number, whether multi-user auth enabled) + - Apache Airflow (provide version number) + +**Runtime configuration settings** +If the issue is related to pipeline execution, document the runtime configuration settings from the Elyra UI, omitting confidential information.