PROBLEM DESCRIPTION


How to enable advanced configs of pipeline settings to be present when we import the pipeline JSON via REST-API?



WORKAROUND/ RESOLUTION


Under the top level iw_mappings array of the JSON, for each entry in the array having entity type as "entity_config",  add the below key:

"to_entity_id": null, "import": true

For entry in the array having entity type as "pipeline",  add the below key:

"to_entity_id": <entity id of the target pipeline where the configuration is being imported into>


Example:


Please find the sample JSON. You need to add the highlighted values in JSON according to your requirements as explained above.


{

"entity": {

"entity_id": {

"$type": "oid",

"$value": "eb81e35520d23ee60a215696"

},

"entity_name": "test_pipeline_2",

"entity_type": "pipeline"

},

"entity_configs": [

{

"configuration": {

"description": "",

"entityId": {

"$type": "oid",

"$value": "eb81e35520d23ee60a215696"

},

"entityType": "pipeline",

"isActive": true,

"key": "key1",

"value": "df_batch_hive_settings=tez.queue.name=queue1;yarn.opts.param=true"

},

"entity_id": {

"$type": "oid",

"$value": "b262858ba198924def12f9f2"

},

"entity_type": "entity_config"

},

{

"configuration": {

"description": "",

"entityId": {

"$type": "oid","$value": "eb81e35520d23ee60a215696"

},

"entityType": "pipeline",

"isActive": true,

"key": "key2",

"value": "value2"

},

"entity_id": {

"$type": "oid",

"$value": "4cc8e8a705ba9f511d08ddc5"

},

"entity_type": "entity_config"

}

],

"export": {

"exportedAt": {

"$date": 1535128393228

},

"exportedBy": "Sripada Ravindranath"

},

"iw_mappings": [

{

"entity_id": {

"$type": "oid",

"$value": "eb81e35520d23ee60a215696"

},

"to_entity_id": {

"$type": "oid",

"$value": "eb81e35520d23ee60a215696"

},

"entity_type": "pipeline",

"recommendation": {

"pipeline_name": "test_pipeline_2"

}

},

{

"entity_id": {

"$type": "oid",

"$value": "5b6343eb8d6aca549ba48458"

},

"to_entity_id": {

"$type": "oid",

"$value": "5b6343eb8d6aca549ba48458"

},

"entity_type": "table",

"recommendation": {

"source_name": "DF_TPC_DS",

"table_name": "STORE"

}

},

{

"entity_id": {

"$type": "oid",

"$value": "b262858ba198924def12f9f2"

},

"entity_type": "entity_config",

"to_entity_id": null,

"import": true,

"recommendation": {

"entity_name": "test_pipeline_2",

"entity_type": "pipeline",

"key": "key1"

}

},

{

"entity_id": {

"$type": "oid",

"$value": "4cc8e8a705ba9f511d08ddc5"

},

"entity_type": "entity_config",

"to_entity_id": null,

"import": true,

"recommendation": {

"entity_name": "test_pipeline_2",

"entity_type": "pipeline",

"key": "key2"

}

}

],

"pipeline": {

"batch_engine": "HIVE",

"description": "",

"model": {

"nodes": {},

"pipeline_graph": []

},

"pipeline_parameters": [],

"view_options": {

"layout": {

"layerSpacing": 25,

"orientation": "horizontal"

},

"linkStyle": "orthogonal"

   }

 }

}