Skip to content

Commit

Permalink
Cleanup DataConfig implementation
Browse files Browse the repository at this point in the history 
* Removed DataConfig::params_config
* Removed DataConfig::components/component_args

All components specific parameters are now grouped in four separate objects:
 + load_dataset_config
 + pre_process_data_config
 + post_process_data_config
 + dataloader_config
  • Loading branch information
@shaahji
shaahji committed Jun 5, 2024
1 parent 7ca6fc4 commit e0ff9e8
Show file tree
Hide file tree
Showing 70 changed files with 1,703 additions and 1,232 deletions.
36 changes: 23 additions & 13 deletions .azure_pipelines/performance_check/configs/bert.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,19 +12,29 @@
{
"name": "glue_mrpc",
"type": "HuggingfaceContainer",
"params_config": {
"batch_size": 1,
"max_samples": 100,
"data_name": "glue",
"input_cols": [
"sentence1",
"sentence2"
],
"label_cols": [
"label"
],
"split": "validation",
"subset": "mrpc"
"load_dataset_config": {
"params": {
"data_name": "glue",
"split": "validation",
"subset": "mrpc"
}
},
"pre_process_data_config": {
"params": {
"input_cols": [
"sentence1",
"sentence2"
],
"label_cols": [
"label"
],
"max_samples": 100
}
},
"dataloader_config": {
"params": {
"batch_size": 1
}
}
}
],
Expand Down
36 changes: 23 additions & 13 deletions .azure_pipelines/performance_check/configs/bert_gpu.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,19 +27,29 @@
{
"name": "glue_mrpc",
"type": "HuggingfaceContainer",
"params_config": {
"batch_size": 1,
"max_samples": 100,
"data_name": "glue",
"input_cols": [
"sentence1",
"sentence2"
],
"label_cols": [
"label"
],
"split": "validation",
"subset": "mrpc"
"load_dataset_config": {
"params": {
"data_name": "glue",
"split": "validation",
"subset": "mrpc"
}
},
"pre_process_data_config": {
"params": {
"input_cols": [
"sentence1",
"sentence2"
],
"label_cols": [
"label"
],
"max_samples": 100
}
},
"dataloader_config": {
"params": {
"batch_size": 1
}
}
}
],
Expand Down
40 changes: 23 additions & 17 deletions .azure_pipelines/performance_check/configs/deberta.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,23 +12,29 @@
{
"name": "glue_mnli_matched",
"type": "HuggingfaceContainer",
"params_config": {
"data_name": "glue",
"subset": "mnli_matched",
"split": "validation",
"input_cols": [
"premise",
"hypothesis"
],
"label_cols": [
"label"
],
"batch_size": 1,
"max_samples": 100,
"component_kwargs": {
"pre_process_data": {
"align_labels": true
}
"load_dataset_config": {
"params": {
"data_name": "glue",
"subset": "mnli_matched",
"split": "validation"
}
},
"pre_process_data_config": {
"params": {
"align_labels": true,
"input_cols": [
"premise",
"hypothesis"
],
"label_cols": [
"label"
],
"max_samples": 100
}
},
"dataloader_config": {
"params": {
"batch_size": 1
}
}
}
Expand Down
40 changes: 23 additions & 17 deletions .azure_pipelines/performance_check/configs/deberta_gpu.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,23 +27,29 @@
{
"name": "glue_mnli_matched",
"type": "HuggingfaceContainer",
"params_config": {
"data_name": "glue",
"subset": "mnli_matched",
"split": "validation",
"input_cols": [
"premise",
"hypothesis"
],
"label_cols": [
"label"
],
"batch_size": 1,
"max_samples": 100,
"component_kwargs": {
"pre_process_data": {
"align_labels": true
}
"load_dataset_config": {
"params": {
"data_name": "glue",
"subset": "mnli_matched",
"split": "validation"
}
},
"pre_process_data_config": {
"params": {
"align_labels": true,
"input_cols": [
"premise",
"hypothesis"
],
"label_cols": [
"label"
],
"max_samples": 100
}
},
"dataloader_config": {
"params": {
"batch_size": 1
}
}
}
Expand Down
34 changes: 22 additions & 12 deletions .azure_pipelines/performance_check/configs/distilbert.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,18 +12,28 @@
{
"name": "glue_sst2",
"type": "HuggingfaceContainer",
"params_config": {
"data_name": "glue",
"subset": "sst2",
"split": "validation",
"input_cols": [
"sentence"
],
"label_cols": [
"label"
],
"batch_size": 1,
"max_samples": 100
"load_dataset_config": {
"params": {
"data_name": "glue",
"subset": "sst2",
"split": "validation"
}
},
"pre_process_data_config": {
"params": {
"input_cols": [
"sentence"
],
"label_cols": [
"label"
],
"max_samples": 100
}
},
"dataloader_config": {
"params": {
"batch_size": 1
}
}
}
],
Expand Down
34 changes: 22 additions & 12 deletions .azure_pipelines/performance_check/configs/distilbert_gpu.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,18 +27,28 @@
{
"name": "glue_sst2",
"type": "HuggingfaceContainer",
"params_config": {
"data_name": "glue",
"subset": "sst2",
"split": "validation",
"input_cols": [
"sentence"
],
"label_cols": [
"label"
],
"batch_size": 1,
"max_samples": 100
"load_dataset_config": {
"params": {
"data_name": "glue",
"subset": "sst2",
"split": "validation"
}
},
"pre_process_data_config": {
"params": {
"input_cols": [
"sentence"
],
"label_cols": [
"label"
],
"max_samples": 100
}
},
"dataloader_config": {
"params": {
"batch_size": 1
}
}
}
],
Expand Down
40 changes: 23 additions & 17 deletions .azure_pipelines/performance_check/configs/roberta_large.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,23 +12,29 @@
{
"name": "glue_mnli_matched",
"type": "HuggingfaceContainer",
"params_config": {
"data_name": "glue",
"subset": "mnli_matched",
"split": "validation",
"input_cols": [
"premise",
"hypothesis"
],
"label_cols": [
"label"
],
"batch_size": 1,
"max_samples": 100,
"component_kwargs": {
"pre_process_data": {
"align_labels": true
}
"load_dataset_config": {
"params": {
"data_name": "glue",
"subset": "mnli_matched",
"split": "validation"
}
},
"pre_process_data_config": {
"params": {
"align_labels": true,
"input_cols": [
"premise",
"hypothesis"
],
"label_cols": [
"label"
],
"max_samples": 100
}
},
"dataloader_config": {
"params": {
"batch_size": 1
}
}
}
Expand Down
40 changes: 23 additions & 17 deletions .azure_pipelines/performance_check/configs/roberta_large_gpu.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,23 +27,29 @@
{
"name": "glue_mnli_matched",
"type": "HuggingfaceContainer",
"params_config": {
"data_name": "glue",
"subset": "mnli_matched",
"split": "validation",
"input_cols": [
"premise",
"hypothesis"
],
"label_cols": [
"label"
],
"batch_size": 1,
"max_samples": 100,
"component_kwargs": {
"pre_process_data": {
"align_labels": true
}
"load_dataset_config": {
"params": {
"data_name": "glue",
"subset": "mnli_matched",
"split": "validation"
}
},
"pre_process_data_config": {
"params": {
"align_labels": true,
"input_cols": [
"premise",
"hypothesis"
],
"label_cols": [
"label"
],
"max_samples": 100
}
},
"dataloader_config": {
"params": {
"batch_size": 1
}
}
}
Expand Down
Loading

0 comments on commit e0ff9e8

Please sign in to comment.