/
REST API Elastic Map Reduce
REST API Elastic Map Reduce
Juliane Wetzel
Owned by Juliane Wetzel
View the current EMR Configuration
| Request Method | GET |
| Description | Returns the EMR configuration. |
| URI Syntax | curl -u 'admin:admin' -X GET 'localhost:8088/api/cluster-configuration/emr' > test.json |
| Example Response | No IAM usage and determining the cluster by given hostname:
Expand source
{
"s3Bucket": "s3://datameer.emr.test",
"s3StorageFolder": "/user/datameer",
"useEc2IamRole": false,
"useYarnResourceManagerHostname": true,
"yarnResourceManagerHostname": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com",
"awsS3AccessKey": "ASFASDFasdfaskldjfhWER",
"awsS3AccessSecret": "876876mnabsfdafw2342+SIGNPp",
"yarnResourceManagerAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8032",
"yarnResourceManagerWebappAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8088",
"yarnResourceManagerSchedulerAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8030",
"yarnApplicationClasspath": "/etc/hadoop/conf:/usr/lib/hadoop-hdfs/./:/usr/lib/hadoop-hdfs/.//*:/usr/lib/hadoop-hdfs/lib/*:/usr/lib/hadoop-lzo/lib/*:/usr/lib/hadoop-mapreduce/.//*:/usr/lib/hadoop-mapreduce/lib/*:/usr/lib/hadoop-yarn/.//*:/usr/lib/hadoop-yarn/lib/*:/usr/lib/hadoop/.//*:/usr/lib/hadoop/lib/*:/usr/share/aws/aws-java-sdk/*:/usr/share/aws/emr/cloudwatch-sink/lib/*:/usr/share/aws/emr/ddb/lib/emr-ddb-hadoop.jar:/usr/share/aws/emr/emrfs/auxlib/*:/usr/share/aws/emr/emrfs/conf:/usr/share/aws/emr/emrfs/lib/*:/usr/share/aws/emr/goodies/lib/emr-hadoop-goodies.jar:/usr/share/aws/emr/kinesis/lib/emr-kinesis-hadoop.jar:/usr/share/aws/emr/security/conf:/usr/share/aws/emr/security/lib/*",
"maxConcurrentJobs": 25,
"emrPollingInterval": 30,
"defaultHadoopProperties": {
"mapred.map.output.compression.type": "BLOCK",
"mapreduce.map.output.compress": "true",
"mapreduce.output.fileoutputformat.compress": "true",
"mapreduce.output.fileoutputformat.compress.type": "BLOCK",
"mapreduce.input.fileinputformat.split.minsize": "134217728",
"mapreduce.job.max.split.locations": "20",
"mapreduce.input.fileinputformat.split.maxsize": "5368709120",
"yarn.app.mapreduce.am.staging-dir": "/user"
},
"hadoopDistributionSpecificProperties": {},
"customProperties": {
"das.execution-framework": "Tez"
},
"defaultLogSeverity": "DEFAULT",
"loggingCustomization": {}
}
No IAM usage and determining the cluster by given cluster name:
Expand source
{
"s3Bucket": "s3://datameer.emr.test",
"s3StorageFolder": "/user/datameer",
"useEc2IamRole": false,
"awsS3AccessKey": "ASFASDFasdfaskldjfhWER",
"awsS3AccessSecret": "876876mnabsfdafw2342+SIGNPp",
"useYarnResourceManagerHostname": true,
"yarnResourceManagerHostname": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com",
"yarnResourceManagerAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8032",
"yarnResourceManagerWebappAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8088",
"yarnResourceManagerSchedulerAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8030",
"yarnApplicationClasspath": "/etc/hadoop/conf:/usr/lib/hadoop-hdfs/./:/usr/lib/hadoop-hdfs/.//*:/usr/lib/hadoop-hdfs/lib/*:/usr/lib/hadoop-lzo/lib/*:/usr/lib/hadoop-mapreduce/.//*:/usr/lib/hadoop-mapreduce/lib/*:/usr/lib/hadoop-yarn/.//*:/usr/lib/hadoop-yarn/lib/*:/usr/lib/hadoop/.//*:/usr/lib/hadoop/lib/*:/usr/share/aws/aws-java-sdk/*:/usr/share/aws/emr/cloudwatch-sink/lib/*:/usr/share/aws/emr/ddb/lib/emr-ddb-hadoop.jar:/usr/share/aws/emr/emrfs/auxlib/*:/usr/share/aws/emr/emrfs/conf:/usr/share/aws/emr/emrfs/lib/*:/usr/share/aws/emr/goodies/lib/emr-hadoop-goodies.jar:/usr/share/aws/emr/kinesis/lib/emr-kinesis-hadoop.jar:/usr/share/aws/emr/security/conf:/usr/share/aws/emr/security/lib/*",
"maxConcurrentJobs": 25,
"emrPollingInterval": 30,
"defaultHadoopProperties": {
"mapred.map.output.compression.type": "BLOCK",
"mapreduce.map.output.compress": "true",
"mapreduce.output.fileoutputformat.compress": "true",
"mapreduce.output.fileoutputformat.compress.type": "BLOCK",
"mapreduce.input.fileinputformat.split.minsize": "134217728",
"mapreduce.job.max.split.locations": "20",
"mapreduce.input.fileinputformat.split.maxsize": "5368709120",
"yarn.app.mapreduce.am.staging-dir": "/user"
},
"hadoopDistributionSpecificProperties": {},
"customProperties": {
"das.execution-framework": "Tez"
},
"defaultLogSeverity": "DEFAULT",
"loggingCustomization": {}
}
IAM usage and determining the cluster by given hostname:
Expand source
{
"s3Bucket": "s3://datameer.emr.test",
"s3StorageFolder": "/user/datameer",
"useEc2IamRole": true,
"useYarnResourceManagerHostname": true,
"yarnResourceManagerHostname": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com",
"yarnResourceManagerAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8032",
"yarnResourceManagerWebappAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8088",
"yarnResourceManagerSchedulerAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8030",
"yarnApplicationClasspath": "/etc/hadoop/conf:/usr/lib/hadoop-hdfs/./:/usr/lib/hadoop-hdfs/.//*:/usr/lib/hadoop-hdfs/lib/*:/usr/lib/hadoop-lzo/lib/*:/usr/lib/hadoop-mapreduce/.//*:/usr/lib/hadoop-mapreduce/lib/*:/usr/lib/hadoop-yarn/.//*:/usr/lib/hadoop-yarn/lib/*:/usr/lib/hadoop/.//*:/usr/lib/hadoop/lib/*:/usr/share/aws/aws-java-sdk/*:/usr/share/aws/emr/cloudwatch-sink/lib/*:/usr/share/aws/emr/ddb/lib/emr-ddb-hadoop.jar:/usr/share/aws/emr/emrfs/auxlib/*:/usr/share/aws/emr/emrfs/conf:/usr/share/aws/emr/emrfs/lib/*:/usr/share/aws/emr/goodies/lib/emr-hadoop-goodies.jar:/usr/share/aws/emr/kinesis/lib/emr-kinesis-hadoop.jar:/usr/share/aws/emr/security/conf:/usr/share/aws/emr/security/lib/*",
"maxConcurrentJobs": 25,
"emrPollingInterval": 30,
"defaultHadoopProperties": {
"mapred.map.output.compression.type": "BLOCK",
"mapreduce.map.output.compress": "true",
"mapreduce.output.fileoutputformat.compress": "true",
"mapreduce.output.fileoutputformat.compress.type": "BLOCK",
"mapreduce.input.fileinputformat.split.minsize": "134217728",
"mapreduce.job.max.split.locations": "20",
"mapreduce.input.fileinputformat.split.maxsize": "5368709120",
"yarn.app.mapreduce.am.staging-dir": "/user"
},
"hadoopDistributionSpecificProperties": {},
"customProperties": {
"das.execution-framework": "Tez"
},
"defaultLogSeverity": "DEFAULT",
"loggingCustomization": {}
}
|
Update the Current EMR Configuration
| Request Method | PUT |
| Description | Updates the EMR configuration. Updating the configuration overrides the current configuration. INFO There is no way to replace a single configuration parameter. |
| URI Syntax | curl -u 'admin:admin' -X PUT -d @test.json 'localhost:8088/api/cluster-configuration/emr' -v -H "Content-Type:application/json" |
| Example Response | No IAM usage and determining the cluster by given hostname:
Expand source
{
"s3Bucket": "s3://datameer.emr.test",
"s3StorageFolder": "/user/datameer",
"useEc2IamRole": false,
"useYarnResourceManagerHostname": true,
"yarnResourceManagerHostname": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com",
"awsS3AccessKey": "ASFASDFasdfaskldjfhWER",
"awsS3AccessSecret": "876876mnabsfdafw2342+SIGNPp",
"yarnApplicationClasspath": "/etc/hadoop/conf:/usr/lib/hadoop-hdfs/./:/usr/lib/hadoop-hdfs/.//*:/usr/lib/hadoop-hdfs/lib/*:/usr/lib/hadoop-lzo/lib/*:/usr/lib/hadoop-mapreduce/.//*:/usr/lib/hadoop-mapreduce/lib/*:/usr/lib/hadoop-yarn/.//*:/usr/lib/hadoop-yarn/lib/*:/usr/lib/hadoop/.//*:/usr/lib/hadoop/lib/*:/usr/share/aws/aws-java-sdk/*:/usr/share/aws/emr/cloudwatch-sink/lib/*:/usr/share/aws/emr/ddb/lib/emr-ddb-hadoop.jar:/usr/share/aws/emr/emrfs/auxlib/*:/usr/share/aws/emr/emrfs/conf:/usr/share/aws/emr/emrfs/lib/*:/usr/share/aws/emr/goodies/lib/emr-hadoop-goodies.jar:/usr/share/aws/emr/kinesis/lib/emr-kinesis-hadoop.jar:/usr/share/aws/emr/security/conf:/usr/share/aws/emr/security/lib/*",
"maxConcurrentJobs": 25,
"emrPollingInterval": 30,
"defaultHadoopProperties": {
"mapred.map.output.compression.type": "BLOCK",
"mapreduce.map.output.compress": "true",
"mapreduce.output.fileoutputformat.compress": "true",
"mapreduce.output.fileoutputformat.compress.type": "BLOCK",
"mapreduce.input.fileinputformat.split.minsize": "134217728",
"mapreduce.job.max.split.locations": "20",
"mapreduce.input.fileinputformat.split.maxsize": "5368709120",
"yarn.app.mapreduce.am.staging-dir": "/user"
},
"hadoopDistributionSpecificProperties": {},
"customProperties": {
"das.execution-framework": "Tez"
},
"defaultLogSeverity": "DEFAULT",
"loggingCustomization": {}
}
No IAM usage and determining the cluster by given Cluster name:
Expand source
{
"s3Bucket": "s3://datameer.emr.test",
"s3StorageFolder": "/user/datameer",
"useEc2IamRole": false,
"awsS3AccessKey": "ASFASDFasdfaskldjfhWER",
"awsS3AccessSecret": "876876mnabsfdafw2342+SIGNPp",
"useYarnResourceManagerHostname": true,
"yarnResourceManagerHostname": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com",
"yarnResourceManagerAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8032",
"yarnResourceManagerWebappAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8088",
"yarnResourceManagerSchedulerAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8030",
"yarnApplicationClasspath": "/etc/hadoop/conf:/usr/lib/hadoop-hdfs/./:/usr/lib/hadoop-hdfs/.//*:/usr/lib/hadoop-hdfs/lib/*:/usr/lib/hadoop-lzo/lib/*:/usr/lib/hadoop-mapreduce/.//*:/usr/lib/hadoop-mapreduce/lib/*:/usr/lib/hadoop-yarn/.//*:/usr/lib/hadoop-yarn/lib/*:/usr/lib/hadoop/.//*:/usr/lib/hadoop/lib/*:/usr/share/aws/aws-java-sdk/*:/usr/share/aws/emr/cloudwatch-sink/lib/*:/usr/share/aws/emr/ddb/lib/emr-ddb-hadoop.jar:/usr/share/aws/emr/emrfs/auxlib/*:/usr/share/aws/emr/emrfs/conf:/usr/share/aws/emr/emrfs/lib/*:/usr/share/aws/emr/goodies/lib/emr-hadoop-goodies.jar:/usr/share/aws/emr/kinesis/lib/emr-kinesis-hadoop.jar:/usr/share/aws/emr/security/conf:/usr/share/aws/emr/security/lib/*",
"maxConcurrentJobs": 25,
"emrPollingInterval": 30,
"defaultHadoopProperties": {
"mapred.map.output.compression.type": "BLOCK",
"mapreduce.map.output.compress": "true",
"mapreduce.output.fileoutputformat.compress": "true",
"mapreduce.output.fileoutputformat.compress.type": "BLOCK",
"mapreduce.input.fileinputformat.split.minsize": "134217728",
"mapreduce.job.max.split.locations": "20",
"mapreduce.input.fileinputformat.split.maxsize": "5368709120",
"yarn.app.mapreduce.am.staging-dir": "/user"
},
"hadoopDistributionSpecificProperties": {},
"customProperties": {
"das.execution-framework": "Tez"
},
"defaultLogSeverity": "DEFAULT",
"loggingCustomization": {}
}
IAM usage and determining the cluster by given hostname:
Expand source
{
"s3Bucket": "s3://datameer.emr.test",
"s3StorageFolder": "/user/datameer",
"useEc2IamRole": true,
"useYarnResourceManagerHostname": true,
"yarnResourceManagerHostname": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com",
"yarnApplicationClasspath": "/etc/hadoop/conf:/usr/lib/hadoop-hdfs/./:/usr/lib/hadoop-hdfs/.//*:/usr/lib/hadoop-hdfs/lib/*:/usr/lib/hadoop-lzo/lib/*:/usr/lib/hadoop-mapreduce/.//*:/usr/lib/hadoop-mapreduce/lib/*:/usr/lib/hadoop-yarn/.//*:/usr/lib/hadoop-yarn/lib/*:/usr/lib/hadoop/.//*:/usr/lib/hadoop/lib/*:/usr/share/aws/aws-java-sdk/*:/usr/share/aws/emr/cloudwatch-sink/lib/*:/usr/share/aws/emr/ddb/lib/emr-ddb-hadoop.jar:/usr/share/aws/emr/emrfs/auxlib/*:/usr/share/aws/emr/emrfs/conf:/usr/share/aws/emr/emrfs/lib/*:/usr/share/aws/emr/goodies/lib/emr-hadoop-goodies.jar:/usr/share/aws/emr/kinesis/lib/emr-kinesis-hadoop.jar:/usr/share/aws/emr/security/conf:/usr/share/aws/emr/security/lib/*",
"maxConcurrentJobs": 25,
"emrPollingInterval": 30,
"defaultHadoopProperties": {
"mapred.map.output.compression.type": "BLOCK",
"mapreduce.map.output.compress": "true",
"mapreduce.output.fileoutputformat.compress": "true",
"mapreduce.output.fileoutputformat.compress.type": "BLOCK",
"mapreduce.input.fileinputformat.split.minsize": "134217728",
"mapreduce.job.max.split.locations": "20",
"mapreduce.input.fileinputformat.split.maxsize": "5368709120",
"yarn.app.mapreduce.am.staging-dir": "/user"
},
"hadoopDistributionSpecificProperties": {},
"customProperties": {
"das.execution-framework": "Tez"
},
"defaultLogSeverity": "DEFAULT",
"loggingCustomization": {}
}
|
, multiple selections available,
Related content
REST API Elastic Map Reduce
REST API Elastic Map Reduce
More like this
REST API EMR
REST API EMR
More like this
REST API Support Engineer Report Plug-In
REST API Support Engineer Report Plug-In
More like this
REST API Support Engineer Report Plug-In
REST API Support Engineer Report Plug-In
More like this
Set up Datameer on EMR
Set up Datameer on EMR
More like this
Set up Datameer on EMR
Set up Datameer on EMR
More like this