REST API Elastic Map Reduce
View the current EMR Configuration
| Request Method | GET |
| Description | Returns the EMR configuration. |
| URI Syntax | curl -u admin:admin -X GET 'localhost:8088/api/cluster-configuration/emr' > test.json |
| Example Response | No IAM usage and determining the cluster by given hostname: {
"s3Bucket": "s3://datameer.emr.test",
"s3StorageFolder": "/user/datameer",
"useEc2IamRole": false,
"useYarnResourceManagerHostname": true,
"yarnResourceManagerHostname": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com",
"awsS3AccessKey": "ASFASDFasdfaskldjfhWER",
"awsS3AccessSecret": "876876mnabsfdafw2342+SIGNPp",
"yarnResourceManagerAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8032",
"yarnResourceManagerWebappAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8088",
"yarnResourceManagerSchedulerAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8030",
"yarnApplicationClasspath": "/etc/hadoop/conf:/usr/lib/hadoop-hdfs/./:/usr/lib/hadoop-hdfs/.//*:/usr/lib/hadoop-hdfs/lib/*:/usr/lib/hadoop-lzo/lib/*:/usr/lib/hadoop-mapreduce/.//*:/usr/lib/hadoop-mapreduce/lib/*:/usr/lib/hadoop-yarn/.//*:/usr/lib/hadoop-yarn/lib/*:/usr/lib/hadoop/.//*:/usr/lib/hadoop/lib/*:/usr/share/aws/aws-java-sdk/*:/usr/share/aws/emr/cloudwatch-sink/lib/*:/usr/share/aws/emr/ddb/lib/emr-ddb-hadoop.jar:/usr/share/aws/emr/emrfs/auxlib/*:/usr/share/aws/emr/emrfs/conf:/usr/share/aws/emr/emrfs/lib/*:/usr/share/aws/emr/goodies/lib/emr-hadoop-goodies.jar:/usr/share/aws/emr/kinesis/lib/emr-kinesis-hadoop.jar:/usr/share/aws/emr/security/conf:/usr/share/aws/emr/security/lib/*",
"maxConcurrentJobs": 25,
"emrPollingInterval": 30,
"defaultHadoopProperties": {
"mapred.map.output.compression.type": "BLOCK",
"mapreduce.map.output.compress": "true",
"mapreduce.output.fileoutputformat.compress": "true",
"mapreduce.output.fileoutputformat.compress.type": "BLOCK",
"mapreduce.input.fileinputformat.split.minsize": "134217728",
"mapreduce.job.max.split.locations": "20",
"mapreduce.input.fileinputformat.split.maxsize": "5368709120",
"yarn.app.mapreduce.am.staging-dir": "/user"
},
"hadoopDistributionSpecificProperties": {},
"customProperties": {
"das.execution-framework": "Tez"
},
"defaultLogSeverity": "DEFAULT",
"loggingCustomization": {}
}
No IAM usage and determining the cluster by given cluster name: {
"s3Bucket": "s3://datameer.emr.test",
"s3StorageFolder": "/user/datameer",
"useEc2IamRole": false,
"awsS3AccessKey": "ASFASDFasdfaskldjfhWER",
"awsS3AccessSecret": "876876mnabsfdafw2342+SIGNPp",
"useYarnResourceManagerHostname": true,
"yarnResourceManagerHostname": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com",
"yarnResourceManagerAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8032",
"yarnResourceManagerWebappAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8088",
"yarnResourceManagerSchedulerAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8030",
"yarnApplicationClasspath": "/etc/hadoop/conf:/usr/lib/hadoop-hdfs/./:/usr/lib/hadoop-hdfs/.//*:/usr/lib/hadoop-hdfs/lib/*:/usr/lib/hadoop-lzo/lib/*:/usr/lib/hadoop-mapreduce/.//*:/usr/lib/hadoop-mapreduce/lib/*:/usr/lib/hadoop-yarn/.//*:/usr/lib/hadoop-yarn/lib/*:/usr/lib/hadoop/.//*:/usr/lib/hadoop/lib/*:/usr/share/aws/aws-java-sdk/*:/usr/share/aws/emr/cloudwatch-sink/lib/*:/usr/share/aws/emr/ddb/lib/emr-ddb-hadoop.jar:/usr/share/aws/emr/emrfs/auxlib/*:/usr/share/aws/emr/emrfs/conf:/usr/share/aws/emr/emrfs/lib/*:/usr/share/aws/emr/goodies/lib/emr-hadoop-goodies.jar:/usr/share/aws/emr/kinesis/lib/emr-kinesis-hadoop.jar:/usr/share/aws/emr/security/conf:/usr/share/aws/emr/security/lib/*",
"maxConcurrentJobs": 25,
"emrPollingInterval": 30,
"defaultHadoopProperties": {
"mapred.map.output.compression.type": "BLOCK",
"mapreduce.map.output.compress": "true",
"mapreduce.output.fileoutputformat.compress": "true",
"mapreduce.output.fileoutputformat.compress.type": "BLOCK",
"mapreduce.input.fileinputformat.split.minsize": "134217728",
"mapreduce.job.max.split.locations": "20",
"mapreduce.input.fileinputformat.split.maxsize": "5368709120",
"yarn.app.mapreduce.am.staging-dir": "/user"
},
"hadoopDistributionSpecificProperties": {},
"customProperties": {
"das.execution-framework": "Tez"
},
"defaultLogSeverity": "DEFAULT",
"loggingCustomization": {}
}
IAM usage and determining the cluster by given hostname: {
"s3Bucket": "s3://datameer.emr.test",
"s3StorageFolder": "/user/datameer",
"useEc2IamRole": true,
"useYarnResourceManagerHostname": true,
"yarnResourceManagerHostname": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com",
"yarnResourceManagerAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8032",
"yarnResourceManagerWebappAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8088",
"yarnResourceManagerSchedulerAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8030",
"yarnApplicationClasspath": "/etc/hadoop/conf:/usr/lib/hadoop-hdfs/./:/usr/lib/hadoop-hdfs/.//*:/usr/lib/hadoop-hdfs/lib/*:/usr/lib/hadoop-lzo/lib/*:/usr/lib/hadoop-mapreduce/.//*:/usr/lib/hadoop-mapreduce/lib/*:/usr/lib/hadoop-yarn/.//*:/usr/lib/hadoop-yarn/lib/*:/usr/lib/hadoop/.//*:/usr/lib/hadoop/lib/*:/usr/share/aws/aws-java-sdk/*:/usr/share/aws/emr/cloudwatch-sink/lib/*:/usr/share/aws/emr/ddb/lib/emr-ddb-hadoop.jar:/usr/share/aws/emr/emrfs/auxlib/*:/usr/share/aws/emr/emrfs/conf:/usr/share/aws/emr/emrfs/lib/*:/usr/share/aws/emr/goodies/lib/emr-hadoop-goodies.jar:/usr/share/aws/emr/kinesis/lib/emr-kinesis-hadoop.jar:/usr/share/aws/emr/security/conf:/usr/share/aws/emr/security/lib/*",
"maxConcurrentJobs": 25,
"emrPollingInterval": 30,
"defaultHadoopProperties": {
"mapred.map.output.compression.type": "BLOCK",
"mapreduce.map.output.compress": "true",
"mapreduce.output.fileoutputformat.compress": "true",
"mapreduce.output.fileoutputformat.compress.type": "BLOCK",
"mapreduce.input.fileinputformat.split.minsize": "134217728",
"mapreduce.job.max.split.locations": "20",
"mapreduce.input.fileinputformat.split.maxsize": "5368709120",
"yarn.app.mapreduce.am.staging-dir": "/user"
},
"hadoopDistributionSpecificProperties": {},
"customProperties": {
"das.execution-framework": "Tez"
},
"defaultLogSeverity": "DEFAULT",
"loggingCustomization": {}
}
|
Update the Current EMR Configuration
| Request Method | PUT |
| Description | Updates the EMR configuration. Updating the configuration overrides the current configuration. INFO There is no way to replace a single configuration parameter. |
| URI Syntax | curl -u admin:admin -X PUT -d @test.json 'localhost:8088/api/cluster-configuration/emr' -v -H "Content-Type:application/json" |
| Example Response | No IAM usage and determining the cluster by given hostname: {
"s3Bucket": "s3://datameer.emr.test",
"s3StorageFolder": "/user/datameer",
"useEc2IamRole": false,
"useYarnResourceManagerHostname": true,
"yarnResourceManagerHostname": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com",
"awsS3AccessKey": "ASFASDFasdfaskldjfhWER",
"awsS3AccessSecret": "876876mnabsfdafw2342+SIGNPp",
"yarnApplicationClasspath": "/etc/hadoop/conf:/usr/lib/hadoop-hdfs/./:/usr/lib/hadoop-hdfs/.//*:/usr/lib/hadoop-hdfs/lib/*:/usr/lib/hadoop-lzo/lib/*:/usr/lib/hadoop-mapreduce/.//*:/usr/lib/hadoop-mapreduce/lib/*:/usr/lib/hadoop-yarn/.//*:/usr/lib/hadoop-yarn/lib/*:/usr/lib/hadoop/.//*:/usr/lib/hadoop/lib/*:/usr/share/aws/aws-java-sdk/*:/usr/share/aws/emr/cloudwatch-sink/lib/*:/usr/share/aws/emr/ddb/lib/emr-ddb-hadoop.jar:/usr/share/aws/emr/emrfs/auxlib/*:/usr/share/aws/emr/emrfs/conf:/usr/share/aws/emr/emrfs/lib/*:/usr/share/aws/emr/goodies/lib/emr-hadoop-goodies.jar:/usr/share/aws/emr/kinesis/lib/emr-kinesis-hadoop.jar:/usr/share/aws/emr/security/conf:/usr/share/aws/emr/security/lib/*",
"maxConcurrentJobs": 25,
"emrPollingInterval": 30,
"defaultHadoopProperties": {
"mapred.map.output.compression.type": "BLOCK",
"mapreduce.map.output.compress": "true",
"mapreduce.output.fileoutputformat.compress": "true",
"mapreduce.output.fileoutputformat.compress.type": "BLOCK",
"mapreduce.input.fileinputformat.split.minsize": "134217728",
"mapreduce.job.max.split.locations": "20",
"mapreduce.input.fileinputformat.split.maxsize": "5368709120",
"yarn.app.mapreduce.am.staging-dir": "/user"
},
"hadoopDistributionSpecificProperties": {},
"customProperties": {
"das.execution-framework": "Tez"
},
"defaultLogSeverity": "DEFAULT",
"loggingCustomization": {}
}
No IAM usage and determining the cluster by given Cluster name: {
"s3Bucket": "s3://datameer.emr.test",
"s3StorageFolder": "/user/datameer",
"useEc2IamRole": false,
"awsS3AccessKey": "ASFASDFasdfaskldjfhWER",
"awsS3AccessSecret": "876876mnabsfdafw2342+SIGNPp",
"useYarnResourceManagerHostname": true,
"yarnResourceManagerHostname": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com",
"yarnResourceManagerAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8032",
"yarnResourceManagerWebappAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8088",
"yarnResourceManagerSchedulerAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8030",
"yarnApplicationClasspath": "/etc/hadoop/conf:/usr/lib/hadoop-hdfs/./:/usr/lib/hadoop-hdfs/.//*:/usr/lib/hadoop-hdfs/lib/*:/usr/lib/hadoop-lzo/lib/*:/usr/lib/hadoop-mapreduce/.//*:/usr/lib/hadoop-mapreduce/lib/*:/usr/lib/hadoop-yarn/.//*:/usr/lib/hadoop-yarn/lib/*:/usr/lib/hadoop/.//*:/usr/lib/hadoop/lib/*:/usr/share/aws/aws-java-sdk/*:/usr/share/aws/emr/cloudwatch-sink/lib/*:/usr/share/aws/emr/ddb/lib/emr-ddb-hadoop.jar:/usr/share/aws/emr/emrfs/auxlib/*:/usr/share/aws/emr/emrfs/conf:/usr/share/aws/emr/emrfs/lib/*:/usr/share/aws/emr/goodies/lib/emr-hadoop-goodies.jar:/usr/share/aws/emr/kinesis/lib/emr-kinesis-hadoop.jar:/usr/share/aws/emr/security/conf:/usr/share/aws/emr/security/lib/*",
"maxConcurrentJobs": 25,
"emrPollingInterval": 30,
"defaultHadoopProperties": {
"mapred.map.output.compression.type": "BLOCK",
"mapreduce.map.output.compress": "true",
"mapreduce.output.fileoutputformat.compress": "true",
"mapreduce.output.fileoutputformat.compress.type": "BLOCK",
"mapreduce.input.fileinputformat.split.minsize": "134217728",
"mapreduce.job.max.split.locations": "20",
"mapreduce.input.fileinputformat.split.maxsize": "5368709120",
"yarn.app.mapreduce.am.staging-dir": "/user"
},
"hadoopDistributionSpecificProperties": {},
"customProperties": {
"das.execution-framework": "Tez"
},
"defaultLogSeverity": "DEFAULT",
"loggingCustomization": {}
}
IAM usage and determining the cluster by given hostname: {
"s3Bucket": "s3://datameer.emr.test",
"s3StorageFolder": "/user/datameer",
"useEc2IamRole": true,
"useYarnResourceManagerHostname": true,
"yarnResourceManagerHostname": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com",
"yarnApplicationClasspath": "/etc/hadoop/conf:/usr/lib/hadoop-hdfs/./:/usr/lib/hadoop-hdfs/.//*:/usr/lib/hadoop-hdfs/lib/*:/usr/lib/hadoop-lzo/lib/*:/usr/lib/hadoop-mapreduce/.//*:/usr/lib/hadoop-mapreduce/lib/*:/usr/lib/hadoop-yarn/.//*:/usr/lib/hadoop-yarn/lib/*:/usr/lib/hadoop/.//*:/usr/lib/hadoop/lib/*:/usr/share/aws/aws-java-sdk/*:/usr/share/aws/emr/cloudwatch-sink/lib/*:/usr/share/aws/emr/ddb/lib/emr-ddb-hadoop.jar:/usr/share/aws/emr/emrfs/auxlib/*:/usr/share/aws/emr/emrfs/conf:/usr/share/aws/emr/emrfs/lib/*:/usr/share/aws/emr/goodies/lib/emr-hadoop-goodies.jar:/usr/share/aws/emr/kinesis/lib/emr-kinesis-hadoop.jar:/usr/share/aws/emr/security/conf:/usr/share/aws/emr/security/lib/*",
"maxConcurrentJobs": 25,
"emrPollingInterval": 30,
"defaultHadoopProperties": {
"mapred.map.output.compression.type": "BLOCK",
"mapreduce.map.output.compress": "true",
"mapreduce.output.fileoutputformat.compress": "true",
"mapreduce.output.fileoutputformat.compress.type": "BLOCK",
"mapreduce.input.fileinputformat.split.minsize": "134217728",
"mapreduce.job.max.split.locations": "20",
"mapreduce.input.fileinputformat.split.maxsize": "5368709120",
"yarn.app.mapreduce.am.staging-dir": "/user"
},
"hadoopDistributionSpecificProperties": {},
"customProperties": {
"das.execution-framework": "Tez"
},
"defaultLogSeverity": "DEFAULT",
"loggingCustomization": {}
}
|