REST API Elastic Map Reduce

View the current EMR Configuration

Request Method	GET
Description	Returns the EMR configuration.
URI Syntax	curl -u 'admin:admin' -X GET 'localhost:8088/api/cluster-configuration/emr' > test.json
Example Response	No IAM usage and determining the cluster by given hostname: Expand source { "s3Bucket": "s3://datameer.emr.test", "s3StorageFolder": "/user/datameer", "useEc2IamRole": false, "useYarnResourceManagerHostname": true, "yarnResourceManagerHostname": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com", "awsS3AccessKey": "ASFASDFasdfaskldjfhWER", "awsS3AccessSecret": "876876mnabsfdafw2342+SIGNPp", "yarnResourceManagerAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8032", "yarnResourceManagerWebappAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8088", "yarnResourceManagerSchedulerAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8030", "yarnApplicationClasspath": "/etc/hadoop/conf:/usr/lib/hadoop-hdfs/./:/usr/lib/hadoop-hdfs/.//:/usr/lib/hadoop-hdfs/lib/:/usr/lib/hadoop-lzo/lib/:/usr/lib/hadoop-mapreduce/.//:/usr/lib/hadoop-mapreduce/lib/:/usr/lib/hadoop-yarn/.//:/usr/lib/hadoop-yarn/lib/:/usr/lib/hadoop/.//:/usr/lib/hadoop/lib/:/usr/share/aws/aws-java-sdk/:/usr/share/aws/emr/cloudwatch-sink/lib/:/usr/share/aws/emr/ddb/lib/emr-ddb-hadoop.jar:/usr/share/aws/emr/emrfs/auxlib/:/usr/share/aws/emr/emrfs/conf:/usr/share/aws/emr/emrfs/lib/:/usr/share/aws/emr/goodies/lib/emr-hadoop-goodies.jar:/usr/share/aws/emr/kinesis/lib/emr-kinesis-hadoop.jar:/usr/share/aws/emr/security/conf:/usr/share/aws/emr/security/lib/", "maxConcurrentJobs": 25, "emrPollingInterval": 30, "defaultHadoopProperties": { "mapred.map.output.compression.type": "BLOCK", "mapreduce.map.output.compress": "true", "mapreduce.output.fileoutputformat.compress": "true", "mapreduce.output.fileoutputformat.compress.type": "BLOCK", "mapreduce.input.fileinputformat.split.minsize": "134217728", "mapreduce.job.max.split.locations": "20", "mapreduce.input.fileinputformat.split.maxsize": "5368709120", "yarn.app.mapreduce.am.staging-dir": "/user" }, "hadoopDistributionSpecificProperties": {}, "customProperties": { "das.execution-framework": "Tez" }, "defaultLogSeverity": "DEFAULT", "loggingCustomization": {} } No IAM usage and determining the cluster by given cluster name: Expand source { "s3Bucket": "s3://datameer.emr.test", "s3StorageFolder": "/user/datameer", "useEc2IamRole": false, "awsS3AccessKey": "ASFASDFasdfaskldjfhWER", "awsS3AccessSecret": "876876mnabsfdafw2342+SIGNPp", "useYarnResourceManagerHostname": true, "yarnResourceManagerHostname": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com", "yarnResourceManagerAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8032", "yarnResourceManagerWebappAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8088", "yarnResourceManagerSchedulerAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8030", "yarnApplicationClasspath": "/etc/hadoop/conf:/usr/lib/hadoop-hdfs/./:/usr/lib/hadoop-hdfs/.//:/usr/lib/hadoop-hdfs/lib/:/usr/lib/hadoop-lzo/lib/:/usr/lib/hadoop-mapreduce/.//:/usr/lib/hadoop-mapreduce/lib/:/usr/lib/hadoop-yarn/.//:/usr/lib/hadoop-yarn/lib/:/usr/lib/hadoop/.//:/usr/lib/hadoop/lib/:/usr/share/aws/aws-java-sdk/:/usr/share/aws/emr/cloudwatch-sink/lib/:/usr/share/aws/emr/ddb/lib/emr-ddb-hadoop.jar:/usr/share/aws/emr/emrfs/auxlib/:/usr/share/aws/emr/emrfs/conf:/usr/share/aws/emr/emrfs/lib/:/usr/share/aws/emr/goodies/lib/emr-hadoop-goodies.jar:/usr/share/aws/emr/kinesis/lib/emr-kinesis-hadoop.jar:/usr/share/aws/emr/security/conf:/usr/share/aws/emr/security/lib/", "maxConcurrentJobs": 25, "emrPollingInterval": 30, "defaultHadoopProperties": { "mapred.map.output.compression.type": "BLOCK", "mapreduce.map.output.compress": "true", "mapreduce.output.fileoutputformat.compress": "true", "mapreduce.output.fileoutputformat.compress.type": "BLOCK", "mapreduce.input.fileinputformat.split.minsize": "134217728", "mapreduce.job.max.split.locations": "20", "mapreduce.input.fileinputformat.split.maxsize": "5368709120", "yarn.app.mapreduce.am.staging-dir": "/user" }, "hadoopDistributionSpecificProperties": {}, "customProperties": { "das.execution-framework": "Tez" }, "defaultLogSeverity": "DEFAULT", "loggingCustomization": {} } IAM usage and determining the cluster by given hostname: Expand source { "s3Bucket": "s3://datameer.emr.test", "s3StorageFolder": "/user/datameer", "useEc2IamRole": true, "useYarnResourceManagerHostname": true, "yarnResourceManagerHostname": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com", "yarnResourceManagerAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8032", "yarnResourceManagerWebappAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8088", "yarnResourceManagerSchedulerAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8030", "yarnApplicationClasspath": "/etc/hadoop/conf:/usr/lib/hadoop-hdfs/./:/usr/lib/hadoop-hdfs/.//:/usr/lib/hadoop-hdfs/lib/:/usr/lib/hadoop-lzo/lib/:/usr/lib/hadoop-mapreduce/.//:/usr/lib/hadoop-mapreduce/lib/:/usr/lib/hadoop-yarn/.//:/usr/lib/hadoop-yarn/lib/:/usr/lib/hadoop/.//:/usr/lib/hadoop/lib/:/usr/share/aws/aws-java-sdk/:/usr/share/aws/emr/cloudwatch-sink/lib/:/usr/share/aws/emr/ddb/lib/emr-ddb-hadoop.jar:/usr/share/aws/emr/emrfs/auxlib/:/usr/share/aws/emr/emrfs/conf:/usr/share/aws/emr/emrfs/lib/:/usr/share/aws/emr/goodies/lib/emr-hadoop-goodies.jar:/usr/share/aws/emr/kinesis/lib/emr-kinesis-hadoop.jar:/usr/share/aws/emr/security/conf:/usr/share/aws/emr/security/lib/", "maxConcurrentJobs": 25, "emrPollingInterval": 30, "defaultHadoopProperties": { "mapred.map.output.compression.type": "BLOCK", "mapreduce.map.output.compress": "true", "mapreduce.output.fileoutputformat.compress": "true", "mapreduce.output.fileoutputformat.compress.type": "BLOCK", "mapreduce.input.fileinputformat.split.minsize": "134217728", "mapreduce.job.max.split.locations": "20", "mapreduce.input.fileinputformat.split.maxsize": "5368709120", "yarn.app.mapreduce.am.staging-dir": "/user" }, "hadoopDistributionSpecificProperties": {}, "customProperties": { "das.execution-framework": "Tez" }, "defaultLogSeverity": "DEFAULT", "loggingCustomization": {} }

Update the Current EMR Configuration

Request Method	PUT
Description	Updates the EMR configuration. Updating the configuration overrides the current configuration. INFO There is no way to replace a single configuration parameter.
URI Syntax	curl -u 'admin:admin' -X PUT -d @test.json 'localhost:8088/api/cluster-configuration/emr' -v -H "Content-Type:application/json"
Example Response	No IAM usage and determining the cluster by given hostname: Expand source { "s3Bucket": "s3://datameer.emr.test", "s3StorageFolder": "/user/datameer", "useEc2IamRole": false, "useYarnResourceManagerHostname": true, "yarnResourceManagerHostname": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com", "awsS3AccessKey": "ASFASDFasdfaskldjfhWER", "awsS3AccessSecret": "876876mnabsfdafw2342+SIGNPp", "yarnApplicationClasspath": "/etc/hadoop/conf:/usr/lib/hadoop-hdfs/./:/usr/lib/hadoop-hdfs/.//:/usr/lib/hadoop-hdfs/lib/:/usr/lib/hadoop-lzo/lib/:/usr/lib/hadoop-mapreduce/.//:/usr/lib/hadoop-mapreduce/lib/:/usr/lib/hadoop-yarn/.//:/usr/lib/hadoop-yarn/lib/:/usr/lib/hadoop/.//:/usr/lib/hadoop/lib/:/usr/share/aws/aws-java-sdk/:/usr/share/aws/emr/cloudwatch-sink/lib/:/usr/share/aws/emr/ddb/lib/emr-ddb-hadoop.jar:/usr/share/aws/emr/emrfs/auxlib/:/usr/share/aws/emr/emrfs/conf:/usr/share/aws/emr/emrfs/lib/:/usr/share/aws/emr/goodies/lib/emr-hadoop-goodies.jar:/usr/share/aws/emr/kinesis/lib/emr-kinesis-hadoop.jar:/usr/share/aws/emr/security/conf:/usr/share/aws/emr/security/lib/", "maxConcurrentJobs": 25, "emrPollingInterval": 30, "defaultHadoopProperties": { "mapred.map.output.compression.type": "BLOCK", "mapreduce.map.output.compress": "true", "mapreduce.output.fileoutputformat.compress": "true", "mapreduce.output.fileoutputformat.compress.type": "BLOCK", "mapreduce.input.fileinputformat.split.minsize": "134217728", "mapreduce.job.max.split.locations": "20", "mapreduce.input.fileinputformat.split.maxsize": "5368709120", "yarn.app.mapreduce.am.staging-dir": "/user" }, "hadoopDistributionSpecificProperties": {}, "customProperties": { "das.execution-framework": "Tez" }, "defaultLogSeverity": "DEFAULT", "loggingCustomization": {} } No IAM usage and determining the cluster by given Cluster name: Expand source { "s3Bucket": "s3://datameer.emr.test", "s3StorageFolder": "/user/datameer", "useEc2IamRole": false, "awsS3AccessKey": "ASFASDFasdfaskldjfhWER", "awsS3AccessSecret": "876876mnabsfdafw2342+SIGNPp", "useYarnResourceManagerHostname": true, "yarnResourceManagerHostname": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com", "yarnResourceManagerAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8032", "yarnResourceManagerWebappAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8088", "yarnResourceManagerSchedulerAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8030", "yarnApplicationClasspath": "/etc/hadoop/conf:/usr/lib/hadoop-hdfs/./:/usr/lib/hadoop-hdfs/.//:/usr/lib/hadoop-hdfs/lib/:/usr/lib/hadoop-lzo/lib/:/usr/lib/hadoop-mapreduce/.//:/usr/lib/hadoop-mapreduce/lib/:/usr/lib/hadoop-yarn/.//:/usr/lib/hadoop-yarn/lib/:/usr/lib/hadoop/.//:/usr/lib/hadoop/lib/:/usr/share/aws/aws-java-sdk/:/usr/share/aws/emr/cloudwatch-sink/lib/:/usr/share/aws/emr/ddb/lib/emr-ddb-hadoop.jar:/usr/share/aws/emr/emrfs/auxlib/:/usr/share/aws/emr/emrfs/conf:/usr/share/aws/emr/emrfs/lib/:/usr/share/aws/emr/goodies/lib/emr-hadoop-goodies.jar:/usr/share/aws/emr/kinesis/lib/emr-kinesis-hadoop.jar:/usr/share/aws/emr/security/conf:/usr/share/aws/emr/security/lib/", "maxConcurrentJobs": 25, "emrPollingInterval": 30, "defaultHadoopProperties": { "mapred.map.output.compression.type": "BLOCK", "mapreduce.map.output.compress": "true", "mapreduce.output.fileoutputformat.compress": "true", "mapreduce.output.fileoutputformat.compress.type": "BLOCK", "mapreduce.input.fileinputformat.split.minsize": "134217728", "mapreduce.job.max.split.locations": "20", "mapreduce.input.fileinputformat.split.maxsize": "5368709120", "yarn.app.mapreduce.am.staging-dir": "/user" }, "hadoopDistributionSpecificProperties": {}, "customProperties": { "das.execution-framework": "Tez" }, "defaultLogSeverity": "DEFAULT", "loggingCustomization": {} } IAM usage and determining the cluster by given hostname: Expand source { "s3Bucket": "s3://datameer.emr.test", "s3StorageFolder": "/user/datameer", "useEc2IamRole": true, "useYarnResourceManagerHostname": true, "yarnResourceManagerHostname": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com", "yarnApplicationClasspath": "/etc/hadoop/conf:/usr/lib/hadoop-hdfs/./:/usr/lib/hadoop-hdfs/.//:/usr/lib/hadoop-hdfs/lib/:/usr/lib/hadoop-lzo/lib/:/usr/lib/hadoop-mapreduce/.//:/usr/lib/hadoop-mapreduce/lib/:/usr/lib/hadoop-yarn/.//:/usr/lib/hadoop-yarn/lib/:/usr/lib/hadoop/.//:/usr/lib/hadoop/lib/:/usr/share/aws/aws-java-sdk/:/usr/share/aws/emr/cloudwatch-sink/lib/:/usr/share/aws/emr/ddb/lib/emr-ddb-hadoop.jar:/usr/share/aws/emr/emrfs/auxlib/:/usr/share/aws/emr/emrfs/conf:/usr/share/aws/emr/emrfs/lib/:/usr/share/aws/emr/goodies/lib/emr-hadoop-goodies.jar:/usr/share/aws/emr/kinesis/lib/emr-kinesis-hadoop.jar:/usr/share/aws/emr/security/conf:/usr/share/aws/emr/security/lib/", "maxConcurrentJobs": 25, "emrPollingInterval": 30, "defaultHadoopProperties": { "mapred.map.output.compression.type": "BLOCK", "mapreduce.map.output.compress": "true", "mapreduce.output.fileoutputformat.compress": "true", "mapreduce.output.fileoutputformat.compress.type": "BLOCK", "mapreduce.input.fileinputformat.split.minsize": "134217728", "mapreduce.job.max.split.locations": "20", "mapreduce.input.fileinputformat.split.maxsize": "5368709120", "yarn.app.mapreduce.am.staging-dir": "/user" }, "hadoopDistributionSpecificProperties": {}, "customProperties": { "das.execution-framework": "Tez" }, "defaultLogSeverity": "DEFAULT", "loggingCustomization": {} }