REST API EMR

Owned by Charles Bishop (Deactivated)
Jan 30, 2018

View the current EMR configuration

Request method - GET

Description

Returns the EMR configuration.

URI syntax

curl -u admin:admin -X GET 'localhost:8088/api/cluster-configuration/emr' > test.json

Example responses

No IAM usage and determining the cluster by given hostname.

{
  "s3Bucket": "s3://datameer.emr.test",
  "s3StorageFolder": "/user/datameer",
  "useEc2IamRole": false,
  "useYarnResourceManagerHostname": true,
  "yarnResourceManagerHostname": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com",
  "awsS3AccessKey": "ASFASDFasdfaskldjfhWER",
  "awsS3AccessSecret": "876876mnabsfdafw2342+SIGNPp",
  "yarnResourceManagerAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8032",
  "yarnResourceManagerWebappAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8088",
  "yarnResourceManagerSchedulerAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8030",
  "yarnApplicationClasspath": "/etc/hadoop/conf:/usr/lib/hadoop-hdfs/./:/usr/lib/hadoop-hdfs/.//*:/usr/lib/hadoop-hdfs/lib/*:/usr/lib/hadoop-lzo/lib/*:/usr/lib/hadoop-mapreduce/.//*:/usr/lib/hadoop-mapreduce/lib/*:/usr/lib/hadoop-yarn/.//*:/usr/lib/hadoop-yarn/lib/*:/usr/lib/hadoop/.//*:/usr/lib/hadoop/lib/*:/usr/share/aws/aws-java-sdk/*:/usr/share/aws/emr/cloudwatch-sink/lib/*:/usr/share/aws/emr/ddb/lib/emr-ddb-hadoop.jar:/usr/share/aws/emr/emrfs/auxlib/*:/usr/share/aws/emr/emrfs/conf:/usr/share/aws/emr/emrfs/lib/*:/usr/share/aws/emr/goodies/lib/emr-hadoop-goodies.jar:/usr/share/aws/emr/kinesis/lib/emr-kinesis-hadoop.jar:/usr/share/aws/emr/security/conf:/usr/share/aws/emr/security/lib/*",
  "maxConcurrentJobs": 25,
  "emrPollingInterval": 30,
  "defaultHadoopProperties": {
    "mapred.map.output.compression.type": "BLOCK",
    "mapreduce.map.output.compress": "true",
    "mapreduce.output.fileoutputformat.compress": "true",
    "mapreduce.output.fileoutputformat.compress.type": "BLOCK",
    "mapreduce.input.fileinputformat.split.minsize": "134217728",
    "mapreduce.job.max.split.locations": "20",
    "mapreduce.input.fileinputformat.split.maxsize": "5368709120",
    "yarn.app.mapreduce.am.staging-dir": "/user"
  },
  "hadoopDistributionSpecificProperties": {},
  "customProperties": {
    "das.execution-framework": "Tez"
  },
  "defaultLogSeverity": "DEFAULT",
  "loggingCustomization": {}
}


No IAM usage and determining the cluster by given cluster name.

{
  "s3Bucket": "s3://datameer.emr.test",
  "s3StorageFolder": "/user/datameer",
  "useEc2IamRole": false,
  "awsS3AccessKey": "ASFASDFasdfaskldjfhWER",
  "awsS3AccessSecret": "876876mnabsfdafw2342+SIGNPp",
  "useYarnResourceManagerHostname": true,
  "yarnResourceManagerHostname": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com",
  "yarnResourceManagerAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8032",
  "yarnResourceManagerWebappAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8088",
  "yarnResourceManagerSchedulerAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8030",
  "yarnApplicationClasspath": "/etc/hadoop/conf:/usr/lib/hadoop-hdfs/./:/usr/lib/hadoop-hdfs/.//*:/usr/lib/hadoop-hdfs/lib/*:/usr/lib/hadoop-lzo/lib/*:/usr/lib/hadoop-mapreduce/.//*:/usr/lib/hadoop-mapreduce/lib/*:/usr/lib/hadoop-yarn/.//*:/usr/lib/hadoop-yarn/lib/*:/usr/lib/hadoop/.//*:/usr/lib/hadoop/lib/*:/usr/share/aws/aws-java-sdk/*:/usr/share/aws/emr/cloudwatch-sink/lib/*:/usr/share/aws/emr/ddb/lib/emr-ddb-hadoop.jar:/usr/share/aws/emr/emrfs/auxlib/*:/usr/share/aws/emr/emrfs/conf:/usr/share/aws/emr/emrfs/lib/*:/usr/share/aws/emr/goodies/lib/emr-hadoop-goodies.jar:/usr/share/aws/emr/kinesis/lib/emr-kinesis-hadoop.jar:/usr/share/aws/emr/security/conf:/usr/share/aws/emr/security/lib/*",
  "maxConcurrentJobs": 25,
  "emrPollingInterval": 30,
  "defaultHadoopProperties": {
    "mapred.map.output.compression.type": "BLOCK",
    "mapreduce.map.output.compress": "true",
    "mapreduce.output.fileoutputformat.compress": "true",
    "mapreduce.output.fileoutputformat.compress.type": "BLOCK",
    "mapreduce.input.fileinputformat.split.minsize": "134217728",
    "mapreduce.job.max.split.locations": "20",
    "mapreduce.input.fileinputformat.split.maxsize": "5368709120",
    "yarn.app.mapreduce.am.staging-dir": "/user"
  },
  "hadoopDistributionSpecificProperties": {},
  "customProperties": {
    "das.execution-framework": "Tez"
  },
  "defaultLogSeverity": "DEFAULT",
  "loggingCustomization": {}
}


IAM usage and determining the cluster by given hostname.

{
  "s3Bucket": "s3://datameer.emr.test",
  "s3StorageFolder": "/user/datameer",
  "useEc2IamRole": true,
  "useYarnResourceManagerHostname": true,
  "yarnResourceManagerHostname": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com",
  "yarnResourceManagerAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8032",
  "yarnResourceManagerWebappAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8088",
  "yarnResourceManagerSchedulerAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8030",
  "yarnApplicationClasspath": "/etc/hadoop/conf:/usr/lib/hadoop-hdfs/./:/usr/lib/hadoop-hdfs/.//*:/usr/lib/hadoop-hdfs/lib/*:/usr/lib/hadoop-lzo/lib/*:/usr/lib/hadoop-mapreduce/.//*:/usr/lib/hadoop-mapreduce/lib/*:/usr/lib/hadoop-yarn/.//*:/usr/lib/hadoop-yarn/lib/*:/usr/lib/hadoop/.//*:/usr/lib/hadoop/lib/*:/usr/share/aws/aws-java-sdk/*:/usr/share/aws/emr/cloudwatch-sink/lib/*:/usr/share/aws/emr/ddb/lib/emr-ddb-hadoop.jar:/usr/share/aws/emr/emrfs/auxlib/*:/usr/share/aws/emr/emrfs/conf:/usr/share/aws/emr/emrfs/lib/*:/usr/share/aws/emr/goodies/lib/emr-hadoop-goodies.jar:/usr/share/aws/emr/kinesis/lib/emr-kinesis-hadoop.jar:/usr/share/aws/emr/security/conf:/usr/share/aws/emr/security/lib/*",
  "maxConcurrentJobs": 25,
  "emrPollingInterval": 30,
  "defaultHadoopProperties": {
    "mapred.map.output.compression.type": "BLOCK",
    "mapreduce.map.output.compress": "true",
    "mapreduce.output.fileoutputformat.compress": "true",
    "mapreduce.output.fileoutputformat.compress.type": "BLOCK",
    "mapreduce.input.fileinputformat.split.minsize": "134217728",
    "mapreduce.job.max.split.locations": "20",
    "mapreduce.input.fileinputformat.split.maxsize": "5368709120",
    "yarn.app.mapreduce.am.staging-dir": "/user"
  },
  "hadoopDistributionSpecificProperties": {},
  "customProperties": {
    "das.execution-framework": "Tez"
  },
  "defaultLogSeverity": "DEFAULT",
  "loggingCustomization": {}
}

Update the current EMR configuration

Request method - PUT

Description

Updates the EMR configuration. Updating the configuration overrides the current configuration. There is no way to replace a single configuration parameter.

URI syntax

curl -u admin:admin -X PUT -d @test.json 'localhost:8088/api/cluster-configuration/emr' -v -H "Content-Type:application/json"

Example responses

No IAM usage and determining the cluster by given hostname.

{
  "s3Bucket": "s3://datameer.emr.test",
  "s3StorageFolder": "/user/datameer",
  "useEc2IamRole": false,
  "useYarnResourceManagerHostname": true,
  "yarnResourceManagerHostname": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com",
  "awsS3AccessKey": "ASFASDFasdfaskldjfhWER",
  "awsS3AccessSecret": "876876mnabsfdafw2342+SIGNPp",
  "yarnApplicationClasspath": "/etc/hadoop/conf:/usr/lib/hadoop-hdfs/./:/usr/lib/hadoop-hdfs/.//*:/usr/lib/hadoop-hdfs/lib/*:/usr/lib/hadoop-lzo/lib/*:/usr/lib/hadoop-mapreduce/.//*:/usr/lib/hadoop-mapreduce/lib/*:/usr/lib/hadoop-yarn/.//*:/usr/lib/hadoop-yarn/lib/*:/usr/lib/hadoop/.//*:/usr/lib/hadoop/lib/*:/usr/share/aws/aws-java-sdk/*:/usr/share/aws/emr/cloudwatch-sink/lib/*:/usr/share/aws/emr/ddb/lib/emr-ddb-hadoop.jar:/usr/share/aws/emr/emrfs/auxlib/*:/usr/share/aws/emr/emrfs/conf:/usr/share/aws/emr/emrfs/lib/*:/usr/share/aws/emr/goodies/lib/emr-hadoop-goodies.jar:/usr/share/aws/emr/kinesis/lib/emr-kinesis-hadoop.jar:/usr/share/aws/emr/security/conf:/usr/share/aws/emr/security/lib/*",
  "maxConcurrentJobs": 25,
  "emrPollingInterval": 30,
  "defaultHadoopProperties": {
    "mapred.map.output.compression.type": "BLOCK",
    "mapreduce.map.output.compress": "true",
    "mapreduce.output.fileoutputformat.compress": "true",
    "mapreduce.output.fileoutputformat.compress.type": "BLOCK",
    "mapreduce.input.fileinputformat.split.minsize": "134217728",
    "mapreduce.job.max.split.locations": "20",
    "mapreduce.input.fileinputformat.split.maxsize": "5368709120",
    "yarn.app.mapreduce.am.staging-dir": "/user"
  },
  "hadoopDistributionSpecificProperties": {},
  "customProperties": {
    "das.execution-framework": "Tez"
  },
  "defaultLogSeverity": "DEFAULT",
  "loggingCustomization": {}
}


No IAM usage and determining the cluster by given Cluster name.

{
  "s3Bucket": "s3://datameer.emr.test",
  "s3StorageFolder": "/user/datameer",
  "useEc2IamRole": false,
  "awsS3AccessKey": "ASFASDFasdfaskldjfhWER",
  "awsS3AccessSecret": "876876mnabsfdafw2342+SIGNPp",
  "useYarnResourceManagerHostname": true,
  "yarnResourceManagerHostname": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com",
  "yarnResourceManagerAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8032",
  "yarnResourceManagerWebappAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8088",
  "yarnResourceManagerSchedulerAddress": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com:8030",
  "yarnApplicationClasspath": "/etc/hadoop/conf:/usr/lib/hadoop-hdfs/./:/usr/lib/hadoop-hdfs/.//*:/usr/lib/hadoop-hdfs/lib/*:/usr/lib/hadoop-lzo/lib/*:/usr/lib/hadoop-mapreduce/.//*:/usr/lib/hadoop-mapreduce/lib/*:/usr/lib/hadoop-yarn/.//*:/usr/lib/hadoop-yarn/lib/*:/usr/lib/hadoop/.//*:/usr/lib/hadoop/lib/*:/usr/share/aws/aws-java-sdk/*:/usr/share/aws/emr/cloudwatch-sink/lib/*:/usr/share/aws/emr/ddb/lib/emr-ddb-hadoop.jar:/usr/share/aws/emr/emrfs/auxlib/*:/usr/share/aws/emr/emrfs/conf:/usr/share/aws/emr/emrfs/lib/*:/usr/share/aws/emr/goodies/lib/emr-hadoop-goodies.jar:/usr/share/aws/emr/kinesis/lib/emr-kinesis-hadoop.jar:/usr/share/aws/emr/security/conf:/usr/share/aws/emr/security/lib/*",
  "maxConcurrentJobs": 25,
  "emrPollingInterval": 30,
  "defaultHadoopProperties": {
    "mapred.map.output.compression.type": "BLOCK",
    "mapreduce.map.output.compress": "true",
    "mapreduce.output.fileoutputformat.compress": "true",
    "mapreduce.output.fileoutputformat.compress.type": "BLOCK",
    "mapreduce.input.fileinputformat.split.minsize": "134217728",
    "mapreduce.job.max.split.locations": "20",
    "mapreduce.input.fileinputformat.split.maxsize": "5368709120",
    "yarn.app.mapreduce.am.staging-dir": "/user"
  },
  "hadoopDistributionSpecificProperties": {},
  "customProperties": {
    "das.execution-framework": "Tez"
  },
  "defaultLogSeverity": "DEFAULT",
  "loggingCustomization": {}
}


IAM usage and determining the cluster by given hostname.

{
  "s3Bucket": "s3://datameer.emr.test",
  "s3StorageFolder": "/user/datameer",
  "useEc2IamRole": true,
  "useYarnResourceManagerHostname": true,
  "yarnResourceManagerHostname": "ec2-34-134-34-134.eu-west-1.compute.amazonaws.com",
  "yarnApplicationClasspath": "/etc/hadoop/conf:/usr/lib/hadoop-hdfs/./:/usr/lib/hadoop-hdfs/.//*:/usr/lib/hadoop-hdfs/lib/*:/usr/lib/hadoop-lzo/lib/*:/usr/lib/hadoop-mapreduce/.//*:/usr/lib/hadoop-mapreduce/lib/*:/usr/lib/hadoop-yarn/.//*:/usr/lib/hadoop-yarn/lib/*:/usr/lib/hadoop/.//*:/usr/lib/hadoop/lib/*:/usr/share/aws/aws-java-sdk/*:/usr/share/aws/emr/cloudwatch-sink/lib/*:/usr/share/aws/emr/ddb/lib/emr-ddb-hadoop.jar:/usr/share/aws/emr/emrfs/auxlib/*:/usr/share/aws/emr/emrfs/conf:/usr/share/aws/emr/emrfs/lib/*:/usr/share/aws/emr/goodies/lib/emr-hadoop-goodies.jar:/usr/share/aws/emr/kinesis/lib/emr-kinesis-hadoop.jar:/usr/share/aws/emr/security/conf:/usr/share/aws/emr/security/lib/*",
  "maxConcurrentJobs": 25,
  "emrPollingInterval": 30,
  "defaultHadoopProperties": {
    "mapred.map.output.compression.type": "BLOCK",
    "mapreduce.map.output.compress": "true",
    "mapreduce.output.fileoutputformat.compress": "true",
    "mapreduce.output.fileoutputformat.compress.type": "BLOCK",
    "mapreduce.input.fileinputformat.split.minsize": "134217728",
    "mapreduce.job.max.split.locations": "20",
    "mapreduce.input.fileinputformat.split.maxsize": "5368709120",
    "yarn.app.mapreduce.am.staging-dir": "/user"
  },
  "hadoopDistributionSpecificProperties": {},
  "customProperties": {
    "das.execution-framework": "Tez"
  },
  "defaultLogSeverity": "DEFAULT",
  "loggingCustomization": {}
}