build.gradle

description = 'Elasticsearch for Apache Hadoop'

buildscript {
    repositories {
        mavenCentral()
        maven { url 'http://repo.spring.io/plugins-release' }
    }
    dependencies {
        classpath('org.springframework.build.gradle:propdeps-plugin:0.0.7')
    }
}

allprojects {
    repositories {
      mavenCentral()
      // cascading
      maven { url "http://conjars.org/repo" }
      // storm dependencies
      maven { url "http://clojars.org/repo" }
      maven { url 'http://repo.spring.io/plugins-release' }
      
      // Hive depends on JDO ec2 missing from Maven Central
      maven { url "http://www.datanucleus.org/downloads/maven2" }
      maven { url "http://oss.sonatype.org/content/groups/public/" }
    }

    apply plugin: "java"
    apply plugin: 'eclipse'
    apply plugin: 'idea'
    apply plugin: 'propdeps'
    apply plugin: 'propdeps-idea'
    apply plugin: 'propdeps-eclipse'
    // report plugins
    apply plugin: 'findbugs'
    apply plugin: 'pmd'
    apply plugin: 'jacoco'
    apply from: "$rootDir/dist.gradle"
}

println "Building version [$version]"

ext.hadoopClient = []
ext.hadoopDistro = project.hasProperty("distro") ? project.getProperty("distro") : "hadoopStable"

def java8 = org.gradle.api.JavaVersion.current().isJava8Compatible()

// Hadoop aliases
def hadoopStableVersion = hadoop12Version
def hadoopYarnVersion = hadoop2Version
def hadoopVersion = hadoopStableVersion 
def hadoopMsg;

switch (hadoopDistro) {

  // Hadoop YARN/2.0.x
  case "hadoopYarn":
    hadoopVersion = hadoopYarnVersion
    hadoopMsg = "Using Apache Hadoop YARN [$hadoopVersion]"
    hadoopClient = ["org.apache.hadoop:hadoop-client:$hadoopVersion"]

    break;
  // Hadoop 1.2.x
  case "hadoop12":
    hadoopVersion = hadoop12Version
    hadoopMsg =  "Using Apache Hadoop 1.2.x [$hadoopVersion]"
    hadoopClient = ["org.apache.hadoop:hadoop-streaming:$hadoopVersion",
              "org.apache.hadoop:hadoop-tools:$hadoopVersion"]

    break;
    
  // Hadoop 1.1.x
  case "hadoop11":
    hadoopVersion = hadoop11Version
    hadoopMsg = "Using Apache Hadoop 1.1.x [$hadoopVersion]"
    hadoopClient = ["org.apache.hadoop:hadoop-streaming:$hadoopVersion",
              "org.apache.hadoop:hadoop-tools:$hadoopVersion"]

    break;

  // Hadoop 1.0.x
  case "hadoop10":
    hadoopVersion = hadoop10Version
    hadoopMsg = "Using Apache Hadoop 1.0.x [$hadoopVersion]"
    hadoopClient = ["org.apache.hadoop:hadoop-streaming:$hadoopVersion",
              "org.apache.hadoop:hadoop-tools:$hadoopVersion"]

    break;

  default:
    hadoopMsg = "Using Apache Hadoop Stable [$hadoopVersion]"
    hadoopVersion = hadoopStableVersion
    hadoopClient = ["org.apache.hadoop:hadoop-streaming:$hadoopVersion",
              "org.apache.hadoop:hadoop-tools:$hadoopVersion"] 
}

def boolean isYarn() {
    return hadoopDistro == "hadoopYarn"
}

allprojects {
    dependencies {
        testCompile "junit:junit:$junitVersion"
        testCompile "org.hamcrest:hamcrest-all:$hamcrestVersion"
    }
}

if (rootProject == project) {
    println hadoopMsg
}

def File gitBranch() {
    // parse the git files to find out the revision
    File gitHead =  file("$rootDir/.git/HEAD")
    if (gitHead != null && !gitHead.exists()) {
      // Try as a sub module
      subModuleGit = file("$rootDir/.git")
      if (subModuleGit != null && subModuleGit.exists()) {
        String content = subModuleGit.text.trim()
        if (content.startsWith("gitdir:")) {
            gitHead = file("$rootDir/" + content.replace('gitdir: ','') + "/HEAD")
        }
      }
    }

    if (gitHead != null && gitHead.exists()) {
        String content = gitHead.text.trim()
        if (content.startsWith("ref:")) {
            return file("$rootDir/.git/" + content.replace('ref: ',''))
        }
        return gitHead
    }
    return null
}

def String gitHash() {
    String rev = "unknown"

    File gitHead =  gitBranch()
    if (gitHead.exists()) {
        rev = gitHead.text.trim()
    }
    return rev
}

def revHash = gitHash()

def nullLog4jConfig = file("$rootDir/mr/src/test/resources/log4jnull.properties").toURI().toURL().toString()

ext.manifestTemplate = manifest {
        attributes["Created-By"] = "${System.getProperty("java.version")} (${System.getProperty("java.specification.vendor")})"
        attributes['Implementation-Title'] = project.name
        attributes['Implementation-Version'] = project.version
        attributes['Implementation-URL'] = "http://github.com/elastic/elasticsearch-hadoop"
        attributes['Implementation-Vendor'] = "Elastic"
        attributes['Implementation-Vendor-Id'] = "org.elasticsearch.hadoop"
        
        def build = System.env['ESHDP.BUILD']
        if (build != null)
            attributes['Build'] = build
        
        attributes['Repository-Revision'] = revHash
}

allprojects { project ->

    compileJava {
        sourceCompatibility = 1.6
        targetCompatibility = 1.6 
    }
    
    compileJava.options*.compilerArgs = [
        "-Xlint:unchecked", "-Xlint:options"
    ]
    
    compileTestJava {
		sourceCompatibility = 1.6
		targetCompatibility = 1.6
    }
    
    sourceSets.test.resources.srcDirs = ["src/test/resources", "src/test/java"]

    test {
        reports.html.enabled = true     

        jacoco {
            enabled = project.hasProperty("enableJacoco")
            destinationFile = file("${rootProject.buildDir}/jacoco/jacocoTest.exec")
            excludes = ["org.elasticsearch.spark.*"]
            if (enabled) {
                System.setProperty("log4j.configuration", "$nullLog4jConfig")
            }
        }
    }

    sourceSets {
        itest {
        }
    }

    dependencies {
        // makes eclipse classpath easier
        testCompile "org.elasticsearch:elasticsearch:$esVersion"
        testRuntime "org.slf4j:slf4j-log4j12:1.7.6"
//        testRuntime "log4j:log4j:$log4jVersion"
        testRuntime "org.apache.logging.log4j:log4j-api:$log4jVersion"
        testRuntime "org.apache.logging.log4j:log4j-core:$log4jVersion"
        testRuntime "org.apache.logging.log4j:log4j-1.2-api:$log4jVersion"
        testRuntime "net.java.dev.jna:jna:4.2.2"
        //testCompile "org.elasticsearch.module:lang-groovy:$esVersion"
        testCompile "org.codehaus.groovy:groovy:$groovyVersion:indy"
        testCompile "org.elasticsearch.plugin:transport-netty3-client:$esVersion"
        testRuntime "org.locationtech.spatial4j:spatial4j:0.6"
        testRuntime "com.vividsolutions:jts:1.13"
        
        itestCompile sourceSets.main.output
        itestCompile configurations.testCompile
        itestCompile configurations.provided
        itestCompile sourceSets.test.output
        itestRuntime configurations.testRuntime
    }

    // deal with the messy conflicts out there
    configurations.all {
      resolutionStrategy {
        
        // force the use of commons-http from Hadoop 1.x
        // to avoid SLF4J warnings, force only one version
        force 'commons-httpclient:commons-httpclient:3.0.1' //, 'org.slf4j:slf4j-log4j12:1.7.6', 'org.slf4j:slf4j-api:1.7.6'
        //force 'com.google.guava:guava:18.0' // force guava version for Elastic
        force 'joda-time:joda-time:2.8' // force guava version for Spark
        force "org.codehaus.jackson:jackson-mapper-asl:$jacksonVersion", "org.codehaus.jackson:jackson-core-asl:$jacksonVersion"
        // used when using Elastic non-shaded version
        force "commons-cli:commons-cli:1.2"
        
        eachDependency { details ->
            // for slf4j use each dependency since there are so many variants
            if (details.requested.name.contains("slf4j-")) {
                    details.useVersion "1.7.6"
            }
            if (details.requested.name.equals("servlet-api")) {
                details.useTarget group: "org.eclipse.jetty.orbit", name: "javax.servlet", version: "3.0.0.v201112011016"
            }

        }
      }
    }

    // adding the M/R project creates duplicates in the Eclipse CP so here we filter them out
    // the lib entries with sources seem to be placed first so they 'win' over those w/o sources
    eclipse {
        classpath.file {
            whenMerged { cp ->
                entries.unique { a, b ->
                    return a.path.compareTo(b.path)
                }
                entries.removeAll { it.path.endsWith('.pom') }
            }
        }
        jdt {
            // use Java 7 since it is required by ES
            // ES-Hadoop is still on Java 6 (since Hadoop <2.7 is still on 6)
            // but ES 5.0 is JDK8 
            javaRuntimeName = "JavaSE-1.8"
            // specify again the compatibility to override the default JavaRE settings
            sourceCompatibility = 1.6
            targetCompatibility = 1.6
        }
    }
    
    jar {
        manifest {
            from manifestTemplate
        }
        
        from("$rootDir/docs/src/info") {
            include "license.txt"
            include "notice.txt"
            into "META-INF"
            expand(copyright: new Date().format('yyyy'), version: project.version)
        }
    }
    
    task sourcesJar(type: Jar, dependsOn:classes) {
        classifier = 'sources'
        from sourceSets.main.allSource
    }

    javadoc {
        ext.srcDir = file("${rootProject}/docs/src/api")
    
        configure(options) {
            //stylesheetFile = file("${rootProject.projectDir}/docs/src/api/javadoc.css")
            //overview = "${rootProject.projectDir}/docs/src/api/overview.html"
            docFilesSubDirs = true
            outputLevel = org.gradle.external.javadoc.JavadocOutputLevel.QUIET
            breakIterator = true
            author = false
            header = project.name
            showFromProtected()
    
            // enable it for java 1.8 or higher
            if (java8)
                addStringOption('Xdoclint:none', '-quiet')

            groups = [
            'Elasticsearch Map/Reduce' : ['org.elasticsearch.hadoop.mr*'],
            'Elasticsearch Cascading' : ['org.elasticsearch.hadoop.cascading*'],
            'Elasticsearch Hive' : ['org.elasticsearch.hadoop.hive*'],
            'Elasticsearch Pig' : ['org.elasticsearch.hadoop.pig*'],
            'Elasticsearch Spark' : ['org.elasticsearch.spark*'],
            'Elasticsearch Storm' : ['org.elasticsearch.storm*'],
            ]

            links = [
                "http://docs.oracle.com/javase/6/docs/api/",
                "http://commons.apache.org/proper/commons-logging/apidocs/",
                "http://hadoop.apache.org/docs/stable2/api/",
                //"http://hbase.apache.org/apidocs/",
                "http://pig.apache.org/docs/r0.15.0/api/",
                // r0.13.1 is invalid as it does not provide the package list
                "http://hive.apache.org/javadocs/r1.2.1/api/",
                "http://docs.cascading.org/cascading/2.6/javadoc/",
                "http://spark.apache.org/docs/latest/api/java/",
                "http://storm.apache.org/releases/current/javadocs/"
            ]

            excludes = [
                "org/elasticsearch/hadoop/mr/compat/**", 
                "org/elasticsearch/hadoop/rest/**", 
                "org/elasticsearch/hadoop/serialization/**", 
                "org/elasticsearch/hadoop/util/**",
                "org/apache/hadoop/hive/**"
            ]
        }

        title = "${rootProject.description} ${version} API"
    }

    task javadocJar(type: Jar) {
        classifier = 'javadoc'
        from javadoc
    }

    task pack(dependsOn: [jar, javadocJar, sourcesJar]) {
        outputs.files {
            jar.archivePath
            javadocJar.archivePath
            sourcesJar.archivePath
        }
        artifacts {
            archives javadocJar
            archives sourcesJar
        }
    }

    configurations { s3 }
    uploadToS3 {
        ext.toDir = "hadoop"
    }

}

configure(rootProject) {
    def eshModules = subprojects

    // jar used for testing Hadoop remotely (es-hadoop + tests)
    task hadoopTestingJar(type: Jar, dependsOn: jar) {
        eshModules.each { subproject ->
            from subproject.sourceSets.test.output
            from subproject.sourceSets.main.output
            from subproject.sourceSets.itest.output
        }
        classifier = 'testing'
        
        logger.info("Created Remote Testing Jar")
    }
}

allprojects { project ->
    task integrationTest(type: Test, dependsOn: rootProject.hadoopTestingJar) {
        testClassesDir = sourceSets.itest.output.classesDir
        classpath = sourceSets.itest.runtimeClasspath
        excludes = ["**/Abstract*.class"]
        
        ignoreFailures = true
        
        minHeapSize = "256m"
        maxHeapSize = "2g"
        if (!java8)
            jvmArgs '-XX:MaxPermSize=496m'

        testLogging {
            displayGranularity 0
            events "started", "failed" //, "standardOut", "standardError"
            exceptionFormat "full"
            showCauses true
            showExceptions true
            showStackTraces true
            stackTraceFilters "groovy"
            minGranularity 2
            maxGranularity 2
        }
        
        reports.html.enabled = false
        
        jacoco {
            enabled = project.hasProperty("enableJacoco")
            append = true
            destinationFile = file("${rootProject.buildDir}/jacoco/jacocoTest.exec")
            excludes = ["org.elasticsearch.spark.*"]
            
            if (enabled) {
                System.setProperty("log4j.configuration", "$nullLog4jConfig")
            }
        }
    }
}

project(":elasticsearch-hadoop-mr") {
    description = "Elasticsearch Hadoop Map/Reduce"
    
    dependencies {
        provided(hadoopClient)
        provided("org.codehaus.jackson:jackson-mapper-asl:$jacksonVersion")
        
        // Testing
        if (hadoopVersion.contains("1.0.")) {
            // missing dependency in Hadoop 1.0.3/1.0.4
            testCompile "commons-io:commons-io:2.1"
        }
        
        testCompile "io.netty:netty-all:4.0.29.Final"
        testCompile "org.elasticsearch:securemock:1.2"
    }
    
    def generatedResources = "$buildDir/generated-resources/main"
    
    sourceSets {
        main {
            output.dir(generatedResources, builtBy: "generateGitHash")
        }
    }
    
    task generateGitHash {
        inputs.file gitBranch()
        outputs.file generatedResources
        
        doLast {
            Properties props = new Properties()
            props.put("version", version)
            props.put("hash", gitHash())
            File output = new File(generatedResources, "esh-build.properties")
            new File(generatedResources).mkdirs()
            output.createNewFile()
            props.store(output.newWriter(), null) 
        }
    }
    
    eclipse.classpath.file {
        whenMerged { cp ->
            // export all jars (to be used upstream by dependent projects)  <-- for some reason Gradle removes all jars
            cp.entries.each { entry -> 
                if (entry.hasProperty("exported"))
                    entry.exported = true
            }
        }
    }
    
    artifacts  {
        s3 javadocJar
        s3 sourcesJar
        s3 jar
    }
}

// reusable method to nest Map/Reduce module in the rest of the projects to make them usable stand-alone
def nestMRProject(target) {
    target.evaluationDependsOn(':elasticsearch-hadoop-mr')
    
    target.dependencies {
        provided(project(":elasticsearch-hadoop-mr"))
        provided(project(path: ":elasticsearch-hadoop-mr", configuration:"compile"))
        
        testCompile project(":elasticsearch-hadoop-mr").sourceSets.test.runtimeClasspath
        itestCompile project(":elasticsearch-hadoop-mr").sourceSets.itest.runtimeClasspath
    }

    target.jar {
        from(zipTree(project(":elasticsearch-hadoop-mr").jar.archivePath)) {
            include "org/elasticsearch/hadoop/**"
            include "esh-build.properties"
        }
    }

    target.javadoc {
        source += project(":elasticsearch-hadoop-mr").sourceSets.main.allJava
        classpath += files(project(":elasticsearch-hadoop-mr").sourceSets.main.compileClasspath)
    }
    
    target.sourcesJar {
        from project(":elasticsearch-hadoop-mr").sourceSets.main.allJava.srcDirs
    }

    target.artifacts  {
        s3 target.javadocJar
        s3 target.sourcesJar
        s3 target.jar
    }
}

project(":elasticsearch-hadoop-cascading") {
    description = "Elasticsearch Hadoop Cascading"
    nestMRProject(project)

    dependencies {
        provided("cascading:cascading-hadoop:$cascadingVersion")
        provided("cascading:cascading-local:$cascadingVersion")
    }
}

project(":elasticsearch-hadoop-pig") {
    description = "Elasticsearch Hadoop Pig"
    nestMRProject(project)
    
    ext.pigClassifier = isYarn() ? "h2" : ""
    
    dependencies {
        provided("org.apache.pig:pig:$pigVersion:$pigClassifier")
        testRuntime("joda-time:joda-time:$jodaVersion")
        
        testRuntime "com.google.guava:guava:11.0"
        testRuntime "jline:jline:0.9.94"

        itestRuntime "dk.brics.automaton:automaton:1.11-8"
    }
}

project(":elasticsearch-hadoop-hive") {
    description = "Elasticsearch Hadoop Hive"
    nestMRProject(project)
    
    dependencies {
        provided("org.apache.hive:hive-service:$hiveVersion") {
            exclude module: "log4j-slf4j-impl"
        }

        itestRuntime("org.apache.hive:hive-jdbc:$hiveVersion") {
            exclude module: "log4j-slf4j-impl"
        }
    }
}


// reusable method to nest Map/Reduce module in the rest of the projects to make them usable stand-alone
def baseSparkProject(target, sparkVersion) {
   def scalaVer = project.hasProperty("scala") ? project.getProperty("scala") : "210"

    switch (scalaVer) {
      case "211":
        ext.scalaVersion = scala211Version
        ext.scalaMajorVersion = scala211MajorVersion
        break;

      default:
        ext.scalaVersion = scala210Version
        ext.scalaMajorVersion = scala210MajorVersion
    }
    
    target.archivesBaseName += "_$scalaMajorVersion"

    target.apply plugin: 'scala'
    
    target.compileScala {
        configure(scalaCompileOptions.forkOptions) {
            memoryMaximumSize = '1g'
            jvmArgs = ['-XX:MaxPermSize=512m']
        }
        scalaCompileOptions.useAnt = false
    }
    
    println "Using Scala $scalaMajorVersion [$scalaVersion]"

    // when using Scala 2.11 use a different folder to cache the artifacts between builds
    if (scalaVer == "211") {
      target.sourceSets.each { 
        it.output.classesDir = file(it.output.classesDir.absolutePath.replaceAll("classes", "classes.211"))
      }
      target.javadoc {
        destinationDir = file("$project.docsDir/javadoc-211")
      }
      target.integrationTest {
        // update test target
        testClassesDir = target.sourceSets.itest.output.classesDir
      }
    }
    
    def coreSrc = file("$target.projectDir/../core").absolutePath.replace('\\','/')
    
    target.sourceSets {
        main.scala.srcDirs += "$coreSrc/main/scala"
        test.scala.srcDirs += "$coreSrc/test/scala"
        itest.java.srcDirs += "$coreSrc/itest/java"
        itest.scala.srcDirs += "$coreSrc/itest/scala"
        itest.resources.srcDirs += "$coreSrc/itest/resources"
    }

    
    // currently the outside project folders are transformed into linked resources however
    // Gradle only supports one so the project will be invalid as not all sources will be in there
    // as such, they are setup here manually for Eclipse. IntelliJ probably needs a similar approach
    target.eclipse {
        project.file.whenMerged { pj ->
            // eliminated resources created by gradle
            
            linkedResources.clear()
            linkedResources.add(new org.gradle.plugins.ide.eclipse.model.Link("core/main/scala", "2", "$coreSrc/main/scala", null))
            linkedResources.add(new org.gradle.plugins.ide.eclipse.model.Link("core/test/scala", "2", "$coreSrc/test/scala", null))
            linkedResources.add(new org.gradle.plugins.ide.eclipse.model.Link("core/itest/java", "2", "$coreSrc/itest/java", null))
            linkedResources.add(new org.gradle.plugins.ide.eclipse.model.Link("core/itest/scala", "2", "$coreSrc/itest/scala", null))
            linkedResources.add(new org.gradle.plugins.ide.eclipse.model.Link("core/itest/resources","2", "$coreSrc/itest/resources", null))
            
        }
        classpath.file {
            whenMerged { cp ->
                entries.removeAll { entry ->
                    entry.kind == 'src' && (entry.path in ["scala", "java", "resources"] || entry.path.startsWith("itest-") || entry.path.endsWith("-scala"))
                }
                
                entries.add(new org.gradle.plugins.ide.eclipse.model.SourceFolder("core/main/scala", null))
                entries.add(new org.gradle.plugins.ide.eclipse.model.SourceFolder("core/test/scala", null))
                entries.add(new org.gradle.plugins.ide.eclipse.model.SourceFolder("core/itest/java", null))
                entries.add(new org.gradle.plugins.ide.eclipse.model.SourceFolder("core/itest/scala", null))
                entries.add(new org.gradle.plugins.ide.eclipse.model.SourceFolder("core/itest/resources", null))
            }
        }
    }
    
    target.dependencies {
        provided("org.apache.spark:spark-core_$scalaMajorVersion:$sparkVersion") {
            exclude group: 'javax.servlet'
            exclude group: 'org.apache.hadoop'
        }
        
        optional("org.apache.spark:spark-sql_$scalaMajorVersion:$sparkVersion") {
            exclude group: 'org.apache.hadoop'
        }
        
        compile("org.scala-lang:scala-library:$scalaVersion")

        itestCompile("org.apache.spark:spark-sql_$scalaMajorVersion:$sparkVersion") {
            exclude group: 'org.apache.hadoop'
        }
    }
    
    nestMRProject(target)
    
    
    // disable jacoco as it will fail
    test {
      jacoco {
        enabled = false
      }
    }
    
    target.scaladoc {
        title = "${rootProject.description} ${version} API"
    }
}

project(":elasticsearch-spark-13") {
    description = "Elasticsearch Spark (for Spark 1.3-1.6)"

    baseSparkProject(project, spark13Version)

    dependencies {
        optional("org.apache.spark:spark-streaming_$scalaMajorVersion:$spark13Version") {
            exclude group: 'org.apache.hadoop'
        }

        itestCompile("org.apache.spark:spark-streaming_$scalaMajorVersion:$spark13Version") {
            exclude group: 'org.apache.hadoop'
        }
    }

    // deal with the messy conflicts out there
    configurations.all {
      resolutionStrategy {
        eachDependency { details ->
            // in a similar vein, change all javax.servlet artifacts to the one used by Spark
            // otherwise these will lead to SecurityException (signer information wrong)
            if (details.requested.name.contains("servlet") && !details.requested.name.contains("guice")) {
                details.useTarget group: "org.eclipse.jetty.orbit", name: "javax.servlet", version: "3.0.0.v201112011016"
            }
        }
      }
      
      exclude group: "org.mortbay.jetty"
    }
}

project(":elasticsearch-spark-20") {
    description = "Elasticsearch Spark (for Spark 2.0)"

    baseSparkProject(project, spark20Version)

    dependencies {
        optional("org.apache.spark:spark-streaming_$scalaMajorVersion:$spark20Version") {
            exclude group: 'org.apache.hadoop'
        }

        itestCompile("org.apache.spark:spark-streaming_$scalaMajorVersion:$spark20Version") {
            exclude group: 'org.apache.hadoop'
        }
    }
}

project(":elasticsearch-storm") {
    description = "Elasticsearch Storm"
    nestMRProject(project)
    
    dependencies {
        provided("org.apache.storm:storm-core:$stormVersion") {
            exclude module: "log4j-slf4j-impl"
        }

        itestCompile "com.google.guava:guava:16.0.1"
        itestRuntime "com.twitter:carbonite:1.4.0"
    }
    
    // add itest to Eclipse
    //eclipse.classpath.plusConfigurations += [configurations.itestCompile]
    
    integrationTest.enabled = false
}

configure(rootProject) {
    description = "Elasticsearch Hadoop"
    
    // include Spark 2.0 support in main jar
    def eshModules = subprojects - project(":elasticsearch-spark-13")
    
    // build root project dependencies
    // extract optional and provided dependencies
    eshModules.each { subproject ->
        for (scope in ["provided", "optional"]) {
             subproject.configurations[scope].allDependencies.findAll { dep ->
                    // look only at external dependencies
                    dep instanceof org.gradle.api.artifacts.ExternalDependency
                }.each { match ->
                    // convert all dependencies outside Hadoop (and Jackson) from provided to optional
                    String depScope = (match.group in ["org.apache.hadoop", "org.codehaus.jackson"] ? "provided" : "optional")
                    project.dependencies.add(depScope, match)
                } 
        }
    }
    
    sourcesJar {
        eshModules.each { subproject ->
            from subproject.sourceSets.main.allJava.srcDirs
        }
    }

    jar {
        dependsOn {
            eshModules.collect {
                    it.tasks.getByName("jar")
                }
        }
        duplicatesStrategy = "exclude"
        eshModules.each { subproject ->
            from(zipTree(subproject.jar.archivePath)) {
                exclude "META-INF/**"
                include "**/*"
            }
        }
    }
    
    javadoc {
        // aggregate
        source eshModules*.javadoc*.source
        classpath += files(eshModules*.sourceSets*.main.flatten().compileClasspath)
    }
    
    // packaging
    task distZip(type: Zip, dependsOn: pack) {
        group = "Distribution"
        description = "Builds -${classifier} archive, containing all jars and docs, suitable for download page."
        
        dependsOn {
            eshModules.collect { it.tasks["pack"] }
        }

        configurations { s3 }
        artifacts { s3 distZip }

        ext.folderName = "${baseName}" + "-" + "${version}"
        
        from(".") {
            include "README.md"
            include "LICENSE.txt"
            include "NOTICE.txt"
            into folderName
            //expand(yyyy: new Date().format("yyyy"), version: project.version)
        }

        from ("build/libs") {
            into "$folderName/dist" 
            exclude "*-testing.jar"
        }

        eshModules.each { subproject ->
            from (subproject.jar.destinationDir) {
                into ("${folderName}/dist")
            }
        }
        
        // execution phase only
        doLast {
            ant.checksum(file: archivePath, algorithm: 'SHA1', format: 'MD5SUM', fileext: '.sha1.txt')
        }
    }
    
    uploadToS3() {
        dependsOn(distZip)
    }

    jacocoTestReport {
        subprojects.each {
            sourceSets it.sourceSets.main
        }
        reports {
            xml.enabled false
            csv.enabled false
            html {
                enabled = true
                destination "${buildDir}/jacoco/html"
            }
        }
        
        afterEvaluate {
            classDirectories = files(classDirectories.files.collect {
                // We're excluding Spark since the Scala bytecode trips Jacoco over and the plugin can't import yet filters
                fileTree(dir: it, excludes: [ "org/elasticsearch/hadoop/yarn/**", "org/apache/hadoop/hive/**", "org/elasticsearch/spark/**", "org/elasticsearch/plugin/hadoop/**"])
            })
        }
    }
    
    task testReport(type: TestReport) {
        destinationDir = file("$buildDir/reports/allTests")
        // Include the results from the `test` task in all subprojects
        reportOn subprojects*.test
        reportOn subprojects*.integrationTest
    }
    
    task wrapper(type: Wrapper) {
        description = 'Generates gradlew[.bat] scripts'
        gradleVersion = '2.14.1'
    }
}

// reporting
allprojects { project ->
    
    findbugs {
      toolVersion = "3.0.1"
      ignoreFailures = true
      effort = "max"
      reportLevel = "low"
      sourceSets = [sourceSets.main]
      
    }
    
    findbugsMain {
      reports {
        xml.enabled = true
        html.enabled = false
      }
      enabled = project.hasProperty("enableFindbugs")
    }

    pmd {
      ignoreFailures = true
      sourceSets = [sourceSets.main]
      toolVersion = "5.3.3"
    }
    
    pmdMain {
      reports {
        xml.enabled = true
        html.enabled = false
      }
      enabled = project.hasProperty("enablePmd")
    }

    jacoco {
        toolVersion = "0.7.5.201505241946"
    }

    if (java8) {
        findbugsMain.enabled = false
        pmdMain.enabled = false
    }
}

// skip creation of javadoc jars 
assemble.dependsOn = ['jar', 'hadoopTestingJar']
defaultTasks 'build'