1 import java.util.jar.JarEntry;
6 apply plugin: 'eclipse'
9 def String getProjectProperty(String propertyName) {
10 String propertyValue = "null"
11 if (hasProperty(propertyName)) {
12 propertyValue = this.properties[propertyName]
15 throw GradleScriptException("PropertyName " + propertyName + " is not defined in properties file")
19 def projectName = "voldemort"
21 def sourceDir = getProjectProperty('src.dir')
22 def distDir = getProjectProperty('dist.dir')
23 def classesDir = getProjectProperty('classes.dir')
24 def javaDir = getProjectProperty('java.dir')
25 def privateLibDir = getProjectProperty('private.lib.dir')
26 def resourcesDir = getProjectProperty('resources.dir')
27 def javaDocDir = getProjectProperty('javadoc.dir')
29 def voldTestClassesDir = getProjectProperty('testclasses.dir')
31 def commonTestSrcDir = getProjectProperty('commontestsrc.dir')
32 def unitTestSrcDir = getProjectProperty('unittestsrc.dir')
33 def intTestSrcDir = getProjectProperty('inttestsrc.dir')
34 def longTestSrcDir = getProjectProperty('longtestsrc.dir')
36 def contribClassesDir = getProjectProperty('contrib.classes.dir')
37 def contribRootDir = getProjectProperty('contrib.root.dir')
39 def voldVersion = getProjectProperty('curr.release')
40 def javacVersion = getProjectProperty('javac.version')
41 def scalaVersion = getProjectProperty('scalac.version')
43 //This is the javaCompile variable version. Directly defining 'def version' will override this and cause nightmare
46 def archiveDirectoryName = projectName + '-' + version
47 def archiveDirectoryPath = distDir + "/" + archiveDirectoryName
49 def deleteDirectoryContents(directory) {
50 project.file(directory).deleteDir()
51 project.file(directory).mkdirs()
54 println 'java source target compatibility version ' + javacVersion
55 sourceCompatibility = javacVersion
56 targetCompatibility = javacVersion
57 compileJava.options.debug = true
60 // Using zinc scala compiler fails intermittently because of missing dependencies
61 // Scala is not main stream so incremeental compilation is not worth the problem
63 // tasks.withType(ScalaCompile) { scalaCompileOptions.useAnt = false }
67 flatDir { dirs privateLibDir }
68 flatDir { dirs contribRootDir }
74 java { srcDirs = [javaDir]}
83 output.classesDir = classesDir
84 output.resourcesDir = resourcesDir
95 output.classesDir = voldTestClassesDir
98 java { srcDirs = [contribRootDir]}
99 compileClasspath += sourceSets.main.runtimeClasspath + sourceSets.test.runtimeClasspath
100 output.classesDir = contribClassesDir
106 from (javaDir) { exclude '**/*.java','**/*.html','**/*.scala', '**/log4j.properties' }
111 // Theoretically this block can be replaced by including the log4j.properties in main resources.
112 // But that causes the log4j.properties to be present in the voldJar . Not sure what is the
113 // implication of this change, so avoiding it for now.
114 from (javaDir) { include 'log4j.properties' }
119 compileTestJava.doLast {
121 from (commonTestSrcDir) { exclude '**/*.java','**/*.html' }
122 from (unitTestSrcDir) { exclude '**/*.java','**/*.html' }
123 into voldTestClassesDir
127 compileContribJava.doLast {
129 from (contribRootDir + '/ec2-testing/resources')
130 into contribClassesDir
134 task testJar(type: Jar) {
135 baseName = projectName + "-test"
136 from sourceSets.test.output
137 destinationDir = project.file(distDir)
140 task voldJar(type:Jar) {
141 baseName = projectName
143 attributes 'Voldemort-Implementation-Version' : version,
144 'Implementation-Title': 'Voldemort',
145 'Implementation-Version': version,
146 'Implementation-Vendor' :'LinkedIn'
148 from sourceSets.main.output
149 destinationDir = project.file(distDir)
153 from sourceSets.main.output
154 webXml = project.file('web.xml')
155 destinationDir = project.file(distDir)
158 task contribJar(type:Jar) {
159 dependsOn voldJar, testJar, sourceSets.contrib.output
160 baseName = projectName + "-contrib"
161 from sourceSets.contrib.output
162 destinationDir = project.file(distDir)
165 task srcJar(type: Jar, dependsOn: classes) {
167 from sourceSets.main.java.srcDirs
168 destinationDir = project.file(distDir)
181 doLast { deleteDirectoryContents(javaDocDir) }
184 task copySources (type: Copy) {
185 from ('.') { include 'bin/**' }
186 from ('.') { include distDir + '/*.jar'}
187 from ('.') { exclude distDir + '/**' ,'bin/**' , 'build/**', '.git/**' , '.gradle/**' }
188 into archiveDirectoryPath
191 task zip (type: Zip) {
192 dependsOn copySources
193 baseName = projectName
196 include archiveDirectoryName + '/bin/**'
200 include archiveDirectoryName + '/**'
201 exclude archiveDirectoryName + '/bin/**'
204 destinationDir = project.file(distDir)
207 task tar (type: Tar) {
208 dependsOn copySources
209 compression = Compression.GZIP
210 baseName = projectName
214 include archiveDirectoryName + '/bin/**'
218 include archiveDirectoryName + '/**'
219 exclude archiveDirectoryName + '/bin/**'
222 destinationDir = project.file(distDir)
225 task copyDeps(type: Copy) {
226 from {configurations.compile }
231 jar.dependsOn contribJar,srcJar, copyDeps
232 compileContribJava.dependsOn voldJar
233 copySources.dependsOn jar
235 tasks.withType(Test) {
236 // ant restarts jvm for each tests, If not restarted the test runs into outOfMemory even
237 // if you set the JVM to 8gb. On inspecting most of the space is consumed by int[] of
238 // Histogram in the NioSelectorManager. I believe this could be explained by
239 // creating lots of client factory which creates lot of NIO threads. Did not proceed
240 // further as I will be maintaining compatbility with ant. Also if you dont fork for each
241 // tests JMX bean related tests will fail.
243 // Do not set the max parallelism as there are tests that uses the same port and will
244 // run into bind exceptions.
249 // If ignoreFailures is not set, then merged reports will not be generated
250 // Gradle aborts further tasks on test failure. so if you run junitAll
251 // which runs 3 tests, reports task will never be run on failure cases.
252 ignoreFailures = true
253 //ignoreFailures = gradle.startParameter.continueOnFailure
258 events "passed", "skipped", "failed"
259 exceptionFormat = 'full'
262 afterTest { test, result ->
263 logger.lifecycle("testFinished: $test, result: $result.resultType")
267 def classesSize = candidateClassFiles.files.size()
268 logger.lifecycle("{} starts executing {} test classes {}",
269 path, classesSize, classesSize > 0? "(" + candidateClassFiles.files*.name[0] + ", ...)" : "")
272 //all standard error messages from tests will get routed to 'DEBUG' level messages.
273 //logging.captureStandardError(LogLevel.DEBUG)
274 //all standard output messages from tests will get routed to 'DEBUG' level messages.
275 //logging.captureStandardOutput(LogLevel.DEBUG)
277 //Set reasonable defaults for reports location
278 reports.html.destination = file("$project.buildDir/reports/$name")
279 reports.junitXml.destination = file("$project.buildDir/$name-results")
281 //Set reasonable defaults classpath and classes dir. They can be reconfigured in an individual task.
282 it.testClassesDir = sourceSets.test.output.classesDir
283 classpath = sourceSets.test.runtimeClasspath
288 def DirsToDelete = ["STORES" , ".temp", ".version", "data"]
289 def deleteRecursively
291 deleteRecursively = { file ->
292 file.eachFile() {f ->
294 if( DirsToDelete.contains(f.getName()) )
296 println "deleting ${f.getAbsolutePath()}"
307 deleteRecursively (new File("config"))
311 task junit(dependsOn: test)
313 Collection<String> testClassesFrom(String dir, String include = '**/*Test.*') {
314 //take all *Test.java files found in given dir, make the path relative and replace .java with .class
315 fileTree(dir: dir, includes: [include]).collect { it.absolutePath.replaceAll(file(dir).absolutePath + "/", "").replaceAll(".java\$", ".class")}
319 description = "Runs acceptance tests"
320 include testClassesFrom(unitTestSrcDir)
323 task junitLong(type: Test) {
324 description = "Runs long junit tests"
325 include testClassesFrom(longTestSrcDir)
328 task junitRebalance(type: Test) {
329 include testClassesFrom(unitTestSrcDir, '**/*Rebalance*Test.java')
332 task junitRebalanceLong(type: Test) {
333 include testClassesFrom(longTestSrcDir, '**/*Rebalance*Test.java')
336 task contribJunit(type: Test) {
337 description = "Run contrib junit tests except EC2 and Krati tests."
338 it.testClassesDir = file(contribClassesDir)
340 exclude '**/*PerformanceTest.class'
341 exclude '**/*RemoteTest.class'
342 exclude '**/Ec2*Test.class'
343 exclude '**/Krati*Test.class'
344 exclude '**/HadoopStoreBuilder*Test.class'
346 classpath += sourceSets.contrib.runtimeClasspath + sourceSets.contrib.output
349 task junitAll(type: TestReport) {
350 reportOn test, junitLong, contribJunit
351 destinationDir = file("$project.buildDir/reports/$name")
354 task aggregatedJunit(type: TestReport) {
355 destinationDir = file("$project.buildDir/reports/$name")
358 tasks.withType(Test) {
359 finalizedBy aggregatedJunit
360 doLast { aggregatedJunit.reportOn it.binResultsDir }
363 task wrapper(type: Wrapper) { gradleVersion = '2.0' }
367 // Avro serialization format
368 compile 'org.apache.avro:avro:1.4.0'
370 // INTERNAL_LIBS azkaban version not found
371 // azkaban-common-0.05.jar
373 // INTERNAL_LIBS Used for tomcat deployment, not sure if anyone uses it
374 // catalina-ant.jar , version not found in maven central
376 // coders decoders containing the Base64,binary encoding
377 compile 'commons-codec:commons-codec:1.4'
379 // TRANSITIVE_DEPENDENCY Contrib jar depends on commons-configuration-1.6.jar
380 // commons-configuration instead depends on commons-collection
381 //compile 'commons-collections:commons-collections:3.2.1'
383 // Used by MySql storage engine classes
384 // The jar supports database connection pooling
385 compile 'commons-dbcp:commons-dbcp:1.2.2'
387 // commons io is used at many places
388 // IOUtils, FileUtils and ByteArrayOutputStream
389 compile 'commons-io:commons-io:1.4'
391 // LZF compression strategy for store and tests.
392 compile 'com.ning:compress-lzf:0.9.1'
394 // TRANSITIVE_DEPENDENCY tusk jar internally uses gson.
395 compile 'com.google.code.gson:gson:2.2.4'
397 // Used all over the place for collections
398 compile 'com.google.guava:guava:14.0.1'
400 // used for readonly store hdfs fetcher.
401 compile 'org.apache.hadoop:hadoop-auth:2.0.5-alpha'
403 // used at lots of places. Seems like there is some overlap between httpclient and core, but not clear
404 compile 'org.apache.httpcomponents:httpclient:4.1.2'
406 // contains both http server and client functionalities. Used for HttpResponse but could be used at more places.
407 compile 'org.apache.httpcomponents:httpcore:4.1.2'
409 // JSON mapping library from Java Objects to JSON
410 compile 'org.codehaus.jackson:jackson-mapper-asl:1.4.0'
412 // JSON processing library
413 compile 'org.codehaus.jackson:jackson-core-asl:1.4.0'
415 // Used for reading XML files and Document.
416 compile 'jdom:jdom:1.1'
418 // je- BDB je 5.0.88 is not available in maven central repository.
419 // Could be possibly found in oracle maven repository, but not sure
421 groupId>com.sleepycat</groupId>
422 <artifactId>je</artifactId>
423 <version>3.3.75</version>
424 groupId>com.sleepycat</groupId>
425 <artifactId>je</artifactId>
426 <version>3.3.75</version>
429 // Jetty is used for HttpService and tests. Jetty Util is used for QueuedThreadPool class.
430 compile 'org.mortbay.jetty:jetty-util:6.1.18'
431 compile 'org.mortbay.jetty:jetty:6.1.18'
433 // A line processing library for command line. No compile time dependency
434 // Used by Voldemort shell
435 compile 'jline:jline:0.9.94'
437 // jna is library for invoking native functions
438 // used in the readonly store
439 compile 'net.java.dev.jna:jna:3.2.7'
441 // joda time is replacement for Java Date and Time
442 // used in readonly store code.
443 compile 'joda-time:joda-time:1.6'
445 // Used for argument command line parsing
446 compile 'net.sf.jopt-simple:jopt-simple:4.6'
448 // INTERNAL_LIBS thrift is one of the supported serialization format.
449 // libthrift 0.5 version is not found in the maven central repository.
450 // The closest version available is 0.6.1
451 // libthrift-0.5.0.jar
453 // log4j - logger used in almost all files
454 compile 'log4j:log4j:1.2.15'
456 // used in readonly store and Co-ordinator
457 compile 'javax.mail:mail:1.4.1'
459 // Used in co-ordinator and rest services
460 compile 'io.netty:netty:3.5.8.Final'
462 // TRANSITIVE_DEPENDENCY Paranamer is a library that allows the parameter names of non-private methods and constructors to be accessed at runtime
463 // Avro has a dependency on paranamer
464 // compile 'com.thoughtworks.paranamer:paranamer:2.1'
466 // protobuf is a supported protocol format between voldemort client and server
467 compile 'com.google.protobuf:protobuf-java:2.3.0'
469 // Command line shell is also supported in scala
470 compile 'org.scala-lang:scala-compiler:2.10.4'
471 compile 'org.scala-lang:scala-library:2.10.4'
472 compile 'org.scala-lang:scala-reflect:2.10.4'
475 compile 'javax.servlet:servlet-api:2.5'
477 // slf4j is another logging abstraction framework.
478 // It is used by the apache.avro, apache.hadoop and r2 clients
479 compile 'org.slf4j:slf4j-api:1.5.6'
480 compile 'org.slf4j:slf4j-log4j12:1.5.6'
482 // snappy is one of the supported compression strategies in voldemort
483 compile 'org.iq80.snappy:snappy:0.2'
485 // could not locate tusk in maven central repository
488 // Velocity is a simple yet powerful Java-based template engine that renders data
489 // from plain Java objects to text, xml, email, SQL, Post Script, HTML etc
490 // Velocity is used for Http Server GUI
491 compile 'org.apache.velocity:velocity:1.6.2'
493 // TRANSITIVE_DEPENDENCY Apache XML Parser
495 // compile 'xerces:xercesImpl:2.9.1'
497 compile fileTree(dir: privateLibDir, includes: ['**/*.jar'])
499 // cern library containing high performance Maps for int and double
500 // Currently only used in the tests
501 testCompile 'colt:colt:1.2.0'
503 // Used in ec2 testing , is a wrapper over any logging framework
504 testCompile 'commons-logging:commons-logging:1.1.1'
506 // Used in resource pool perf testing class
507 testCompile 'commons-pool:commons-pool:1.5.2'
509 testRuntime 'mysql:mysql-connector-java:5.1.31'
511 // Used for unit tests and other automated testing
512 testCompile 'junit:junit:4.6'
514 // Mockito is written by our beloved friend Szczepan Faber :)
515 // Mocking framework used in some tests
516 testCompile 'org.mockito:mockito-all:1.8.5'
518 contribCompile sourceSets.main.output
519 contribCompile sourceSets.test.output
521 // declaring contribCompile dependencies as compile dependencies
522 // otherwise while copying dependencies to lib directory
523 // conflict resolution is not done properly across sourceSets
524 // and we end up with 2 versions of few jars like ( log4j, servlet etc. )
525 compile 'commons-configuration:commons-configuration:1.6'
526 compile 'org.apache.hadoop:hadoop-core:1.0.4'
528 compile 'com.linkedin.pegasus:r2:1.8.3'
529 compile 'com.linkedin.pegasus:data:1.8.3'
530 compile 'com.linkedin.pegasus:pegasus-common:1.8.3'
532 compile 'com.google.code.typica:typica:1.7.2'
533 compile 'com.sna-projects.krati:krati:0.4.9'
538 buildCommand 'org.eclipse.jdt.core.javabuilder'
541 // Currently the javac Version of jar is set to 1.5
542 // But @Override works differently between both, so overriding here
543 sourceCompatibility = targetCompatibility = 1.6
546 defaultOutputDir = project.file('classes')
548 // SourceSets creates multiple src/java in .classpath with varying includes
549 // But eclipse 4.3 is complaining about duplicate classpaths
550 // If contrib root is included instead of the seperate projects, eclipse
551 // expects the package to be different. ( contrib.restclient.test.voldemort.client
552 // versus voldemort.client ). So removing all the src entries and adding the previous
553 // src entries which used to work.
554 whenMerged { classpath ->
555 classpath.entries.removeAll { entry ->
556 (entry.kind == 'src' )
560 def node = it.asNode()
563 "contrib/ec2-testing/resources",
564 "contrib/ec2-testing/src/java",
565 "contrib/ec2-testing/test",
566 "contrib/hadoop-store-builder/test",
567 "contrib/hadoop-store-builder/src/java",
573 "contrib/krati/src/java",
574 "contrib/krati/test",
575 "contrib/collections/src/java",
576 "contrib/collections/test",
577 "contrib/restclient/src/java",
578 "contrib/restclient/test"
581 node.appendNode('classpathentry', [kind: 'src', path: "$it"])