Merge pull request #179 from torodb/release_1_0_0_beta2

Release 1 0 0 beta2

.gitignore

@@ -1,5 +1,13 @@
-**/.idea/
 target/
+pom.xml.tag
+pom.xml.releaseBackup
+pom.xml.versionsBackup
+pom.xml.next
+release.properties
+dependency-reduced-pom.xml
+buildNumber.properties
+.mvn/timing.properties
+**/.idea/
 *.iml
 **/nbproject/**
 **/nbactions.xml
@@ -8,7 +16,8 @@ target/
 .project
 .classpath
 .settings/
+.checkstyle
 hs_err*.log
 /documentation/static
 /documentation/site
-.DS_Store
+.DS_Store

.travis.yml

@@ -1,30 +1,49 @@
 language: java
-sudo: false
+sudo: required
 dist: trusty
-script: mvn -Psafer -Pintegration -Passembler -B -e -T 1C verify
 jdk: oraclejdk8
 
-env:
-  # the SONAR_TOKEN variable
-  - secure: "eQaksffQhrlaAKEFWIGR7Wbo01QKscfPl7MOQHQAytP74uU6in1VcPITdHr2UgTH/PhjMJ5Uevw7mrzDN+pDE8csNxVF7JozEAIh6DYhmBmmBOYu3+AypDqdtaBOmgAPkTLxSthiy5hsmOgCjfXPGrWqpQJ918n0RtdIL1p7W4L5N3I2yDcQOpFSXXizWADpGH/fErhihKdepNCco0JCfxLmpFa+i+goqsF1EEJPb2Ylz4LYhlez2NqULGQWio2+ucrYLDZHFTrdRyQ9Tq/RA62zdynmWQAYuE5nYmO1vmvk2mR+xQpIyevvAQY4RmffYKkK/Hq334H8rIH7etCn/e0fzaLnOveyIttoG0yaqpY/mPIZkOQBV0pSjCFONvt0gsgFfPYv5sHvRBM2dj1knwOLqwhqseR6en8+TaqeP2aYj/ittU3+7QJ9yHorfAYG14ofxm+Ue4o9wJ5FsyAzw5IpYnu28WRGy7+7kFfzo1prNh7dlqjDKrBMeK292seVPH+RDMaIs2ZAzfUZElhOtqvXGsuBfDtO2FG7L9ke+tbVB/z/srM0cXPJW0xcPdoeVgoH5+CyLTaR4+8tT0recQwfKrl7zluwKk+zWmxCs5BuV+JHcWrgC+sD0iKvOHbf/w+s5bLOs0kB4YjoiUZ0pK+9NphR/Mew7shnTFZi63o="
-  # TODO: Add a SONAR_GITHUB_TOKEN to support Sonar on pull requests (or wait until they improve their compatibility)
+before_install:
+  - wget https://archive.apache.org/dist/maven/maven-3/3.2.2/binaries/apache-maven-3.2.2-bin.tar.gz
+  - tar xf apache-maven-3.2.2-bin.tar.gz
+  - export M2_HOME=$PWD/apache-maven-3.2.2
+  - export PATH=$M2_HOME/bin:$PATH
+#Required to use Codacy
+  - sudo apt-get install jq
+  - wget -O ~/codacy-coverage-reporter-assembly-latest.jar $(curl https://api.github.com/repos/codacy/codacy-coverage-reporter/releases/latest | jq -r .assets[0].browser_download_url)
 
 addons:
-  sonarqube:
-    branches:
-      - devel
-      - master
   apt:
     packages:
       - oracle-java8-installer
+
+cache:
+  directories:
+  - $HOME/.m2/repository
+
+before_cache:
+  - rm -f $HOME/.m2/repository/com/torodb
 
 before_script:
-- export GIT_BRANCH=$TRAVIS_BRANCH
+  - export GIT_BRANCH=$TRAVIS_BRANCH
 
-cache:
-  directories:
-    - '$HOME/.sonar/cache'
+script: |
+        if [ "$TRAVIS_EVENT_TYPE" == cron ] && [ "$TRAVIS_BRANCH" == devel ]
+        then
+            bash .travis/build-packages
+        else
+            mvn -Psafer -Pintegration -B -e -T 1C -Dcheckstyle.consoleOutput=false verify
+        fi
 
 after_success:
-  - if [[ $TRAVIS_REPO_SLUG = torodb/torodb ]]; then bash <(curl -s https://codecov.io/bash) || echo 'Codecov did not collect coverage reports'; else echo 'Codecov not notified'; fi
-  - if [[ $TRAVIS_REPO_SLUG = torodb/torodb ]]; then mvn sonar:sonar || echo 'Error while notifying SonarQube'; else echo 'SonarQube not notified'; fi
+  - |
+    if [ "$TRAVIS_EVENT_TYPE" != cron ] && [ "$TRAVIS_REPO_SLUG" == torodb/stampede ]
+    then
+        # Upload reports to Codecov
+        bash <(curl -s https://codecov.io/bash) || echo 'Codecov did not collect coverage reports';
+        # Upload reports to Codacy
+        java -cp ~/codacy-coverage-reporter-assembly-latest.jar com.codacy.CodacyCoverageReporter -l Java -r reporting/target/site/jacoco-aggregate/jacoco.xml || echo 'Codacy report fail'
+    else
+        echo 'Skipping a metrics reports because this repo/build is not permitted'
+    fi
+  

.travis/build-packages

@@ -0,0 +1,169 @@
+#!/bin/bash
+
+set -e +x +v +o history
+
+function finish {
+    rm -f ~/.aws/config
+
+    if [ -f ~/.m2/settings.xml ]
+    then
+      rm ~/.m2/settings.xml
+    fi
+    if [ -f ~/.m2/settings.xml.bak ]
+    then
+      mv ~/.m2/settings.xml.bak ~/.m2/settings.xml
+    fi
+
+    if [ -f ~/.ssh/id_rsa ]
+    then
+      rm -f ~/.ssh/id_rsa
+    fi
+    if [ -f ~/.ssh/id_rsa.bak ]
+    then
+      mv ~/.ssh/id_rsa.bak ~/.ssh/id_rsa
+    fi
+    if [ -f ~/.ssh/id_rsa.pub ]
+    then
+      rm -f ~/.ssh/id_rsa.pub
+    fi
+    if [ -f ~/.ssh/id_rsa.pub.bak ]
+    then
+      mv ~/.ssh/id_rsa.pub.bak ~/.ssh/id_rsa.pub
+    fi
+
+    if [ -f ~/.ssh/sign ]
+    then
+      rm -f ~/.ssh/sign
+    fi
+    if [ -f ~/.ssh/sign.bak ]
+    then
+      mv ~/.ssh/sign.bak ~/.ssh/sign
+    fi
+    if [ -f ~/.ssh/sign.pub ]
+    then
+      rm -f ~/.ssh/sign.pub
+    fi
+    if [ -f ~/.ssh/sign.pub.bak ]
+    then
+      mv ~/.ssh/sign.pub.bak ~/.ssh/sign.pub
+    fi
+
+    gpg --delete-secret-key --batch --yes "$launchpad_sign_public_key_fingerprint" || true
+    gpg --delete-key --batch --yes "$launchpad_sign_public_key_fingerprint" || true
+
+    rm -f ~/.config/copr
+}
+
+trap finish SIGINT SIGTERM EXIT
+
+
+echo "Building binary package"
+
+mkdir -p ~/.aws
+echo "
+[default]
+aws_access_key_id=$aws_access_key_id
+aws_secret_access_key=$aws_secret_access_key
+region=eu-west-1
+output=json
+" > ~/.aws/config
+
+set -x
+mvn package -f main/pom.xml -P assembler -Ds3.push=true -DskipTests
+set +x
+
+
+echo "Building docker package"
+
+mkdir -p ~/.m2
+if [ -f ~/.m2/settings.xml ]
+then
+  mv ~/.m2/settings.xml ~/.m2/settings.xml.bak
+fi
+echo "
+<settings>
+  <servers>
+    <server>
+      <id>docker.io</id>
+      <username>$docker_username</username>
+      <password>$docker_password</password>
+    </server>
+  </servers>
+</settings>
+" > ~/.m2/settings.xml
+
+set -x
+mvn package -f main/pom.xml -P docker -Ddocker.skip.push=false -DskipTests
+set +x
+
+
+echo "Building deb package"
+
+if [ -f ~/.ssh/id_rsa ]
+then
+  mv ~/.ssh/id_rsa ~/.ssh/id_rsa.bak
+fi
+if [ -f ~/.ssh/id_rsa.pub ]
+then
+  mv ~/.ssh/id_rsa.pub ~/.ssh/id_rsa.pub.bak
+fi
+
+echo "$launchpad_private_key" > ~/.ssh/id_rsa
+echo "$launchpad_public_key" > ~/.ssh/id_rsa.pub
+chmod 600 ~/.ssh/id_rsa
+chmod 644 ~/.ssh/id_rsa.pub
+
+echo "$launchpad_sign_private_key" > ~/.ssh/sign
+echo "$launchpad_sign_public_key" > ~/.ssh/sign.pub
+chmod 600 ~/.ssh/sign
+chmod 644 ~/.ssh/sign.pub
+
+gpg --import ~/.ssh/sign.pub
+gpg --import ~/.ssh/sign
+echo "$(echo "$launchpad_sign_public_key_fingerprint"|tr -d '[:space:]'):6:"|gpg --import-ownertrust
+
+echo "
+[8kdata-release]
+fqdn = ppa.launchpad.net
+method = ftp
+incoming = ~8kdata/ubuntu/ppa/
+login = anonymous
+allow_unsigned_uploads = 0
+
+[8kdata-devel]
+fqdn = ppa.launchpad.net
+method = ftp
+incoming = ~8kdata/ubuntu/ppa-dev/
+login = anonymous
+allow_unsigned_uploads = 0
+" > ~/.dput.cf
+
+set -x
+mvn package -f main/pom.xml -P deb -Dlaunchpad.push=true -DskipTests -Dpackage.name=torodb-stampede
+mvn package -f main/pom.xml -P deb -Dlaunchpad.push=true -DskipTests -Dpackage.name=torodb-stampede-postgres
+set +x
+
+
+echo "Building snap package"
+
+set -x
+mvn package -f main/pom.xml -P snap -Dlaunchpad.push=true -DskipTests -Dpackage.name=torodb-stampede
+mvn package -f main/pom.xml -P snap -Dlaunchpad.push=true -DskipTests -Dpackage.name=torodb-stampede-postgres
+set +x
+
+
+echo "Building rpm package"
+
+mkdir -p ~/.config
+echo "
+[copr-cli]
+login = $copr_login
+username = $copr_user
+token = $copr_token
+copr_url = https://copr.fedorainfracloud.org
+" > ~/.config/copr
+
+set -x
+mvn package -f main/pom.xml -P rpm -Dcopr.push=true -DskipTests -Dpackage.name=torodb-stampede
+mvn package -f main/pom.xml -P rpm -Dcopr.push=true -DskipTests -Dpackage.name=torodb-stampede-postgres
+set +x

README.md

@@ -1,43 +1,59 @@
-# ToroDB
+# ToroDB Stampede
 
-[![Master branch build status](https://travis-ci.org/torodb/torodb.svg?branch=master)](https://travis-ci.org/torodb/torodb) [![Quality Gate](https://sonarqube.com/api/badges/gate?key=com.torodb:torodb-pom)](https://sonarqube.com/dashboard/index/com.torodb:torodb-pom)
+> Transform your NoSQL data from a MongoDB replica set into a relational database in PostgreSQL.
 
-ToroDB is a technology designed to fulfill the gap between document oriented
-and SQL databases. There are two products that use this technology: ToroDB
-Server and ToroDB Stampede. Both platforms are open source and any feedback,
-contributions, help and/or patches are very welcome. Please join the
-[torodb-dev][2] mailing list for further discussion.
+There are other solutions that are able to store the JSON document in a 
+relational table using PostgreSQL JSON support, but it doesn't solve the real 
+problem of 'how to really use that data'. ToroDB Stampede replicates the 
+document structure in different relational tables and stores the document data
+in different tuples using those tables.
 
-For more information, please see [ToroDB's website][1]
+![](documentation/docs/images/tables_distribution.jpeg)
 
-## ToroDB Server
-It is a MongoDB-compatible server that supports speaks the MongoDB Wire 
-Protocol (and therefore can be used with the same drivers used to connect to 
-any standard MongoDB server) but stores your data into a reliable and trusted 
-ACID database. 
+## Installation
 
-More information about ToroDB Server can be found on [its own folder](/server)
-in this repository.
+Due to the use of different external systems like MongoDB and PostgreSQL, the
+installation requires some previous steps. Take a look at out 
+[quickstart][1] in the
+documentation.
 
-## ToroDB Stampede
-ToroDB Stampede is a business analytic solution that replicates your data in 
-real time from a MongoDB replica set into a SQL database, allowing you to use
-any business intelligence products (like [Tableau][3] or [Pentaho][4]) to 
-analyze NoSQL data.
+## Usage example
 
-More information about ToroDB Stampede can be found on 
-[its own folder](/stampede) in this repository.
+MongoDB is a great idea, but sooner or later some kind of business 
+intelligence, or complex aggregated queries are required. At this point MongoDB
+is not so powerful and ToroDB Stampede borns to solve that problem (see 
+[our post about that][2]).
 
-## Code QA
- * Master branch build status: [![Master branch build status](https://travis-ci.org/torodb/torodb.svg?branch=master)](https://travis-ci.org/torodb/torodb) [![Quality Gate](https://sonarqube.com/api/badges/gate?key=com.torodb:torodb-pom)](https://sonarqube.com/dashboard/index/com.torodb:torodb-pom)
- * Devel branch build status :  [![Devel branch build status](https://travis-ci.org/torodb/torodb.svg?branch=devel)](https://travis-ci.org/torodb/torodb) [![Quality Gate](https://sonarqube.com/api/badges/gate?key=com.torodb:torodb-pom:devel)](https://sonarqube.com/dashboard/index/com.torodb:torodb-pom:devel)
+The kind of replication done by ToroDB Stampede allows the execution of 
+aggregated queries in a relational backend (PostgreSQL) with a noticeable time 
+improvement.
 
-## Are you a developer? Want to contribute? Questions about the source code?
+A deeper explanation is available in our 
+[how to use][3] section in the 
+documentation.
 
-Please see [CONTRIBUTING][5].
+## Development setup
 
-[1]: http://www.torodb.com
-[2]: https://groups.google.com/forum/#!forum/torodb-dev
-[3]: http://www.tableau.com
-[4]: http://www.pentaho.com/
-[5]: https://github.com/torodb/torodb/blob/master/CONTRIBUTING.md
+As it was said in the installation section, the requirements of external 
+systems can make more difficult to explain briefly how to setup the development 
+environment here. So if you want to take a look on how to prepare your 
+development environment, take a look to our 
+[documentation][4].
+
+## Release History
+
+* 1.0.0-beta2
+    * Released on April 06th 2017
+* 1.0.0-beta1
+    * Released on December 30th 2016
+
+## Meta
+
+ToroDB – [@nosqlonsql](https://twitter.com/nosqlonsql) – info@8kdata.com
+
+Distributed under the GNU AGPL v3 license. See ``LICENSE`` for more information.
+
+[1]: https://www.torodb.com/stampede/docs/quickstart
+[2]: https://www.8kdata.com/blog/the-conundrum-of-bi-aggregate-queries-on-mongodb/
+[3]: https://www.torodb.com/stampede/docs/how-to-use
+[4]: https://www.torodb.com/stampede/docs/installation/previous-requirements/

RELEASE.md

@@ -0,0 +1,29 @@
+## Release Notes for Stampede 1.0.0-beta2
+
+### Changes
+
+* Add support for MongoDB 3.4
+    * Deal with MongoDB views
+    * Support BSON Type Decimal128
+* Support Sharding Replication. ToroDB Stampede can replicate from N shards into the same ToroDB database.
+    *  Adjust Guice to provide a better way to inject different metrics and loggers
+    *  Adapt metrics and logging so each shard has their own values
+    *  Adapt the Data Import Mode to the sharding model
+* Stampede Packaging
+    * RPM package
+    * DEB package
+    * Snap package
+* Allow SSL connection to the backend (PostgreSQL)
+* Add FlexyPool to ToroDB
+* Integration Tests
+* Support all BSON types  
+* Deal with system collections
+* Unify logging system and improve error messages
+* Calibrate maximum threads using also connection pool size
+* Review and test Windows/Mac installation/configuration documentation
+* Improve ToroDB Parameter configuration
+
+
+### Bugs Fixed
+
+* Stampede did not support documents whose '\_id' is a container (object or array)