Merge branch 'integration'

* integration:
  Fixed the path of the generated ice.properties file in the instructions shown at the end of the install.sh script
  Do not fail in install.sh if the "message.properties" file has already been deleted
  Do not fail in install.sh if the local work directories already exists
  Force the install.sh script to exit if an error occur
  Add a syntax validation on install.sh in the Travis CI testing process
  Fixed OS detection in install.sh
  Add support for localized billing accounts
  Fix Grails version in URL
  Update README with reference to ice.config.location
  Allow extra configuration file
  Add us-east-1f support
  minor fixes
  Highstock CDN supports HTTPS now, let's use it by default
  Continue grouping resources even if ResourceId is missing
  Parse services with '/'. Fixes #100
  Upgrade to Grails 2.4.4

.travis.yml

@@ -11,4 +11,5 @@ install:
   - yes | ./grailsw refresh-dependencies
 
 script: 
+  - bash -n install.sh
   - ./grailsw compile

README.md

@@ -126,7 +126,7 @@ Using basic setup, you don't need any extra code change and you will use the pro
           # Specify your currency conversion rate here. The default value is 1. If 1 pound = 1.5 dollar, then the rate is 0.6666667.
           ice.currencyRate=0.6666667
 
-  2.8 By default, Ice pulls in [Highstock](http://www.highcharts.com/) from its CDN. This CDN, unfortunately, does not (at present) support HTTPS. If you're serving Ice over HTTPS, your browser shouldn't or won't download Highstock from there. To fix this, you can serve Highstock somewhere else, and set this property:
+  2.8 By default, Ice pulls in [Highstock](https://www.highcharts.com/) from its CDN.
 
           ice.highstockUrl=https://example.com/js/highstock.js
 
@@ -263,6 +263,12 @@ Options with * require writing your own code.
 
         ice.use_blended=true
 
+10. Extra Grails configuration file
+
+  If you need to setup custom Grails settings, you can specify an additional configuration file to be loaded by Grails by setting the ``ice.config.location`` system property to the location of that file.
+
+  See http://docs.grails.org/2.4.4/guide/single.html#configExternalized for more information.
+
 ## Example IAM Permissions
 
 Grant the following permissions to either an instance role, or the user running the reports:

grails-app/conf/Config.groovy

@@ -17,13 +17,11 @@
 // locations to search for config files that get merged into the main config
 // config files can either be Java properties files or ConfigSlurper scripts
 
-grails.config.locations = [
+grails.config.locations = []
 
-]
-
-// if(System.properties["${appName}.config.location"]) {
-//    grails.config.locations << "file:" + System.properties["${appName}.config.location"]
-// }
+if(System.properties["${appName}.config.location"]) {
+    grails.config.locations << "file:" + System.properties["${appName}.config.location"]
+}
 
 grails.project.groupId = appName // change this to alter the default package name and Maven publishing destination
 grails.mime.file.extensions = true // enables the parsing of file extensions from URLs into the request format

install.sh

@@ -1,18 +1,27 @@
 #!/bin/bash
 
+# Exit the script if an error occur
+set -e
+
 # Get Ice in a ready-to-run state on a fresh AWI instance.
 
-GRAILS_VERSION=2.2.1
+GRAILS_VERSION=2.4.4
 
 # Install prerequisites
-
 if [ -f /etc/redhat-release ]; then
     echo "Installing redhat packages"
-    sudo yum -y install git java-1.6.0-openjdk-devel.x86_64 wget unzip
-else
-   [ -f /etc/debian-release ];
+    sudo yum -y install git java-1.7.0-openjdk-devel.x86_64 wget unzip
+
+elif [[ -f /etc/debian-release || -f /etc/debian_version ]];then
     echo "Installing debian packages"
-    sudo apt-get -y install git openjdk-6-jdk wget unzip
+    sudo apt-get -y install git openjdk-7-jdk wget unzip
+
+elif [[ -f /etc/issue && $(grep "Amazon Linux AMI" /etc/issue) ]]; then
+    echo "Assuming AWS AMI, installing packages"
+    sudo yum -y install git java-1.7.0-openjdk-devel.x86_64 wget unzip
+
+else 
+    echo "Unknown operating system. You may have to install Java 7 manually."
 fi
 
 INSTALL_DIR=$(pwd)
@@ -53,11 +62,11 @@ fi
 grails ${JAVA_OPTS} wrapper
 
 # (Bug: Ice can't deal with this file existing and being empty.)
-rm grails-app/i18n/messages.properties
+rm -f grails-app/i18n/messages.properties
 
 # Create our local work directories (both for processing and reading)
-mkdir ${HOME_DIR}/ice_processor
-mkdir ${HOME_DIR}/ice_reader
+mkdir -p ${HOME_DIR}/ice_processor
+mkdir -p ${HOME_DIR}/ice_reader
 
 # Set up the config file
 cp src/java/sample.properties src/java/ice.properties
@@ -74,9 +83,9 @@ do
   echo -n "-> "
   read -r PROCBUCKET
 done
-sed -rie 's/=billing_s3bucketprefix\//=/; s|\/mnt\/|'"${HOME_DIR}"'\/|; s/=work_s3bucketprefix\//=/; s/^ice.account.*//; s/=billing_s3bucketname1/='${BILLBUCKET}'/; s/=work_s3bucketname/='${PROCBUCKET}'/' src/java/ice.properties
+sed -ri 's/=billing_s3bucketprefix\//=/; s|\/mnt\/|'"${HOME_DIR}"'\/|; s/=work_s3bucketprefix\//=/; s/^ice.account.*//; s/=billing_s3bucketname1/='${BILLBUCKET}'/; s/=work_s3bucketname/='${PROCBUCKET}'/' src/java/ice.properties
 
 echo Ice is now ready to run as a processor. If you want to run the reader, edit:
-echo ~/ice/src/java/ice.properties
+echo "${INSTALL_DIR}/src/java/ice.properties"
 echo and alter the appropriate flags. You can now start Ice by running the following from the Ice root directory:
 echo ./grailsw -Djava.net.preferIPv4Stack=true -Dice.s3AccessKeyId=\<access key ID\> -Dice.s3SecretKey=\<access key\> run-app

src/java/com/netflix/ice/basic/BasicDataManager.java

@@ -65,7 +65,7 @@ public ReadOnlyData load(DateTime monthDate) throws Exception {
     public BasicDataManager(Product product, ConsolidateType consolidateType, boolean isCost) {
         this.product = product;
         this.consolidateType = consolidateType;
-        this.dbName = (isCost ? "cost_" : "usage_") + consolidateType + "_" + (product == null ? "all" : product.name);
+        this.dbName = (isCost ? "cost_" : "usage_") + consolidateType + "_" + (product == null ? "all" : product.s3Name);
 
         start(300);
     }

src/java/com/netflix/ice/basic/BasicLineItemProcessor.java

@@ -204,7 +204,7 @@ else if (result == Result.monthly) {
         }
 
         double resourceCostValue = costValue;
-        if (items.length > resourceIndex && !StringUtils.isEmpty(items[resourceIndex]) && config.resourceService != null) {
+        if (items.length > resourceIndex && config.resourceService != null) {
 
             if (config.useCostForResourceGroup.equals("modeled") && product == Product.ec2_instance)
                 operation = Operation.getReservedInstances(config.reservationService.getDefaultReservationUtilization(0L));

src/java/com/netflix/ice/basic/BasicManagers.java

@@ -25,6 +25,7 @@
 import com.netflix.ice.processor.TagGroupWriter;
 import com.netflix.ice.reader.*;
 import com.netflix.ice.tag.Product;
+import com.netflix.ice.tag.Tag;
 
 import java.util.Collection;
 import java.util.Map;
@@ -99,6 +100,7 @@ private void doWork() {
             }
             else {
                 String name = key.substring((config.workS3BucketPrefix + TagGroupWriter.DB_PREFIX).length());
+                name = Tag.fromS3(name);
                 product = config.productService.getProductByName(name);
             }
             if (!products.contains(product)) {

src/java/com/netflix/ice/basic/BasicTagGroupManager.java

@@ -48,7 +48,7 @@ public class BasicTagGroupManager extends Poller implements TagGroupManager {
     private Interval totalInterval;
 
     BasicTagGroupManager(Product product) {
-        this.dbName = TagGroupWriter.DB_PREFIX + (product == null ? "all" : product.name);
+        this.dbName = TagGroupWriter.DB_PREFIX + (product == null ? "all" : product.s3Name);
         file = new File(config.localDir, dbName);
         try {
             poll();

src/java/com/netflix/ice/common/AwsUtils.java

@@ -49,9 +49,9 @@
  */
 public class AwsUtils {
     private final static Logger logger = LoggerFactory.getLogger(AwsUtils.class);
-    private static Pattern billingFileWithTagsPattern = Pattern.compile(".+-aws-billing-detailed-line-items-with-resources-and-tags-(\\d\\d\\d\\d-\\d\\d).csv.zip");
-    private static Pattern billingFileWithMonitoringPattern = Pattern.compile(".+-aws-billing-detailed-line-items-with-monitoring-(\\d\\d\\d\\d-\\d\\d).csv");
-    private static Pattern billingFilePattern = Pattern.compile(".+-aws-billing-detailed-line-items-(\\d\\d\\d\\d-\\d\\d).csv.zip");
+    private static Pattern billingFileWithTagsPattern = Pattern.compile(".+-aws-billing-detailed-line-items-with-resources-and-tags-(?:[A-Z]+-)?(\\d\\d\\d\\d-\\d\\d).csv.zip");
+    private static Pattern billingFileWithMonitoringPattern = Pattern.compile(".+-aws-billing-detailed-line-items-with-monitoring-(?:[A-Z]+-)?(\\d\\d\\d\\d-\\d\\d).csv");
+    private static Pattern billingFilePattern = Pattern.compile(".+-aws-billing-detailed-line-items-(?:[A-Z]+-)?(\\d\\d\\d\\d-\\d\\d).csv.zip");
     public static final DateTimeFormatter monthDateFormat = DateTimeFormat.forPattern("yyyy-MM").withZone(DateTimeZone.UTC);
     public static final DateTimeFormatter dayDateFormat = DateTimeFormat.forPattern("yyyy-MM-dd").withZone(DateTimeZone.UTC);
     public static final DateTimeFormatter dateFormatter = DateTimeFormat.forPattern("yyyy-MM-dd HHa").withZone(DateTimeZone.UTC);

src/java/com/netflix/ice/common/IceOptions.java

@@ -40,8 +40,7 @@ public class IceOptions {
     public static final String CURRENCY_RATE = "ice.currencyRate";
 
     /**
-     * The URL of highstock.js. The default value is the Highcharts CDN; change this if you need to
-     * serve it from somewhere else (for example, if you need HTTPS).
+     * The URL of highstock.js. The default value is the Highcharts CDN (HTTPS)
      */
     public static final String HIGHSTOCK_URL = "ice.highstockUrl";
 

src/java/com/netflix/ice/processor/BillingFileProcessor.java

@@ -429,7 +429,7 @@ private void archive() throws Exception {
     private void archiveHourly(Map<Product, ReadWriteData> dataMap, String prefix) throws Exception {
         DateTime monthDateTime = new DateTime(startMilli, DateTimeZone.UTC);
         for (Product product: dataMap.keySet()) {
-            String prodName = product == null ? "all" : product.name;
+            String prodName = product == null ? "all" : product.s3Name;
             DataWriter writer = new DataWriter(prefix + "hourly_" + prodName + "_" + AwsUtils.monthDateFormat.print(monthDateTime), false);
             writer.archive(dataMap.get(product));
         }
@@ -448,7 +448,7 @@ private void archiveSummary(Map<Product, ReadWriteData> dataMap, String prefix)
 
         for (Product product: dataMap.keySet()) {
 
-            String prodName = product == null ? "all" : product.name;
+            String prodName = product == null ? "all" : product.s3Name;
             ReadWriteData data = dataMap.get(product);
             Collection<TagGroup> tagGroups = data.getTagGroups();
 
@@ -758,4 +758,3 @@ private class BillingFile {
         }
     }
 }
-

src/java/com/netflix/ice/processor/DataWriter.java

@@ -19,6 +19,7 @@
 
 import com.netflix.ice.common.AwsUtils;
 import com.netflix.ice.common.TagGroup;
+import com.netflix.ice.tag.Tag;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -38,6 +39,7 @@ public class DataWriter {
 
     DataWriter(String name, boolean loadData) throws Exception {
 
+        name = Tag.toS3(name);
         dbName = name;
         file = new File(config.localDir, dbName);
         if (loadData) {
@@ -81,4 +83,3 @@ void archive(ReadWriteData data) throws IOException {
         logger.info(this.dbName + " uploading done.");
     }
 }
-

src/java/com/netflix/ice/processor/TagGroupWriter.java

@@ -20,6 +20,7 @@
 import com.google.common.collect.Maps;
 import com.netflix.ice.common.AwsUtils;
 import com.netflix.ice.common.TagGroup;
+import com.netflix.ice.tag.Tag;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
@@ -38,6 +39,7 @@ public class TagGroupWriter {
 
     TagGroupWriter(String name) throws Exception {
 
+        name = Tag.toS3(name);
         dbName = DB_PREFIX + name;
         file = new File(config.localDir, dbName);
         logger.info("creating TagGroupWriter for " + file);
@@ -74,4 +76,3 @@ void archive(Long monthMilli,Collection<TagGroup> tagGroups) throws IOException
         logger.info(dbName + " uploading done.");
     }
 }
-

src/java/com/netflix/ice/tag/Tag.java

@@ -28,8 +28,10 @@ public int compareTo(Tag t) {
     };
 
     public final String name;
+    public final String s3Name;
     Tag(String name) {
         this.name = name;
+        this.s3Name = Tag.toS3(name);
     }
 
     @Override
@@ -40,6 +42,22 @@ public boolean equals(Object o) {
             return false;
     }
 
+    /**
+    * Normalize a tagname suitable to be an S3 Filename
+    */
+    public static String toS3(String name) {
+        name = name.replaceAll("/","--");
+        return name;
+    }
+
+    /**
+    * Normalize a tagname from an S3 Filename
+    */
+    public static String fromS3(String name) {
+        name = name.replaceAll("--","/");
+        return name;
+    }
+
     @Override
     public String toString() {
         return this.name;

src/java/com/netflix/ice/tag/Zone.java

@@ -39,6 +39,7 @@ private Zone (Region region, String name) {
     public static final Zone US_EAST_1C = new Zone(Region.US_EAST_1, "us-east-1c");
     public static final Zone US_EAST_1D = new Zone(Region.US_EAST_1, "us-east-1d");
     public static final Zone US_EAST_1E = new Zone(Region.US_EAST_1, "us-east-1e");
+    public static final Zone US_EAST_1F = new Zone(Region.US_EAST_1, "us-east-1f");
 
     public static final Zone US_EAST_2A = new Zone(Region.US_EAST_2, "us-east-2a");
     public static final Zone US_EAST_2B = new Zone(Region.US_EAST_2, "us-east-2b");
@@ -100,6 +101,7 @@ private Zone (Region region, String name) {
         zonesByName.put(US_EAST_1C.name, US_EAST_1C);
         zonesByName.put(US_EAST_1D.name, US_EAST_1D);
         zonesByName.put(US_EAST_1E.name, US_EAST_1E);
+        zonesByName.put(US_EAST_1F.name, US_EAST_1F);
 
         zonesByName.put(US_EAST_2A.name, US_EAST_2A);
         zonesByName.put(US_EAST_2B.name, US_EAST_2B);

src/java/sample.properties

@@ -13,7 +13,7 @@ ice.reservationPeriod=threeyear
 # default reservation utilization, possible values are LIGHT, MEDIUM, HEAVY. If you have both (LIGHT or MEDIUM) and HEAVY RIs, make sure you do not put HEAVY here.
 ice.reservationUtilization=MEDIUM
 
-# the highstock url; host it somewhere else and change this if you need HTTPS
+# the highstock url
 ice.highstockUrl=https://code.highcharts.com/stock/4.2.1/highstock.js
 
 # url prefix, e.g. http://ice.netflix.com/. Will be used in alert emails.