use code generator for EC2 APIs

gen/gen.jl

@@ -1,14 +1,18 @@
 using Compat
 using LightXML
 
+# define placeholder types
+type AWSEnv end
+
 include("../src/codegen.jl")
 
 const DIR = dirname(@__FILE__)
 const SRCDIR = joinpath(DIR, "..", "src")
 
 const SPEC_NAMES = [
     "AutoScaling",
-    "SQS"
+    "SQS",
+    "EC2"
 ]
 
 function do_gen(spec_name::String, includes::Vector{String}, out_types_file::String, out_operations_file::String)

gen/specgen.jl

@@ -16,15 +16,45 @@ using Requests
 
 function attr_type(elem)
     txt = nodeText(elem)
-    is_array = contains(txt, "Array")
+    is_array = contains(txt, "Array") || contains(txt, "array")
     parts = split(txt)
-    !is_array && (return String(parts[2]))
-    return "Vector{$(parts[4])}"
+    elem_type = is_array ? parts[4] : parts[2]
+    if lowercase(elem_type) == "boolean"
+        elem_type = "Bool"
+    elseif lowercase(elem_type) in ("string", "strings")
+        elem_type = "String"
+    elseif lowercase(elem_type) in ("integer", "long", "integers", "longs")
+        elem_type = "Int"
+    elseif lowercase(elem_type) in ("float", "floats", "double", "doubles")
+        elem_type = "Float64"
+    elseif lowercase(elem_type) == "base64-encoded"
+        elem_type = "Vector{UInt8}"
+    elseif lowercase(elem_type) == "timestamp"
+        elem_type = "Base.Dates.DateTime"
+    else
+        elem_type = String(elem_type)
+    end
+    is_array ? "Vector{$(elem_type)}" : elem_type
+end
+
+function attr_name(elem)
+    elem_name = nodeText(elem)
+    if endswith(elem_name, ".N")
+        elem_name = rsplit(elem_name, '.'; limit=2)[1]
+    elseif contains(elem_name, "(request)") && contains(elem_name, "(response)") && contains(elem_name, ",")
+        for name in split(elem_name, ','; limit=2)
+            if contains(name, "(request)")
+                elem_name = split(name)[1]
+                break
+            end
+        end
+    end
+    String(elem_name)
 end
 
 function parse_type_attrs(elem)
     qs = matchall(Selector(".term"), elem)
-    attr_names = map(nodeText, qs)
+    attr_names = map(attr_name, qs)
 
     qs = matchall(Selector("p:contains(\"Type:\")"), elem)
     attr_types = map(attr_type, qs)
@@ -47,11 +77,22 @@ function parse_type_id(elem)
     qs[1].attributes["id"]
 end
 
+function retry_get(url)
+    while true
+        try
+            return get(url)
+        catch ex
+            warn("will retry ", url, " - error: ", ex)
+            sleep(5)
+        end
+    end
+end
+
 """
 Generate type specification for an individual type.
 """
 function gen_type(url)
-    r = get(url)
+    r = retry_get(url)
     h = parsehtml(bytestring(r.data))
 
     typename = parse_type_name(h.root)
@@ -69,7 +110,7 @@ function gen_types(url)
     all_types = Vector{Pair{String,Vector{Pair{String,String}}}}()
 
     info("type list: ", url)
-    r = get(url)
+    r = retry_get(url)
     h = parsehtml(bytestring(r.data))
 
     qs = matchall(Selector(".listitem > p > a"), h.root)
@@ -89,7 +130,7 @@ end
 Generate API specification for an individual operation.
 """
 function gen_api(url)
-    r = get(url)
+    r = retry_get(url)
     h = parsehtml(bytestring(r.data))
 
     apiname = parse_type_name(h.root)
@@ -122,7 +163,7 @@ function gen_apis(url)
     all_apis = Dict()
 
     info("api list: ", url)
-    r = get(url) 
+    r = retry_get(url)
     h = parsehtml(bytestring(r.data))
 
     qs = matchall(Selector(".listitem > p > a"), h.root)

src/AWS.jl

@@ -231,7 +231,7 @@ include("codegen.jl")
 include("aws_utils.jl")
 include("crypto.jl")
 include("sign.jl")
-include("EC2.jl")
+include("ec2.jl")
 include("S3.jl")
 include("sqs.jl")
 include("autoscaling.jl")

src/codegen.jl

@@ -38,7 +38,13 @@ is_basic_type{T<:BASIC_TYPES}(v::T) = true
 is_basic_type(v) = false
 
 _ename(name::Symbol) = _ename(string(name))
-_ename(name::String) = lcfirst(name)
+function _ename(name::String)
+    name = lcfirst(name)
+    if name in ("type", "return", "end")
+        name = "_" * name
+    end
+    name
+end
 tbasename{T<:AbstractAWSType}(obj::T) = tbasename(T)
 tbasename{T}(::Type{T}) = rsplit(string(T), "."; limit=2)[end]
 
@@ -105,6 +111,8 @@ function _parse_from_xml{T<:AbstractAWSType}(::Type{T}, types::Dict{Symbol,Vecto
         (pd_type === nothing) && (pd_type = pd)
         for (elemname,elemtype) in typeprops
             ename = string(elemname)
+            # handle field names that match julia reserved types
+            startswith(ename, "_") && (ename = ename[2:end])
             if ((elemtype <: Dict) || ((elemtype <: Vector) && !(elemtype <: Vector{UInt8}))) && isempty(member_pfx)
                 elem = LightXML.get_elements_by_tagname(pd_type, ename)
             else

src/ec2.jl

@@ -0,0 +1,28 @@
+module EC2
+
+using LightXML
+using AWS.Crypto
+using Requests
+using AWS.AWSEnv
+using AWS
+using Compat
+
+import AWS: AbstractAWSType, AWSError, AWSResponse, _ename, _parse_from_xml, _prep_params, aws_execute
+
+const EC2Error = AWSError
+const EC2Response = AWSResponse
+
+parse_from_xml{T<:AbstractAWSType}(::Type{T}, types::Dict{Symbol,Vector{Pair{Symbol,DataType}}}, pd::Union{LightXML.XMLElement,Void}) = _parse_from_xml(T, types, pd; member_pfx="item")
+prep_params{T<:AbstractAWSType}(obj::T) = _prep_params(obj; member_pfx="")
+
+ec2_execute(env::AWSEnv, action::String, params::Vector{Tuple}=Vector{Tuple}()) = aws_execute(env, action, "ec2", EC2ApiVer, params)
+ec2_execute(env::AWSEnv, action::String, req::AbstractAWSType) = ec2_execute(env, action, prep_params(req))
+ec2_execute(env::AWSEnv, action::String, req::Void) = ec2_execute(env, action)
+
+include("ec2_types.jl")
+include("ec2_operations.jl")
+include("ec2_spec.jl")
+
+export EC2Error, EC2Response
+
+end # module EC2