diff --git a/Gruntfile.js b/Gruntfile.js index 769bbaa66..f1e2595f1 100644 --- a/Gruntfile.js +++ b/Gruntfile.js @@ -3,10 +3,8 @@ module.exports = function(grunt) { pkg: grunt.file.readJSON('package.json'), tag: { banner: '/* <%= pkg.name %>\n' + - ' * @version <%= pkg.version %>\n' + - ' * @author <%= pkg.author %>\n' + + ' * version <%= pkg.version %>\n' + ' * Project: <%= pkg.homepage %>\n' + - ' * Copyright <%= pkg.year %>. <%= pkg.license %> licensed.\n' + ' */\n' }, copy: { @@ -19,7 +17,7 @@ module.exports = function(grunt) { }, clean: { build: { - src: ['dist/viewer'] + src: ['dist'] } }, autoprefixer: { @@ -58,10 +56,7 @@ module.exports = function(grunt) { }], options: { banner: '<%= tag.banner %>', - sourceMap: true, - sourceMapName: function(filePath) { - return filePath + '.map'; - } + sourceMap: true } } }, diff --git a/README.md b/README.md index 485cc97cc..8c92d882a 100644 --- a/README.md +++ b/README.md @@ -4,11 +4,11 @@ This JS web app can be easily configured or used as a boilerplate/starting point for basic viewers. It also demonstrates best practices for modular design and OOP via classes in JS using dojo's great [declare](http://dojotoolkit.org/reference-guide/1.9/dojo/_base/declare.html) system. +![screen shot 2014-08-20 at 9 59 48 pm](https://cloud.githubusercontent.com/assets/661156/3991302/5aa2e0f2-28df-11e4-94d0-9c813937d933.png) ## Demo Site [http://davidspriggs.github.io/ConfigurableViewerJSAPI/viewer](http://davidspriggs.github.io/ConfigurableViewerJSAPI/viewer) -Note: Not all functions work in the demo site due to a limitation in GitHub project hosting (no functioning proxy page). ## Installation: * Download the latest release [here](https://github.com/DavidSpriggs/ConfigurableViewerJSAPI/releases). @@ -18,10 +18,10 @@ Note: Not all functions work in the demo site due to a limitation in GitHub proj * Enjoy! ## Customize: -* Use the ConfigurableViewerJSAPI\js\config\viewer.js file to customize your own map layers, task urls and widgets. -* Use the [wiki](https://github.com/DavidSpriggs/ConfigurableViewerJSAPI/wiki) documentation for guidance on configuring widgets. +* Use the `ConfigurableViewerJSAPI\js\config\viewer.js` file to customize your own map layers, task urls and widgets. +* Use the [wiki](https://github.com/DavidSpriggs/ConfigurableViewerJSAPI/wiki) documentation for guidance on configuring individual widgets. -## Widgets Included: +# Widgets Included: * Base Maps * Bookmarks * Directions @@ -30,49 +30,53 @@ Note: Not all functions work in the demo site due to a limitation in GitHub proj * Find * Geocoder * Growler -* Help Button +* Help * Home -* Identify (for dynamic layers) +* Identify * Legend * Locate Button (Geolocation) * Measure * Overview Map -* Print (Advanced) +* Print * Scalebar * StreetView * Table of contents -* If there is a feature you would like to request, add it to the projects [trello board](https://trello.com/b/TjjipGmV/configurable-map-viewer) for consideration. +* Find +* Map Right click menu with various widget functions. +* Highly configurable UI, right or left sidebars with widgets in both, top and bottom regions for other content. + +## Proposing Features +If there is a feature you would like to request, add it to the [issues list](https://github.com/DavidSpriggs/ConfigurableViewerJSAPI/issues) for consideration. ## Change log: See [releases](https://github.com/DavidSpriggs/ConfigurableViewerJSAPI/releases) for change logs. -## IRC +# Community We have an IRC channel: `#cmv` on freenode for the project. If you have questions, stop on by. I recommend [HexChat](http://hexchat.github.io) as an IRC client or you can use freenode's [webchat](http://webchat.freenode.net) client. -## Contributing to the project +### Contributing to the project There are many ways to contribute: 1. Contribute code as widgets (see below). 2. Created documentation in [the wiki](https://github.com/DavidSpriggs/ConfigurableViewerJSAPI/wiki). -3. Submit issues you find the the [issue log](https://github.com/DavidSpriggs/ConfigurableViewerJSAPI/issues?state=open). -4. Vote, comment on, and submit ideas for things to build and improvements to the viewer in the [trello board](https://trello.com/b/TjjipGmV/configurable-map-viewer). +3. Submit issues you find in the [issues log](https://github.com/DavidSpriggs/ConfigurableViewerJSAPI/issues?state=open). ### Grunt tasks This project uses grunt to automate tasks like minifying css and js as well as js linting and css prefixing. ### To get started setup you dev machine: - Install [node](http://nodejs.org). -- Install the grunt cli (command line interface) globally from the command line with : `npm install -g grunt-cli` this only needs to be done once per dev machine. -- Install jshint globally from the command line with : `npm install -g jshint` this only needs to be done once per dev machine. +- Install the grunt cli (command line interface) globally from the command line with : `npm install -g grunt-cli`, this only needs to be done once per dev machine. +- Install jshint globally from the command line with : `npm install -g jshint`, this only needs to be done once per dev machine. ### Get the code and install dev dependencies: - Fork the repo into your own github account. -- Clone your fork and in the repos directory: -- Install the local dev dependencies for the project in the repo from the command line: `npm install` This only needs to be done once per dev machine. -- Run grunt from the repo with: `grunt` this will lint your js as you code. +- Clone your fork and in cloned directory: +- Install the local dev dependencies for the project in the repo from the command line: `npm install`, this only needs to be done once per dev machine. +- Run grunt from the repo with: `grunt` this will launch a mini dev server and lint your js as you code. - Run grunt from the repo with: `grunt build` this will create a `dist` folder with minified code ready for deployment. -- There are other grunt tasks use: `grunt -h` to see a list +- There are other grunt tasks, use: `grunt -h` to see a list. -## License +# License MIT diff --git a/package.json b/package.json index f9dfcb6e2..3eb9ff12e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { - "name": "ConfigurableViewerJSAPI", - "version": "v1.0.1", + "name": "ConfigurableMapViewerCMV", + "version": "1.2.0", "author": "David Spriggs", "license": "MIT", "year": "2014", diff --git a/resource-proxy.zip b/resource-proxy.zip new file mode 100644 index 000000000..4e1279876 Binary files /dev/null and b/resource-proxy.zip differ diff --git a/viewer/proxy/PROXY_README.md b/viewer/proxy/PROXY_README.md new file mode 100755 index 000000000..371a189b3 --- /dev/null +++ b/viewer/proxy/PROXY_README.md @@ -0,0 +1,96 @@ +DotNet Proxy File +================= + +A .NET proxy that handles support for +* Accessing cross domain resources +* Requests that exceed 2048 characters +* Accessing resources secured with token based authentication. +* [OAuth 2.0 app logins](https://developers.arcgis.com/en/authentication). +* Enabling logging +* Both resource and referer based rate limiting + +##Instructions + +* Download and unzip the .zip file or clone the repository. You can download [a released version](https://github.com/Esri/resource-proxy/releases) (recommended) or the [most recent daily build](https://github.com/Esri/resource-proxy/archive/master.zip). +* Install the contents of the DotNet folder as a .NET Web Application, specifying a .NET 4.0 application pool or later +* Test that the proxy is able to forward requests directly in the browser using: +``` +http://[yourmachine]/DotNet/proxy.ashx?http://services.arcgisonline.com/ArcGIS/rest/services/?f=pjson +``` +* Edit the proxy.config file in a text editor to set up your proxy configuration settings. +* Update your application to use the proxy for the specified services. In this JavaScript example requests to route.arcgis.com will utilize the proxy. + +``` + urlUtils.addProxyRule({ + urlPrefix: "route.arcgis.com", + proxyUrl: "http://[yourmachine]/proxy/proxy.ashx" + }); +``` +* Security tip: By default, the proxy.config allows any referrer. To lock this down, replace the ```*``` in the ```allowedReferers``` property with your own application URLs. + +##Proxy Configuration Settings + +* Use the ProxyConfig tag to specify the following proxy level settings. + * **mustMatch="true"** : When true only the sites listed using serverUrl will be proxied. Set to false to proxy any site, which can be useful in testing. However, we recommend setting it to "true" for production sites. + * **allowedReferers="http://server.com/app1,http://server.com/app2"** : A comma-separated list of referer URLs. Only requests coming from referers in the list will be proxied. +* Add a new \ entry for each service that will use the proxy. The proxy.config allows you to use the serverUrl tag to specify one or more ArcGIS Server services that the proxy will forward requests to. The serverUrl tag has the following attributes: + * **url**: Location of the ArcGIS Server service (or other URL) to proxy. Specify either the specific URL or the root (in which case you should set matchAll="false"). + * **matchAll="true"**: When true all requests that begin with the specified URL are forwarded. Otherwise, the URL requested must match exactly. + * **username**: Username to use when requesting a token - if needed for ArcGIS Server token based authentication. + * **password**: Password to use when requesting a token - if needed for ArcGIS Server token based authentication. + * **clientId**. Used with clientSecret for OAuth authentication to obtain a token - if needed for OAuth 2.0 authentication. **NOTE**: If used to access hosted services, the service(s) must be owned by the user accessing it, (with the exception of credit-based esri services, e.g. routing, geoenrichment, etc.) + * **clientSecret**: Used with clientId for OAuth authentication to obtain a token - if needed for OAuth 2.0 authentication. + * **oauth2Endpoint**: When using OAuth 2.0 authentication specify the portal specific OAuth 2.0 authentication endpoint. The default value is https://www.arcgis.com/sharing/oauth2/. + * **accessToken**: OAuth2 access token to use instead of on-demand access-token generation using clientId & clientSecret. + * **rateLimit**: The maximum number of requests with a particular referer over the specified **rateLimitPeriod**. + * **rateLimitPeriod**: The time period (in minutes) within which the specified number of requests (rate_limit) sent with a particular referer will be tracked. The default value is 60 (one hour). + +Note: Refresh the proxy application after updates to the proxy.config have been made. + +Example of proxy using application credentials and limiting requests to 10/minute +``` + + +``` +Example of a tag for a resource which does not require authentication +``` + + +``` +Note: You may have to refresh the proxy application after updates to the proxy.config have been made. + +##Folders and Files + +The proxy consists of the following files: +* proxy.config: This file contains the configuration settings for the proxy. This is where you will define all the resources that will use the proxy. After updating this file you might need to refresh the proxy application using IIS tools in order for the changes to take effect. +* **Important note:** In order to keep your credentials safe, ensure that your web server will not display the text inside your proxy.config in the browser (ie: http://[yourmachine]/proxy/proxy.config). +* proxy.ashx: The actual proxy application. In most cases you will not need to modify this file. +* web.config: An XML file that stores ASP.NET configuration data. Use this file to configure logging for the proxy. By default the proxy will write log messages to a file named auth_proxy.log located in 'C:\Temp\Shared\proxy_logs'. Note that the folder location needs to exist in order for the log file to be successfully created. +##Requirements + +* ASP.NET 4.0 or greater (4.5 is required on Windows 8/Server 2012, see [this article] (http://www.iis.net/learn/get-started/whats-new-in-iis-8/iis-80-using-aspnet-35-and-aspnet-45) for more information) + +##Issues + +Found a bug or want to request a new feature? Let us know by submitting an issue. + +##Contributing + +All contributions are welcome. + +##Licensing + +Copyright 2014 Esri + +Licensed under the Apache License, Version 2.0 (the "License"); +You may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" basis, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for specific language governing permissions and limitations under the license. diff --git a/viewer/proxy/Web.config b/viewer/proxy/Web.config new file mode 100755 index 000000000..8d1a40128 --- /dev/null +++ b/viewer/proxy/Web.config @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + diff --git a/viewer/proxy/proxy.ashx b/viewer/proxy/proxy.ashx old mode 100644 new mode 100755 index a36effccc..6446e630d --- a/viewer/proxy/proxy.ashx +++ b/viewer/proxy/proxy.ashx @@ -1,281 +1,834 @@ <%@ WebHandler Language="C#" Class="proxy" %> + /* - This proxy page does not have any security checks. It is highly recommended - that a user deploying this proxy page on their web server, add appropriate - security checks, for example checking request path, username/password, target - url, etc. -*/ + * DotNet proxy client. + * + * Version 1.1 beta + * See https://github.com/Esri/resource-proxy for more information. + * + */ + +#define TRACE using System; -using System.Drawing; using System.IO; using System.Web; -using System.Collections.Generic; -using System.Text; using System.Xml.Serialization; using System.Web.Caching; +using System.Collections.Concurrent; +using System.Diagnostics; -/// -/// Forwards requests to an ArcGIS Server REST resource. Uses information in -/// the proxy.config file to determine properties of the server. -/// public class proxy : IHttpHandler { - - public void ProcessRequest (HttpContext context) { - // use the following line to ignore invalid (self signed) ssl certs: - System.Net.ServicePointManager.ServerCertificateValidationCallback = delegate(object s, System.Security.Cryptography.X509Certificates.X509Certificate certificate, System.Security.Cryptography.X509Certificates.X509Chain chain, System.Net.Security.SslPolicyErrors sslPolicyErrors) { return true; }; - + + class RateMeter { + double _rate; //internal rate is stored in requests per second + int _countCap; + double _count = 0; + DateTime _lastUpdate = DateTime.Now; + + public RateMeter(int rate_limit, int rate_limit_period) { + _rate = (double) rate_limit / rate_limit_period / 60; + _countCap = rate_limit; + } + + //called when rate-limited endpoint is invoked + public bool click() { + TimeSpan ts = DateTime.Now - _lastUpdate; + _lastUpdate = DateTime.Now; + //assuming uniform distribution of requests over time, + //reducing the counter according to # of seconds passed + //since last invocation + _count = Math.Max(0, _count - ts.TotalSeconds * _rate); + if (_count <= _countCap) { + //good to proceed + _count++; + return true; + } + return false; + } + + public bool canBeCleaned() { + TimeSpan ts = DateTime.Now - _lastUpdate; + return _count - ts.TotalSeconds * _rate <= 0; + } + } + + private static string PROXY_REFERER = "http://localhost/proxy/proxy.ashx"; + private static string DEFAULT_OAUTH = "https://www.arcgis.com/sharing/oauth2/"; + private static int CLEAN_RATEMAP_AFTER = 10000; //clean the rateMap every xxxx requests + + private static Object _rateMapLock = new Object(); + + public void ProcessRequest(HttpContext context) { HttpResponse response = context.Response; + if (context.Request.Url.Query.Length < 1) + { + string errorMsg = "No URL specified"; + log(TraceLevel.Error, errorMsg); + sendErrorResponse(context.Response, null, errorMsg, System.Net.HttpStatusCode.BadRequest); + return; + } - // Get the URL requested by the client (take the entire querystring at once - // to handle the case of the URL itself containing querystring parameters) string uri = context.Request.Url.Query.Substring(1); - // Get token, if applicable, and append to the request - string token = getTokenFromConfigFile(uri); - if (!String.IsNullOrEmpty(token)) - { - if (uri.Contains("?")) - uri += "&token=" + token; - else - uri += "?token=" + token; + //if url is encoded, decode it. + if (uri.StartsWith("http%3a%2f%2f", StringComparison.InvariantCultureIgnoreCase) || uri.StartsWith("https%3a%2f%2f", StringComparison.InvariantCultureIgnoreCase)) + uri = HttpUtility.UrlDecode(uri); + + log(TraceLevel.Info, uri); + ServerUrl serverUrl; + bool passThrough = false; + try { + serverUrl = getConfig().GetConfigServerUrl(uri); + passThrough = serverUrl == null; + } + //if XML couldn't be parsed + catch (InvalidOperationException ex) { + + string errorMsg = ex.InnerException.Message + " " + uri; + log(TraceLevel.Error, errorMsg); + sendErrorResponse(context.Response, null, errorMsg, System.Net.HttpStatusCode.InternalServerError); + return; + } + //if mustMatch was set to true and URL wasn't in the list + catch (ArgumentException ex) { + string errorMsg = ex.Message + " " + uri; + log(TraceLevel.Error, errorMsg); + sendErrorResponse(context.Response, null, errorMsg, System.Net.HttpStatusCode.Forbidden); + return; } - - System.Net.HttpWebRequest req = (System.Net.HttpWebRequest)System.Net.HttpWebRequest.Create(uri); - req.Method = context.Request.HttpMethod; - req.ServicePoint.Expect100Continue = false; - req.Referer = context.Request.Headers["referer"]; - - // Set body of request for POST requests - if (context.Request.InputStream.Length > 0) + //use actual request header instead of a placeholder, if present + if (context.Request.Headers["referer"] != null) + PROXY_REFERER = context.Request.Headers["referer"]; + + //referer + //check against the list of referers if they have been specified in the proxy.config + String[] allowedReferersArray = ProxyConfig.GetAllowedReferersArray(); + if (allowedReferersArray != null && allowedReferersArray.Length > 0) { - byte[] bytes = new byte[context.Request.InputStream.Length]; - context.Request.InputStream.Read(bytes, 0, (int)context.Request.InputStream.Length); - req.ContentLength = bytes.Length; - - string ctype = context.Request.ContentType; - if (String.IsNullOrEmpty(ctype)) { - req.ContentType = "application/x-www-form-urlencoded"; - } - else { - req.ContentType = ctype; + bool allowed = false; + string requestReferer = context.Request.Headers["referer"]; + + foreach (var referer in allowedReferersArray) + { + if ((allowedReferersArray.Length == 1) && referer == String.Empty) + break; + + if (requestReferer != null && requestReferer != String.Empty && (ProxyConfig.isUrlPrefixMatch(referer, requestReferer)) || referer == "*") + { + allowed = true; + break; + } } - - using (Stream outputStream = req.GetRequestStream()) + + if (!allowed) { - outputStream.Write(bytes, 0, bytes.Length); + string errorMsg = "Proxy is being used from an unsupported referer: " + context.Request.Headers["referer"]; + log(TraceLevel.Error, errorMsg); + sendErrorResponse(context.Response, null, errorMsg, System.Net.HttpStatusCode.Forbidden); + return; } } - else { - req.Method = "GET"; + + //Throttling: checking the rate limit coming from particular client IP + if (!passThrough && serverUrl.RateLimit > -1) { + lock (_rateMapLock) + { + ConcurrentDictionary ratemap = (ConcurrentDictionary)context.Application["rateMap"]; + if (ratemap == null) + { + ratemap = new ConcurrentDictionary(); + context.Application["rateMap"] = ratemap; + context.Application["rateMap_cleanup_counter"] = 0; + } + string key = "[" + serverUrl.Url + "]x[" + context.Request.UserHostAddress + "]"; + RateMeter rate; + if (!ratemap.TryGetValue(key, out rate)) + { + rate = new RateMeter(serverUrl.RateLimit, serverUrl.RateLimitPeriod); + ratemap.TryAdd(key, rate); + } + if (!rate.click()) + { + log(TraceLevel.Warning, " Pair " + key + " is throttled to " + serverUrl.RateLimit + " requests per " + serverUrl.RateLimitPeriod + " minute(s). Come back later."); + sendErrorResponse(context.Response, "This is a metered resource, number of requests have exceeded the rate limit interval.", "Unable to proxy request for requested resource", System.Net.HttpStatusCode.PaymentRequired); + return; + } + + //making sure the rateMap gets periodically cleaned up so it does not grow uncontrollably + int cnt = (int)context.Application["rateMap_cleanup_counter"]; + cnt++; + if (cnt >= CLEAN_RATEMAP_AFTER) + { + cnt = 0; + cleanUpRatemap(ratemap); + } + context.Application["rateMap_cleanup_counter"] = cnt; + } } - - // Send the request to the server - System.Net.WebResponse serverResponse = null; - - - try + + //readying body (if any) of POST request + byte[] postBody = readRequestPostBody(context); + string post = System.Text.Encoding.UTF8.GetString(postBody); + + System.Net.NetworkCredential credentials = null; + string requestUri = uri; + bool hasClientToken = false; + string token = string.Empty; + string tokenParamName = null; + + if (!passThrough && serverUrl.Domain != null) { - serverResponse = req.GetResponse(); + credentials = new System.Net.NetworkCredential(serverUrl.Username, serverUrl.Password, serverUrl.Domain); } - catch (System.Net.WebException webExc) + else { - response.StatusCode = 500; - response.StatusDescription = webExc.Status.ToString(); - response.Write(webExc.Response); - response.End(); - return; + //if token comes with client request, it takes precedence over token or credentials stored in configuration + hasClientToken = uri.Contains("?token=") || uri.Contains("&token=") || post.Contains("?token=") || post.Contains("&token="); + + if (!passThrough && !hasClientToken) + { + // Get new token and append to the request. + // But first, look up in the application scope, maybe it's already there: + token = (String)context.Application["token_for_" + serverUrl.Url]; + bool tokenIsInApplicationScope = !String.IsNullOrEmpty(token); + + //if still no token, let's see if there is an access token or if are credentials stored in configuration which we can use to obtain new token + if (!tokenIsInApplicationScope) + { + token = serverUrl.AccessToken; + if (String.IsNullOrEmpty(token)) + token = getNewTokenIfCredentialsAreSpecified(serverUrl, uri); + } + + if (!String.IsNullOrEmpty(token) && !tokenIsInApplicationScope) + { + //storing the token in Application scope, to do not waste time on requesting new one untill it expires or the app is restarted. + context.Application.Lock(); + context.Application["token_for_" + serverUrl.Url] = token; + context.Application.UnLock(); + } + } + + //name by which token parameter is passed (if url actually came from the list) + tokenParamName = serverUrl != null ? serverUrl.TokenParamName : null; + + if (String.IsNullOrEmpty(tokenParamName)) + tokenParamName = "token"; + + requestUri = addTokenToUri(uri, token, tokenParamName); } - // Set up the response to the client - if (serverResponse != null) { - response.ContentType = serverResponse.ContentType; - using (Stream byteStream = serverResponse.GetResponseStream()) - { - // Text response - if (serverResponse.ContentType.Contains("text") || - serverResponse.ContentType.Contains("json")) + + //forwarding original request + System.Net.WebResponse serverResponse = null; + try { + serverResponse = forwardToServer(context, requestUri, postBody, credentials); + } catch (System.Net.WebException webExc) { + + string errorMsg = webExc.Message + " " + uri; + log(TraceLevel.Error, errorMsg); + + if (webExc.Response != null) + { + copyHeaders(webExc.Response as System.Net.HttpWebResponse, context.Response); + + using (Stream responseStream = webExc.Response.GetResponseStream()) { - using (StreamReader sr = new StreamReader(byteStream)) + byte[] bytes = new byte[32768]; + int bytesRead = 0; + + while ((bytesRead = responseStream.Read(bytes, 0, bytes.Length)) > 0) { - string strResponse = sr.ReadToEnd(); - response.Write(strResponse); + responseStream.Write(bytes, 0, bytesRead); } + + context.Response.StatusCode = (int)(webExc.Response as System.Net.HttpWebResponse).StatusCode; + context.Response.OutputStream.Write(bytes, 0, bytes.Length); } - else - { - // Binary response (image, lyr file, other binary file) - BinaryReader br = new BinaryReader(byteStream); - byte[] outb = br.ReadBytes((int)serverResponse.ContentLength); - br.Close(); + } + else + { + System.Net.HttpStatusCode statusCode = System.Net.HttpStatusCode.InternalServerError; + sendErrorResponse(context.Response, null, errorMsg, statusCode); + } + return; + } - // Tell client not to cache the image since it's dynamic - response.CacheControl = "no-cache"; + if (passThrough || string.IsNullOrEmpty(token) || hasClientToken) + //if token is not required or provided by the client, just fetch the response as is: + fetchAndPassBackToClient(serverResponse, response, true); + else { + //credentials for secured service have come from configuration file: + //it means that the proxy is responsible for making sure they were properly applied: - // Send the image to the client - // (Note: if large images/files sent, could modify this to send in chunks) - response.OutputStream.Write(outb, 0, outb.Length); - } + //first attempt to send the request: + bool tokenRequired = fetchAndPassBackToClient(serverResponse, response, false); - serverResponse.Close(); + + //checking if previously used token has expired and needs to be renewed + if (tokenRequired) { + log(TraceLevel.Info, "Renewing token and trying again."); + //server returned error - potential cause: token has expired. + //we'll do second attempt to call the server with renewed token: + token = getNewTokenIfCredentialsAreSpecified(serverUrl, uri); + serverResponse = forwardToServer(context, addTokenToUri(uri, token, tokenParamName), postBody); + + //storing the token in Application scope, to do not waste time on requesting new one untill it expires or the app is restarted. + context.Application.Lock(); + context.Application["token_for_" + serverUrl.Url] = token; + context.Application.UnLock(); + + fetchAndPassBackToClient(serverResponse, response, true); } } response.End(); } - + public bool IsReusable { - get { - return false; + get { return true; } + } + +/** +* Private +*/ + private byte[] readRequestPostBody(HttpContext context) { + if (context.Request.InputStream.Length > 0) { + byte[] bytes = new byte[context.Request.InputStream.Length]; + context.Request.InputStream.Read(bytes, 0, (int)context.Request.InputStream.Length); + return bytes; } + return new byte[0]; + } + + private System.Net.WebResponse forwardToServer(HttpContext context, string uri, byte[] postBody, System.Net.NetworkCredential credentials = null) + { + return + postBody.Length > 0? + doHTTPRequest(uri, postBody, "POST", context.Request.Headers["referer"], context.Request.ContentType, credentials): + doHTTPRequest(uri, context.Request.HttpMethod, credentials); } - // Gets the token for a server URL from a configuration file - // TODO: ?modify so can generate a new short-lived token from username/password in the config file - private string getTokenFromConfigFile(string uri) + /// + /// Attempts to copy all headers from the fromResponse to the the toResponse. + /// + /// The response that we are copying the headers from + /// The response that we are copying the headers to + private void copyHeaders(System.Net.WebResponse fromResponse, HttpResponse toResponse) { - try + foreach (var headerKey in fromResponse.Headers.AllKeys) { - ProxyConfig config = ProxyConfig.GetCurrentConfig(); - if (config != null) - return config.GetToken(uri); - else - throw new ApplicationException( - "Proxy.config file does not exist at application root, or is not readable."); + switch (headerKey.ToLower()) + { + case "content-type": + case "transfer-encoding": + continue; + default: + toResponse.AddHeader(headerKey, fromResponse.Headers[headerKey]); + break; + } } - catch (InvalidOperationException) - { - // Proxy is being used for an unsupported service (proxy.config has mustMatch="true") - HttpResponse response = HttpContext.Current.Response; - response.StatusCode = (int)System.Net.HttpStatusCode.Forbidden; - response.End(); + toResponse.ContentType = fromResponse.ContentType; + } + + private bool fetchAndPassBackToClient(System.Net.WebResponse serverResponse, HttpResponse clientResponse, bool ignoreAuthenticationErrors) { + if (serverResponse != null) { + copyHeaders(serverResponse, clientResponse); + using (Stream byteStream = serverResponse.GetResponseStream()) { + // Text response + if (serverResponse.ContentType.Contains("text") || + serverResponse.ContentType.Contains("json") || + serverResponse.ContentType.Contains("xml")) { + using (StreamReader sr = new StreamReader(byteStream)) { + string strResponse = sr.ReadToEnd(); + if ( + !ignoreAuthenticationErrors + && strResponse.IndexOf("{\"error\":{") > -1 + && (strResponse.IndexOf("\"code\":498") > -1 || strResponse.IndexOf("\"code\":499") > -1) + ) + return true; + clientResponse.Write(strResponse); + } + } else { + // Binary response (image, lyr file, other binary file) + + // Tell client not to cache the image since it's dynamic + clientResponse.CacheControl = "no-cache"; + byte[] buffer = new byte[32768]; + int read; + while ((read = byteStream.Read(buffer, 0, buffer.Length)) > 0) + { + clientResponse.OutputStream.Write(buffer, 0, read); + } + clientResponse.OutputStream.Close(); + } + serverResponse.Close(); + } } - catch (Exception e) + return false; + } + + private System.Net.WebResponse doHTTPRequest(string uri, string method, System.Net.NetworkCredential credentials = null) + { + byte[] bytes = null; + String contentType = null; + log(TraceLevel.Info, "Sending request!"); + + if (method.Equals("POST")) { - if (e is ApplicationException) - throw e; - - // just return an empty string at this point - // -- may want to throw an exception, or add to a log file + String[] uriArray = uri.Split('?'); + + if (uriArray.Length > 1) + { + contentType = "application/x-www-form-urlencoded"; + String queryString = uriArray[1]; + + bytes = System.Text.Encoding.UTF8.GetBytes(queryString); + } } + + return doHTTPRequest(uri, bytes, method, PROXY_REFERER, contentType, credentials); + } + + private System.Net.WebResponse doHTTPRequest(string uri, byte[] bytes, string method, string referer, string contentType, System.Net.NetworkCredential credentials = null) + { + System.Net.HttpWebRequest req = (System.Net.HttpWebRequest)System.Net.HttpWebRequest.Create(uri); + req.ServicePoint.Expect100Continue = false; + req.Referer = referer; + req.Method = method; + + if (credentials != null) + req.Credentials = credentials; - return string.Empty; + if (bytes != null && bytes.Length > 0 || method == "POST") { + req.Method = "POST"; + req.ContentType = string.IsNullOrEmpty(contentType) ? "application/x-www-form-urlencoded" : contentType; + if (bytes != null && bytes.Length > 0) + req.ContentLength = bytes.Length; + using (Stream outputStream = req.GetRequestStream()) { + outputStream.Write(bytes, 0, bytes.Length); + } + } + return req.GetResponse(); } -} -[XmlRoot("ProxyConfig")] -public class ProxyConfig -{ - #region Static Members + private string webResponseToString(System.Net.WebResponse serverResponse) { + using (Stream byteStream = serverResponse.GetResponseStream()) { + using (StreamReader sr = new StreamReader(byteStream)) { + string strResponse = sr.ReadToEnd(); + return strResponse; + } + } + } - private static object _lockobject = new object(); + private string getNewTokenIfCredentialsAreSpecified(ServerUrl su, string reqUrl) { + string token = ""; + string infoUrl = ""; + + bool isUserLogin = !String.IsNullOrEmpty(su.Username) && !String.IsNullOrEmpty(su.Password); + bool isAppLogin = !String.IsNullOrEmpty(su.ClientId) && !String.IsNullOrEmpty(su.ClientSecret); + if (isUserLogin || isAppLogin) { + log(TraceLevel.Info, "Matching credentials found in configuration file. OAuth 2.0 mode: " + isAppLogin); + if (isAppLogin) { + //OAuth 2.0 mode authentication + //"App Login" - authenticating using client_id and client_secret stored in config + su.OAuth2Endpoint = string.IsNullOrEmpty(su.OAuth2Endpoint) ? DEFAULT_OAUTH : su.OAuth2Endpoint; + if (su.OAuth2Endpoint[su.OAuth2Endpoint.Length - 1] != '/') + su.OAuth2Endpoint += "/"; + log(TraceLevel.Info, "Service is secured by " + su.OAuth2Endpoint + ": getting new token..."); + string uri = su.OAuth2Endpoint + "token?client_id=" + su.ClientId + "&client_secret=" + su.ClientSecret + "&grant_type=client_credentials&f=json"; + string tokenResponse = webResponseToString(doHTTPRequest(uri, "POST")); + token = extractToken(tokenResponse, "token"); + if (!string.IsNullOrEmpty(token)) + token = exchangePortalTokenForServerToken(token, su); + } else { + //standalone ArcGIS Server/ArcGIS Online token-based authentication + + //if a request is already being made to generate a token, just let it go + if (reqUrl.ToLower().Contains("/generatetoken")) { + string tokenResponse = webResponseToString(doHTTPRequest(reqUrl, "POST")); + token = extractToken(tokenResponse, "token"); + return token; + } + + //lets look for '/rest/' in the requested URL (could be 'rest/services', 'rest/community'...) + if (reqUrl.ToLower().Contains("/rest/")) + infoUrl = reqUrl.Substring(0, reqUrl.IndexOf("/rest/", StringComparison.OrdinalIgnoreCase)); + + //if we don't find 'rest', lets look for the portal specific 'sharing' instead + else if (reqUrl.ToLower().Contains("/sharing/")) { + infoUrl = reqUrl.Substring(0, reqUrl.IndexOf("/sharing/", StringComparison.OrdinalIgnoreCase)); + infoUrl = infoUrl + "/sharing"; + } + else + throw new ApplicationException("Unable to determine the correct URL to request a token to access private resources"); + + if (infoUrl != "") { + log(TraceLevel.Info," Querying security endpoint..."); + infoUrl += "/rest/info?f=json"; + //lets send a request to try and determine the URL of a token generator + string infoResponse = webResponseToString(doHTTPRequest(infoUrl, "GET")); + String tokenServiceUri = getJsonValue(infoResponse, "tokenServicesUrl"); + if (string.IsNullOrEmpty(tokenServiceUri)) + tokenServiceUri = getJsonValue(infoResponse, "tokenServiceUrl"); + if (tokenServiceUri != "") { + log(TraceLevel.Info," Service is secured by " + tokenServiceUri + ": getting new token..."); + string uri = tokenServiceUri + "?f=json&request=getToken&referer=" + PROXY_REFERER + "&expiration=60&username=" + su.Username + "&password=" + su.Password; + string tokenResponse = webResponseToString(doHTTPRequest(uri, "POST")); + token = extractToken(tokenResponse, "token"); + } + } + + + } + } + return token; + } + + private string exchangePortalTokenForServerToken(string portalToken, ServerUrl su) { + //ideally, we should POST the token request + log(TraceLevel.Info," Exchanging Portal token for Server-specific token for " + su.Url + "..."); + string uri = su.OAuth2Endpoint.Substring(0, su.OAuth2Endpoint.IndexOf("/oauth2/", StringComparison.OrdinalIgnoreCase)) + + "/generateToken?token=" + portalToken + "&serverURL=" + su.Url + "&f=json"; + string tokenResponse = webResponseToString(doHTTPRequest(uri, "GET")); + return extractToken(tokenResponse, "token"); + } - public static ProxyConfig LoadProxyConfig(string fileName) + private static void sendErrorResponse(HttpResponse response, String errorDetails, String errorMessage, System.Net.HttpStatusCode errorCode) { - ProxyConfig config = null; + String message = string.Format("{{error: {{code: {0},message:\"{1}\"", (int)errorCode, errorMessage); + if (!string.IsNullOrEmpty(errorDetails)) + message += string.Format(",details:[message:\"{0}\"]", errorDetails); + message += "}}"; + response.StatusCode = (int)errorCode; + //this displays our customized error messages instead of IIS's custom errors + response.TrySkipIisCustomErrors = true; + response.Write(message); + response.Flush(); + } - lock (_lockobject) + private static string getClientIp(HttpRequest request) + { + if (request == null) + return null; + string remoteAddr = request.ServerVariables["HTTP_X_FORWARDED_FOR"]; + if (string.IsNullOrWhiteSpace(remoteAddr)) + { + remoteAddr = request.ServerVariables["REMOTE_ADDR"]; + } + else { - if (System.IO.File.Exists(fileName)) + // the HTTP_X_FORWARDED_FOR may contain an array of IP, this can happen if you connect through a proxy. + string[] ipRange = remoteAddr.Split(','); + remoteAddr = ipRange[ipRange.Length - 1]; + } + return remoteAddr; + } + + private string addTokenToUri(string uri, string token, string tokenParamName) { + if (!String.IsNullOrEmpty(token)) + uri += uri.Contains("?")? "&" + tokenParamName + "=" + token : "?" + tokenParamName + "=" + token; + return uri; + } + + private string extractToken(string tokenResponse, string key) { + string token = getJsonValue(tokenResponse, key); + if (string.IsNullOrEmpty(token)) + log(TraceLevel.Error," Token cannot be obtained: " + tokenResponse); + else + log(TraceLevel.Info," Token obtained: " + token); + return token; + } + + private string getJsonValue(string text, string key) { + int i = text.IndexOf(key); + String value = ""; + if (i > -1) { + value = text.Substring(text.IndexOf(':', i) + 1).Trim(); + value = value.Length > 0 && value[0] == '"' ? + value.Substring(1, value.IndexOf('"', 1) - 1): + value = value.Substring(0, Math.Max(0, Math.Min(Math.Min(value.IndexOf(","), value.IndexOf("]")), value.IndexOf("}")))); + } + return value; + } + + private void cleanUpRatemap(ConcurrentDictionary ratemap) { + foreach (string key in ratemap.Keys){ + RateMeter rate = ratemap[key]; + if (rate.canBeCleaned()) + ratemap.TryRemove(key, out rate); + } + } + +/** +* Static +*/ + private static ProxyConfig getConfig() { + ProxyConfig config = ProxyConfig.GetCurrentConfig(); + if (config != null) + return config; + else + throw new ApplicationException("The proxy configuration file cannot be found, or is not readable."); + } + + //writing Log file + private static void log(TraceLevel logLevel, string msg) { + string logMessage = string.Format("{0} {1}", DateTime.Now.ToString("yyyy-MM-dd HH:mm:ss"), msg); + if (TraceLevel.Error == logLevel) + { + Trace.TraceError(logMessage); + logMessageToFile(logMessage); + } + else if (TraceLevel.Warning == logLevel) + { + Trace.TraceWarning(logMessage); + logMessageToFile(logMessage); + } + else + { + Trace.TraceInformation(logMessage); + logMessageToFile(logMessage); + } + } + + private static object _lockobject = new object(); + + private static void logMessageToFile(String message) + { + //Only log messages to disk if logFile has value in configuration, otherwise log nothing. + ProxyConfig config = ProxyConfig.GetCurrentConfig(); + if (config.LogFile != null) + { + string log = config.LogFile; + if (!log.Contains("\\") || log.Contains(".\\")) { - XmlSerializer reader = new XmlSerializer(typeof(ProxyConfig)); - using (System.IO.StreamReader file = new System.IO.StreamReader(fileName)) + if (log.Contains(".\\")) //If this type of relative pathing .\log.txt { - config = (ProxyConfig)reader.Deserialize(file); + log = log.Replace(".\\", ""); + } + string configDirectory = HttpContext.Current.Server.MapPath("proxy.config"); //Cannot use System.Web.Hosting.HostingEnvironment.ApplicationPhysicalPath b/ config may be in a child directory + string path = configDirectory.Replace("proxy.config",""); + log = path + log; + } + + lock(_lockobject) { + using (StreamWriter sw = File.AppendText(log)) + { + sw.WriteLine(message); } } } + } +} + +[XmlRoot("ProxyConfig")] +public class ProxyConfig +{ + private static object _lockobject = new object(); + public static ProxyConfig LoadProxyConfig(string fileName) { + ProxyConfig config = null; + lock (_lockobject) { + if (System.IO.File.Exists(fileName)) { + XmlSerializer reader = new XmlSerializer(typeof(ProxyConfig)); + using (System.IO.StreamReader file = new System.IO.StreamReader(fileName)) { + try { + config = (ProxyConfig)reader.Deserialize(file); + } + catch (Exception ex) { + throw ex; + } + } + } + } return config; } - public static ProxyConfig GetCurrentConfig() - { + public static ProxyConfig GetCurrentConfig() { ProxyConfig config = HttpRuntime.Cache["proxyConfig"] as ProxyConfig; - if (config == null) - { - string fileName = GetFilename(HttpContext.Current); + if (config == null) { + string fileName = HttpContext.Current.Server.MapPath("proxy.config"); config = LoadProxyConfig(fileName); - - if (config != null) - { + if (config != null) { CacheDependency dep = new CacheDependency(fileName); HttpRuntime.Cache.Insert("proxyConfig", config, dep); } } - return config; } - public static string GetFilename(HttpContext context) + //referer + //create an array with valid referers using the allowedReferers String that is defined in the proxy.config + public static String[] GetAllowedReferersArray() { - return context.Server.MapPath("proxy.config"); + if (allowedReferers == null) + return null; + + return allowedReferers.Split(','); + } + + //referer + //check if URL starts with prefix... + public static bool isUrlPrefixMatch(String prefix, String uri) + { + + return uri.ToLower().StartsWith(prefix.ToLower()) || + uri.ToLower().Replace("https://", "http://").StartsWith(prefix.ToLower()) || + uri.ToLower().Substring(uri.IndexOf("//")).StartsWith(prefix.ToLower()); } - #endregion ServerUrl[] serverUrls; + public String logFile; bool mustMatch; + //referer + static String allowedReferers; [XmlArray("serverUrls")] [XmlArrayItem("serverUrl")] - public ServerUrl[] ServerUrls - { + public ServerUrl[] ServerUrls { get { return this.serverUrls; } - set { this.serverUrls = value; } + set + { + this.serverUrls = value; + } } - [XmlAttribute("mustMatch")] - public bool MustMatch - { + public bool MustMatch { get { return mustMatch; } - set { mustMatch = value; } + set + { mustMatch = value; } + } + + //logFile + [XmlAttribute("logFile")] + public String LogFile + { + get { return logFile; } + set + { logFile = value; } } - public string GetToken(string uri) + + //referer + [XmlAttribute("allowedReferers")] + public string AllowedReferers { - foreach (ServerUrl su in serverUrls) + get { return allowedReferers; } + set { - if (su.MatchAll && uri.StartsWith(su.Url, StringComparison.InvariantCultureIgnoreCase)) - { - return su.Token; - } - else - { - if (String.Compare(uri, su.Url, StringComparison.InvariantCultureIgnoreCase) == 0) - return su.Token; - } + allowedReferers = value; } + } - if (mustMatch) - throw new InvalidOperationException(); + public ServerUrl GetConfigServerUrl(string uri) { + //split both request and proxy.config urls and compare them + string[] uriParts = uri.Split(new char[] {'/','?'}, StringSplitOptions.RemoveEmptyEntries); + string[] configUriParts = new string[] {}; + + foreach (ServerUrl su in serverUrls) { + //if a relative path is specified in the proxy.config, append what's in the request itself + if (!su.Url.StartsWith("http")) + su.Url = su.Url.Insert(0, uriParts[0]); - return string.Empty; + configUriParts = su.Url.Split(new char[] { '/','?' }, StringSplitOptions.RemoveEmptyEntries); + + //if the request has less parts than the config, don't allow + if (configUriParts.Length > uriParts.Length) continue; + + int i = 0; + for (i = 0; i < configUriParts.Length; i++) { + + if (!configUriParts[i].ToLower().Equals(uriParts[i].ToLower())) break; + } + if (i == configUriParts.Length) { + //if the urls don't match exactly, and the individual matchAll tag is 'false', don't allow + if (configUriParts.Length == uriParts.Length || su.MatchAll) + return su; + } + } + + if (mustMatch) + throw new ArgumentException("Proxy is being used for an unsupported service:"); + + return null; } + + } -public class ServerUrl -{ +public class ServerUrl { string url; bool matchAll; - string token; - + string oauth2Endpoint; + string domain; + string username; + string password; + string clientId; + string clientSecret; + string accessToken; + string tokenParamName; + string rateLimit; + string rateLimitPeriod; + [XmlAttribute("url")] - public string Url - { + public string Url { get { return url; } set { url = value; } } - [XmlAttribute("matchAll")] - public bool MatchAll - { + public bool MatchAll { get { return matchAll; } set { matchAll = value; } } - - [XmlAttribute("token")] - public string Token + [XmlAttribute("oauth2Endpoint")] + public string OAuth2Endpoint { + get { return oauth2Endpoint; } + set { oauth2Endpoint = value; } + } + [XmlAttribute("domain")] + public string Domain { - get { return token; } - set { token = value; } + get { return domain; } + set { domain = value; } + } + [XmlAttribute("username")] + public string Username { + get { return username; } + set { username = value; } + } + [XmlAttribute("password")] + public string Password { + get { return password; } + set { password = value; } + } + [XmlAttribute("clientId")] + public string ClientId { + get { return clientId; } + set { clientId = value; } + } + [XmlAttribute("clientSecret")] + public string ClientSecret { + get { return clientSecret; } + set { clientSecret = value; } + } + [XmlAttribute("accessToken")] + public string AccessToken { + get { return accessToken; } + set { accessToken = value; } + } + [XmlAttribute("tokenParamName")] + public string TokenParamName { + get { return tokenParamName; } + set { tokenParamName = value; } + } + [XmlAttribute("rateLimit")] + public int RateLimit { + get { return string.IsNullOrEmpty(rateLimit)? -1 : int.Parse(rateLimit); } + set { rateLimit = value.ToString(); } + } + [XmlAttribute("rateLimitPeriod")] + public int RateLimitPeriod { + get { return string.IsNullOrEmpty(rateLimitPeriod)? 60 : int.Parse(rateLimitPeriod); } + set { rateLimitPeriod = value.ToString(); } } } diff --git a/viewer/proxy/proxy.config b/viewer/proxy/proxy.config old mode 100644 new mode 100755 index 7a2793f9a..ef689f898 --- a/viewer/proxy/proxy.config +++ b/viewer/proxy/proxy.config @@ -1,25 +1,9 @@ - - - - - - - - - - + + + + + + diff --git a/viewer/proxy/proxy.xsd b/viewer/proxy/proxy.xsd new file mode 100755 index 000000000..d6c057fa8 --- /dev/null +++ b/viewer/proxy/proxy.xsd @@ -0,0 +1,31 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/viewer/proxy/proxypage_java.zip b/viewer/proxy/proxypage_java.zip deleted file mode 100644 index a5d059be8..000000000 Binary files a/viewer/proxy/proxypage_java.zip and /dev/null differ diff --git a/viewer/proxy/proxypage_net.zip b/viewer/proxy/proxypage_net.zip deleted file mode 100644 index 0c13888ba..000000000 Binary files a/viewer/proxy/proxypage_net.zip and /dev/null differ diff --git a/viewer/proxy/proxypage_php.zip b/viewer/proxy/proxypage_php.zip deleted file mode 100644 index d65094ee9..000000000 Binary files a/viewer/proxy/proxypage_php.zip and /dev/null differ diff --git a/viewer/proxy/readme.md b/viewer/proxy/readme.md index 9e98aa091..630a8993d 100644 --- a/viewer/proxy/readme.md +++ b/viewer/proxy/readme.md @@ -1,7 +1,7 @@ # AGS Proxy page help For full info please read here: -http://help.arcgis.com/en/webapi/javascript/arcgis/jshelp/#ags_proxy +[https://developers.arcgis.com/javascript/jshelp/ags_proxy.html](https://developers.arcgis.com/javascript/jshelp/ags_proxy.html) You will more than likely need a proxy page for printing and other large cross domain requests. There are diffrent flavors (.net, jsp, php) of the proxy page depending on your server side technology. See the above link for full details. \ No newline at end of file