Chunkinfo

admin/config/config.yml

@@ -24,8 +24,6 @@ dn_port: 6101 # Start dn ports at 6101
 dn_ram: 3g # memory for DN container (per container)
 sn_port: 5101 # Start sn ports at 5101
 sn_ram: 3g # memory for SN container
-rangeget_port: 6900 # singleton proxy at port 6900
-rangeget_ram: 2g # memory for RANGEGET container
 target_sn_count: 0 # desired number of SN containers
 target_dn_count: 0 # desire number of DN containers
 log_level: INFO    # log level.  One of ERROR, WARNING, INFO, DEBUG
@@ -54,11 +52,6 @@ metadata_mem_cache_size: 128m # 128 MB - metadata cache size per DN node
 metadata_mem_cache_expire: 3600 # expire cache items after one hour
 chunk_mem_cache_size: 128m # 128 MB - chunk cache size per DN node
 chunk_mem_cache_expire: 3600 # expire cache items after one hour
-data_cache_size: 128m # cache for rangegets
-data_cache_max_req_size: 128k # max size for rangeget fetches
-data_cache_expire_time: 3600 # expire cache items after one hour
-data_cache_page_size: 4m # page size for range get cache, set to zero to disable proxy
-data_cache_max_concurrent_read: 16 # maximum number of inflight storage read requests
 timeout: 30 # http timeout - 30 sec
 password_file: /config/passwd.txt # filepath to a text file of username/passwords. set to '' for no-auth access
 groups_file: /config/groups.txt # filepath to text file defining user groups
@@ -97,3 +90,10 @@ aws_lambda_gateway: null # use lambda endpoint for region HSDS is running in
 k8s_app_label: null # The app label for k8s deployments (use k8s_dn_label_selector instead)
 write_zero_chunks: False # write chunk to storage even when it's all zeros (or in general equal to the fill value)
 max_chunks_per_request: 1000 # maximum number of chunks to be serviced by one request
+rangeget_port: 6900 # singleton proxy at port 6900
+rangeget_ram: 2g # memory for RANGEGET container
+data_cache_size: 128m # cache for rangegets
+data_cache_max_req_size: 128k # max size for rangeget fetches
+data_cache_expire_time: 3600 # expire cache items after one hour
+data_cache_page_size: 4m # page size for range get cache, set to zero to disable proxy
+data_cache_max_concurrent_read: 16 # maximum number of inflight storage read requests

admin/docker/docker-compose.aws.yml

@@ -39,7 +39,6 @@ services:
       - ${PWD}/admin/config/:/config/
     links:
       - head
-      - rangeget
 
   sn:
     image: hdfgroup/hsds
@@ -64,22 +63,3 @@ services:
     links:
       - head
 
-  rangeget:
-    image: hdfgroup/hsds
-    restart: ${RESTART_POLICY}
-    mem_limit: ${RANGEGET_RAM}
-    environment:
-      - NODE_TYPE=rangeget
-      - AWS_S3_GATEWAY=${AWS_S3_GATEWAY}
-      - AWS_REGION=${AWS_REGION}
-      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
-      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
-      - LOG_LEVEL=${LOG_LEVEL}
-    ports:
-      - ${RANGEGET_PORT}
-    depends_on:
-      - head
-    volumes:
-      - ${PWD}/admin/config/:/config/
-    links:
-      - head

admin/docker/docker-compose.azure.yml

@@ -33,7 +33,6 @@ services:
       - ${PWD}/admin/config/:/config/
     links:
       - head
-      - rangeget
 
   sn:
     image: hdfgroup/hsds
@@ -53,21 +52,4 @@ services:
     volumes:
       - ${PWD}/admin/config/:/config/
     links:
-      - head
-
-  rangeget:
-    image: hdfgroup/hsds
-    restart: ${RESTART_POLICY}
-    mem_limit: ${RANGEGET_RAM}
-    environment:
-      - NODE_TYPE=rangeget
-      - AZURE_CONNECTION_STRING=${AZURE_CONNECTION_STRING}
-      - LOG_LEVEL=${LOG_LEVEL}
-    ports:
-      - ${RANGEGET_PORT}
-    depends_on:
-      - head
-    volumes:
-      - ${PWD}/admin/config/:/config/
-    links:
-      - head
+      - head

admin/docker/docker-compose.posix.yml

@@ -33,7 +33,6 @@ services:
       - ${PWD}/admin/config/:/config/
     links:
       - head
-      - rangeget
   sn:
     image: hdfgroup/hsds
     restart: ${RESTART_POLICY}
@@ -53,21 +52,3 @@ services:
       - ${PWD}/admin/config/:/config/
     links:
       - head
-
-  rangeget:
-    image: hdfgroup/hsds
-    restart: ${RESTART_POLICY}
-    mem_limit: ${RANGEGET_RAM}
-    environment:
-      - NODE_TYPE=rangeget
-      - ROOT_DIR=/data
-      - LOG_LEVEL=${LOG_LEVEL}
-    ports:
-      - ${RANGEGET_PORT}
-    depends_on:
-      - head
-    volumes:
-      - ${ROOT_DIR}:/data
-      - ${PWD}/admin/config/:/config/
-    links:
-      - head

admin/docker/docker-compose.yml

@@ -39,7 +39,6 @@ services:
       - ${PWD}/admin/config/:/config/
     links:
       - head
-      - rangeget
 
   sn:
     image: hdfgroup/hsds
@@ -64,22 +63,3 @@ services:
     links:
       - head
 
-  rangeget:
-    image: hdfgroup/hsds
-    restart: ${RESTART_POLICY}
-    mem_limit: ${RANGEGET_RAM}
-    environment:
-      - NODE_TYPE=rangeget
-      - AWS_S3_GATEWAY=${AWS_S3_GATEWAY}
-      - AWS_REGION=${AWS_REGION}
-      - AWS_SECRET_ACCESS_KEY=${AWS_SECRET_ACCESS_KEY}
-      - AWS_ACCESS_KEY_ID=${AWS_ACCESS_KEY_ID}
-      - LOG_LEVEL=${LOG_LEVEL}
-    ports:
-      - ${RANGEGET_PORT}
-    depends_on:
-      - head
-    volumes:
-      - ${PWD}/admin/config/:/config/
-    links:
-      - head

hsds/basenode.py

@@ -628,10 +628,6 @@ def baseInit(node_type):
 
         app["dn_urls"] = dn_urls
         app["dn_ids"] = dn_ids
-        rangeget_url = config.getCmdLineArg("rangeget_url")
-        if rangeget_url:
-            log.debug(f"store rangeget_url: {rangeget_url}")
-            app["rangeget_url"] = rangeget_url
 
         # check to see if we are running in a DCOS cluster
     elif "IS_DOCKER" in os.environ:

hsds/chunk_crawl.py

@@ -202,10 +202,27 @@ async def read_chunk_hyperslab(
     # params["select"] = select
     if "s3path" in chunk_info:
         params["s3path"] = chunk_info["s3path"]
+
     if "s3offset" in chunk_info:
-        params["s3offset"] = chunk_info["s3offset"]
+        s3offset = chunk_info["s3offset"]
+        if isinstance(s3offset, list):
+            # convert to a colon seperated string
+            s3offset = ":".join(map(str, s3offset))
+        params["s3offset"] = s3offset
+
     if "s3size" in chunk_info:
-        params["s3size"] = chunk_info["s3size"]
+        s3size = chunk_info["s3size"]
+        if isinstance(s3size, list):
+            # convert to a colon seperated string
+            s3size = ":".join(map(str, s3size))
+        params["s3size"] = s3size
+
+    if "hyper_dims" in chunk_info:
+        hyper_dims = chunk_info["hyper_dims"]
+        if isinstance(hyper_dims, list):
+            # convert to colon seperated string
+            hyper_dims = ":".join(map(str, hyper_dims))
+        params["hyper_dims"] = hyper_dims
 
     # set query-based params
     if query is not None:
@@ -633,16 +650,13 @@ async def work(self):
                 start = time.time()
                 chunk_id = await self._q.get()
                 if self._limit > 0 and self._hits >= self._limit:
-                    log.debug(
-                        "ChunkCrawler - max hits exceeded, skipping fetch for chunk: {chunk_id}"
-                    )
+                    msg = f"ChunkCrawler - maxhits exceeded, skipping fetch for chunk: {chunk_id}"
+                    log.debug(msg)
                 else:
                     dn_url = getDataNodeUrl(self._app, chunk_id)
                     if isUnixDomainUrl(dn_url):
                         # need a client per url for unix sockets
-                        client = get_http_client(
-                            self._app, url=dn_url, cache_client=True
-                        )
+                        client = get_http_client(self._app, url=dn_url, cache_client=True)
                     else:
                         # create a pool of clients and store the handles in the app dict
                         if client_name not in self._clients:

hsds/chunk_dn.py

@@ -270,6 +270,8 @@ async def GET_Chunk(request):
     s3path = None
     s3offset = None
     s3size = None
+    hyper_dims = None
+    dims = None
     query = None
     limit = 0
 
@@ -306,25 +308,69 @@ async def GET_Chunk(request):
     log.debug(f"GET_Chunk - using bucket: {bucket}")
 
     if "s3offset" in params:
+        param_s3offset = params["s3offset"]
+        log.debug(f"s3offset param: {param_s3offset}")
         try:
-            s3offset = int(params["s3offset"])
+            if param_s3offset.find(":") > 0:
+                # colon seperated index values, convert to list
+                s3offset = list(map(int, param_s3offset.split(":")))
+            else:
+                s3offset = int(param_s3offset)
         except ValueError:
-            log.error(f"invalid s3offset params: {params['s3offset']}")
+            log.error(f"invalid s3offset params: {param_s3offset}")
             raise HTTPBadRequest()
+        log.debug(f"s3offset: {s3offset}")
+
     if "s3size" in params:
+        param_s3size = params["s3size"]
+        log.debug(f"s3size param: {param_s3size}")
         try:
-            s3size = int(params["s3size"])
+            if param_s3size.find(":") > 0:
+                s3size = list(map(int, param_s3size.split(":")))
+            else:
+                s3size = int(param_s3size)
         except ValueError:
-            log.error(f"invalid s3size params: {params['s3size']}")
+            log.error(f"invalid s3size params: {param_s3size}")
+            raise HTTPBadRequest()
+        log.debug(f"s3size: {s3size}")
+
+    if "hyper_dims" in params:
+        param_hyper_dims = params["hyper_dims"]
+        try:
+            if param_hyper_dims.find(":") > 0:
+                hyper_dims = list(map(int, param_hyper_dims.split(":")))
+            else:
+                hyper_dims = [int(param_hyper_dims), ]
+        except ValueError:
+            log.error(f"invalid hyper_dims params: {param_hyper_dims}")
             raise HTTPBadRequest()
+        log.debug(f"hyper_dims: {hyper_dims}")
 
     if "query" in params:
         query = params["query"]
         log.debug(f"got query: {query}")
 
     if "Limit" in params:
-        limit = int(params["Limit"])
+        param_limit = params["Limit"]
         log.debug(f"limit: {limit}")
+        try:
+            limit = int(param_limit)
+        except ValueError:
+            log.error(f"invalid Limit param: {param_limit}")
+            raise HTTPBadRequest()
+
+    if s3path:
+        # calculate how many chunk bytes we'll read
+        num_bytes = 0
+        if isinstance(s3size, int):
+            num_bytes = s3size
+        else:
+            # list
+            num_bytes = np.prod(s3size)
+        log.debug(f"reading {num_bytes} from {s3path}")
+        if num_bytes == 0:
+            log.warn(f"GET_Chunk for s3path: {s3path} with empty byte range")
+            raise HTTPNotFound()
 
     dset_id = getDatasetId(chunk_id)
 
@@ -356,6 +402,8 @@ async def GET_Chunk(request):
         kwargs["s3path"] = s3path
         kwargs["s3offset"] = s3offset
         kwargs["s3size"] = s3size
+        if hyper_dims:
+            kwargs["hyper_dims"] = hyper_dims
     else:
         kwargs["bucket"] = bucket