check ES index creation properly

exec/TXG.hs

@@ -514,18 +514,34 @@ esPutReq mgr esConf version = do
   resp <- liftIO $ httpLbs esReq mgr
   return $ eitherDecode @Value (responseBody resp)
 
-esCheckIndex :: MonadIO m => MonadThrow m => Manager -> Logger Text -> ElasticSearchConfig -> ChainwebVersion -> m (Either String Value)
+esCheckIndex :: MonadIO m => MonadThrow m => Manager -> Logger Text -> ElasticSearchConfig -> ChainwebVersion -> m (Either String ())
 esCheckIndex mgr logger esConf version = do
   let indexName :: String
       indexName = printf "chainweb-%s-%s" (T.unpack $ chainwebVersionToText version) (fromMaybe "" $ esIndex esConf)
   req <- HTTP.parseUrlThrow $ printf "http://%s:%s/%s?pretty" (T.unpack $ hostnameToText $ esHost esConf) (show $ esPort esConf) indexName
   resp <- liftIO $ httpLbs req mgr
 
   case eitherDecode @Value (responseBody resp) of
-    err@(Left _) -> return err
-    r@(Right _) -> do
-      liftIO $ loggerFunIO logger Info $ "Index " <> T.pack indexName <> " exists"
-      return r
+    Left l -> do
+      liftIO $ loggerFunIO logger Error $ "esCheckIndex: Failed to decode response: " <> T.pack (show l)
+      return $ Left l
+    Right val -> do
+      let status = statusCode $ responseStatus resp
+          errConds e = and
+            [ status == 400
+            , (e ^? key "type" . _String) == Just "resource_already_exists_exception"
+            , (e ^? key "index" . _String) == Just (T.pack indexName)
+            ]
+      case val ^? key "error" of
+        Just errObj | errConds errObj -> do
+            liftIO $ loggerFunIO logger Info $ "Index " <> T.pack indexName <> " already exists"
+            return $ Right ()
+        _ -> do
+          let indexCreated = object ["acknowledged" .= True, "shards_acknowledged" .= True, "index" .= indexName ]
+          if val == indexCreated && status == 200
+            then liftIO $ loggerFunIO logger Info $ "Index " <> T.pack indexName <> " created"
+            else throwM $ ElasticSearchException $ "esCheckIndex: Unexpected response: " <> T.pack (show val)
+          return $ Right ()
 
 createElasticsearchIndex :: MonadIO m => MonadThrow m => ElasticSearchConfig -> ChainwebVersion -> m HTTP.Request
 createElasticsearchIndex esConf version = do
@@ -883,9 +899,9 @@ simpleExpressions config (ChainwebHost h _p2p service) tcut tv distribution = do
         toRetry _ = \case
           Right _ -> return False
           Left _ -> return True
-    void $ retrying policy toRetry (const $ esPutReq (confManager gencfg) esConfig (nodeVersion config))
     logger' <- ask
     void $ retrying policy toRetry (const $ esCheckIndex (confManager gencfg) logger' esConfig (nodeVersion config))
+    void $ retrying policy toRetry (const $ esPutReq (confManager gencfg) esConfig (nodeVersion config))
   let chs = maybe (versionChains $ nodeVersion config) NES.fromList
              . NEL.nonEmpty
              $ nodeChainIds config

lib/TXG/Utils.hs

@@ -36,6 +36,7 @@ module TXG.Utils
 -- * Misc Utils
 , fromJuste
 , TextFormatException(..)
+, ElasticSearchException(..)
 , encodeToText
 , unsafeManager
 
@@ -98,6 +99,8 @@ import Text.Read (readEither)
 
 instance Exception TextFormatException
 
+instance Exception ElasticSearchException
+
 data ChainwebVersion
     = Mainnet01
     | Testnet04
@@ -200,6 +203,9 @@ unsafeManager :: IO Manager
 unsafeManager = newTlsManagerWith
     $ mkManagerSettings (TLSSettingsSimple True True True) Nothing
 
+newtype ElasticSearchException = ElasticSearchException T.Text
+    deriving (Show, Eq, Ord, IsString, Generic)
+
 -- -------------------------------------------------------------------------- --
 -- Commad line