From b197842ad200bccdf2271f0072aceacec5454447 Mon Sep 17 00:00:00 2001 From: Adrian Salceanu Date: Tue, 11 Jun 2024 21:13:02 +0200 Subject: [PATCH] Actions logging and async notifications --- Project.toml | 3 +- .../applications/ApplicationsController.jl | 151 ++++++++++++------ routes.jl | 26 +-- src/Actions.jl | 15 ++ src/GenieBuilder.jl | 11 ++ src/Licensing.jl | 82 ++++------ 6 files changed, 166 insertions(+), 122 deletions(-) create mode 100644 src/Actions.jl diff --git a/Project.toml b/Project.toml index 24f77e3..cfbb198 100755 --- a/Project.toml +++ b/Project.toml @@ -1,9 +1,10 @@ name = "GenieBuilder" uuid = "c9453c14-af8a-11ec-351d-c7c9a2035d70" authors = ["Adrian Salceanu"] -version = "0.16.105" +version = "0.16.106" [deps] +Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" Dates = "ade2ca70-3891-5945-98fb-dc099432e06a" Genie = "c43c736e-a2d1-11e8-161f-af95117fbd1e" GenieAutoReload = "c6228e60-a24f-11e9-1776-c313472ebacc" diff --git a/app/resources/applications/ApplicationsController.jl b/app/resources/applications/ApplicationsController.jl index 43845f5..f608830 100755 --- a/app/resources/applications/ApplicationsController.jl +++ b/app/resources/applications/ApplicationsController.jl @@ -16,6 +16,7 @@ using Scratch using ZipFile using TOML using Logging +using GenieBuilder.Licensing, GenieBuilder.Actions import StippleUI import GeniePackageManager @@ -193,7 +194,7 @@ Registers a file system path as an app with GenieBuilder """ function register(name::AbstractString = "", path::AbstractString = pwd(); autostart::Bool = true) try - notify("started:register_app") + @async notify("started:register_app") |> errormonitor path = abspath(normpath(path)) # |> realpath #TODO: put back in next release isdir(path) || throw(ArgumentError("Path $path is not a directory")) @@ -207,14 +208,22 @@ function register(name::AbstractString = "", path::AbstractString = pwd(); autos app = save!(app) persist_status(app, OFFLINE_STATUS) - notify("ended:register_app", app.id) + @async notify("ended:register_app", app.id) |> errormonitor + + @async GenieBuilder.Licensing.log(; + type = Actions.ACTION_REGISTER_APP, + metadata = Dict( + "name" => app.name, + "path" => app.path, + ) + ) |> errormonitor autostart && start(app) return app |> json catch ex @error(ex) - notify("failed:register_app", nothing, FAILSTATUS, ERROR_STATUS) + @async notify("failed:register_app", nothing, FAILSTATUS, ERROR_STATUS) |> errormonitor return (:status => FAILSTATUS, :error => ex) |> json end @@ -228,10 +237,20 @@ Unregisters an app with GenieBuilder function unregister(app::Application) app_id = app.id - notify("started:unregister", app_id) + @async notify("started:unregister", app_id) |> errormonitor stop(app) + + @async GenieBuilder.Licensing.log(; + type = Actions.ACTION_UNREGISTER_APP, + metadata = Dict( + "name" => app.name, + "path" => app.path, + ) + ) |> errormonitor + SearchLight.delete(app) - notify("ended:unregister", app_id) + + @async notify("ended:unregister", app_id) |> errormonitor (:status => OKSTATUS) |> json end @@ -242,14 +261,14 @@ end Creates the Genie app skeleton """ function create(app::Application; name::AbstractString = "", path::AbstractString = pwd()) - notify("started:create", app.id) + @async notify("started:create", app.id) |> errormonitor try boilerplate(app.path) - notify("ended:create", app.id) + @async notify("ended:create", app.id) |> errormonitor catch ex @error ex - notify("failed:create", app.id, FAILSTATUS, ERROR_STATUS) + @async notify("failed:create", app.id, FAILSTATUS, ERROR_STATUS) |> errormonitor return (:status => FAILSTATUS, :error => ex) |> json end @@ -346,7 +365,7 @@ function status_request(app, donotify::Bool = true; statuscheck::Bool = false, p end status = try - donotify && notify("started:status_request", app.id) + donotify && (@async notify("started:status_request", app.id) |> errormonitor) res = HTTP.request("GET", "$(apphost):$(app.port)$(GenieDevTools.defaultroute)/id") if res.status >= 500 ERROR_STATUS @@ -359,7 +378,7 @@ function status_request(app, donotify::Bool = true; statuscheck::Bool = false, p if isa(ex, HTTP.Exceptions.ConnectError) OFFLINE_STATUS else - donotify && notify("failed:status_request", app.id, FAILSTATUS, ERROR_STATUS) + donotify && (@async notify("failed:status_request", app.id, FAILSTATUS, ERROR_STATUS) |> errormonitor) ERROR_STATUS end end @@ -369,7 +388,7 @@ function status_request(app, donotify::Bool = true; statuscheck::Bool = false, p status = STARTING_STATUS end - donotify && notify("ended:status_request", app.id) + donotify && (@async notify("ended:status_request", app.id) |> errormonitor) persist && persist_status(app, status) # println("Status request: $status") @@ -383,10 +402,10 @@ end REST endpoint to check the status of an app """ function status(app::Application) - notify("started:status", app.id) + @async notify("started:status", app.id) |> errormonitor status = status_request(app; statuscheck = true) - notify("ended:status:$status", app.id) + @async notify("ended:status:$status", app.id) |> errormonitor (:status => status) |> json end function status(_::Nothing) @@ -443,8 +462,8 @@ function setupwatch(path::AbstractString, appid::Int, app::Application) end end isempty(newest_file) && return - ApplicationsController.notify("changed:files", appid) - ApplicationsController.notify("filechanged:$newest_file", appid) + @async ApplicationsController.notify("changed:files", appid) |> errormonitor + @async ApplicationsController.notify("filechanged:$newest_file", appid) |> errormonitor end ]) @@ -525,7 +544,7 @@ function start(app::Application) end try - notify("started:start", app.id) + @async notify("started:start", app.id) |> errormonitor persist_status(app, STARTING_STATUS) appsthreads[fullpath(app)] = Base.Threads.@spawn begin @@ -613,7 +632,7 @@ function start(app::Application) save!(app) catch ex @error ex - notify("failed:start", app.id, FAILSTATUS, ERROR_STATUS) + @async notify("failed:start", app.id, FAILSTATUS, ERROR_STATUS) |> errormonitor if isempty(app.error) app.error = ex @@ -624,7 +643,15 @@ function start(app::Application) end end - notify("ended:start", app.id) + @async notify("ended:start", app.id) |> errormonitor + + @async GenieBuilder.Licensing.log(; + type = Actions.ACTION_START_APP, + metadata = Dict( + "name" => app.name, + "path" => app.path, + ) + ) |> errormonitor @async watch(fullpath(app), app.id.value) |> errormonitor @async tailapplog(app) |> errormonitor @@ -648,7 +675,7 @@ function start(app::Application) @error ex end - notify("failed:start", app.id, FAILSTATUS, ERROR_STATUS) + @async notify("failed:start", app.id, FAILSTATUS, ERROR_STATUS) |> errormonitor return (:status => FAILSTATUS) |> json end @@ -677,11 +704,11 @@ function tailapplog(app::Application) type = GenieDevTools.logtype(line) line = "log:message $line" line = Genie.WebChannels.tagbase64encode(line) - notify(; message = line, + @async notify(; message = line, appid = app.id, type = type, status = type == :error ? ERROR_STATUS : OKSTATUS, - ) + ) |> errormonitor if type == :error app.error = line persist_status(app, ERROR_STATUS) @@ -694,11 +721,11 @@ end function parselog(; line::AbstractString, type::Symbol, appid) :: Nothing output = GenieDevTools.parselog(line) if ! isnothing(output) - notify(; message = output, + @async notify(; message = output, appid = appid, type = type, status = ERROR_STATUS - ) + ) |> errormonitor end return @@ -714,20 +741,20 @@ function stop(app::Application) try persist_status(app, STOPPING_STATUS) - notify("started:stop", app.id) + @async notify("started:stop", app.id) |> errormonitor @async HTTP.request("GET", "$(apphost):$(app.port)$(GenieDevTools.defaultroute)/exit") |> errormonitor sleep(2) if status_request(app, false; statuscheck = true) == OFFLINE_STATUS - notify("ended:stop", app.id) + @async notify("ended:stop", app.id) |> errormonitor app.error = "" persist_status(app, OFFLINE_STATUS) end catch ex @error ex - notify("failed:stop", app.id, FAILSTATUS, ERROR_STATUS) + @async notify("failed:stop", app.id, FAILSTATUS, ERROR_STATUS) |> errormonitor status = ERROR_STATUS end @@ -746,6 +773,14 @@ function stop(app::Application) unwatch(fullpath(app), app.id) + @async GenieBuilder.Licensing.log(; + type = Actions.ACTION_STOP_APP, + metadata = Dict( + "name" => app.name, + "path" => app.path, + ) + ) |> errormonitor + (:status => status) |> json end stop(name::AbstractString = "", path::AbstractString = pwd()) = stop(findone(Application; name = isempty(name) ? name_from_path(path) : name)) @@ -810,15 +845,15 @@ Returns the contents of a directory from an app function dir(app::Application) if @isonline(app) res = try - notify("started:dir", app.id) + @async notify("started:dir", app.id) |> errormonitor HTTP.request("GET", "$(apphost):$(app.port)$(GenieDevTools.defaultroute)/dir?path=$(params(:path, "."))") catch ex @error ex - notify("failed:dir", app.id, FAILSTATUS, ERROR_STATUS) + @async notify("failed:dir", app.id, FAILSTATUS, ERROR_STATUS) |> errormonitor end - notify("ended:dir", app.id) + @async notify("ended:dir", app.id) |> errormonitor res |> json2json end @@ -832,15 +867,15 @@ Returns the contents of a file from an app function edit(app::Application) if @isonline(app) res = try - notify("started:edit", app.id) + @async notify("started:edit", app.id) |> errormonitor HTTP.request("GET", "$(apphost):$(app.port)$(GenieDevTools.defaultroute)/edit?path=$(params(:path, "."))") catch ex @error ex - notify("failed:edit", app.id, FAILSTATUS, ERROR_STATUS) + @async notify("failed:edit", app.id, FAILSTATUS, ERROR_STATUS) |> errormonitor end - notify("ended:edit", app.id) + @async notify("ended:edit", app.id) |> errormonitor res |> json2json end @@ -861,17 +896,17 @@ function save(app::Application) end res = try - notify("started:save", app.id) + @async notify("started:save", app.id) |> errormonitor HTTP.request( "POST", "$(apphost):$(app.port)$(GenieDevTools.defaultroute)/save?path=$(params(:path, "."))", [], HTTP.Form(Dict("payload" => jsonpayload()["payload"]))) catch ex @error ex - notify("failed:save", app.id, FAILSTATUS, ERROR_STATUS) + @async notify("failed:save", app.id, FAILSTATUS, ERROR_STATUS) |> errormonitor end - notify("ended:save", app.id) + @async notify("ended:save", app.id) |> errormonitor if ! is_existing_file && fp !== nothing unwatch(app.path, app.id) @@ -890,17 +925,17 @@ Returns the pages of an app function pages(app::Application) if @isonline(app) res = try - notify("started:pages", app.id) + @async notify("started:pages", app.id) |> errormonitor HTTP.request("GET", "$(apphost):$(app.port)$(GenieDevTools.defaultroute)/pages?CHANNEL__=$(app.channel)") |> json2json catch ex @error ex - notify("failed:pages", app.id, FAILSTATUS, ERROR_STATUS) + @async notify("failed:pages", app.id, FAILSTATUS, ERROR_STATUS) |> errormonitor (:status => :error) |> json end - notify("ended:pages", app.id) + @async notify("ended:pages", app.id) |> errormonitor res end @@ -938,12 +973,12 @@ Starts a remote REPL for an app function startrepl(app::Application) if @isonline(app) res = try - notify("started:startrepl", app.id) + @async notify("started:startrepl", app.id) |> errormonitor HTTP.request("GET", "$(apphost):$(app.port)$(GenieDevTools.defaultroute)/startrepl?port=$(app.replport != 0 ? app.replport : available_port())") catch ex @error ex - notify("failed:startrepl", app.id, FAILSTATUS, ERROR_STATUS) + @async notify("failed:startrepl", app.id, FAILSTATUS, ERROR_STATUS) |> errormonitor end status = try @@ -958,7 +993,7 @@ function startrepl(app::Application) @error ex end - notify("ended:startrepl", app.id) + @async notify("ended:startrepl", app.id) |> errormonitor status |> json end @@ -971,7 +1006,8 @@ Starts the package manager web app for an app """ function startpkgmng(app::Application) try - notify("started:pkgmng", app.id) + @async notify("started:pkgmng", app.id) |> errormonitor + cmd = Cmd(`julia --startup-file=no -e ' using Pkg; Pkg._auto_gc_enabled[] = false; @@ -989,11 +1025,13 @@ function startpkgmng(app::Application) @async cmd |> run |> errormonitor catch ex @error ex - notify("failed:pkgmng", app.id, FAILSTATUS, ERROR_STATUS) + @async notify("failed:pkgmng", app.id, FAILSTATUS, ERROR_STATUS) |> errormonitor rethrow(ex) end - notify("ended:startrepl", app.id) + @async notify("ended:startpkgmng", app.id) |> errormonitor + + @async GenieBuilder.Licensing.log(type = Actions.ACTION_START_PKG_MNG) |> errormonitor Dict( :status => OKSTATUS, @@ -1008,15 +1046,17 @@ Stops the package manager web app for an app """ function stoppkgmng(app::Application) res = try - notify("started:stoppkgmng", app.id) + @async notify("started:stoppkgmng", app.id) |> errormonitor HTTP.request("GET", "$(apphost):$(app.pkgmngport)$(GeniePackageManager.defaultroute)/exit") catch ex @error ex - notify("failed:stoppkgmng", app.id, FAILSTATUS, ERROR_STATUS) + @async notify("failed:stoppkgmng", app.id, FAILSTATUS, ERROR_STATUS) |> errormonitor end - notify("ended:stoppkgmng", app.id) + @async notify("ended:stoppkgmng", app.id) |> errormonitor + + @async GenieBuilder.Licensing.log(type = Actions.ACTION_STOP_PKG_MNG) |> errormonitor (:status => OKSTATUS) |> json end @@ -1073,7 +1113,7 @@ Notifies the GenieBuilder UI that it is ready to receive requests """ function ready() :: Nothing @info "GenieBuilder ready! RUN_STATUS = $(GenieBuilder.RUN_STATUS[])" - notify("ended:gbstart", nothing) + @async notify("ended:gbstart", nothing) |> errormonitor nothing end @@ -1149,17 +1189,26 @@ function download(app::Application) close(w) end + @async GenieBuilder.Licensing.log(; + type = Actions.ACTION_DOWNLOAD_APP, + metadata = Dict( + "name" => app.name, + "path" => app.path, + ) + ) |> errormonitor + Genie.Router.download("$appname.zip", root = zip_temp_path) end -function send_user_message(; text, button_text = "", button_link = "") - notify(; message = JSON3.write( +function send_user_message(; text, button_text = "", button_link = "") :: Nothing + @async notify(; message = JSON3.write( Dict( :message => text, :button_text => button_text, :button_link => button_link ) ), - type = "show_info_message") + type = "show_info_message") |> errormonitor + nothing end function start_session() diff --git a/routes.jl b/routes.jl index dd1caee..b578422 100755 --- a/routes.jl +++ b/routes.jl @@ -6,7 +6,7 @@ using RemoteREPL using JSON3 import GenieDevTools using Logging -using GenieBuilder.Licensing +using GenieBuilder.Licensing, GenieBuilder.Actions Genie.config.websockets_server = true @@ -62,6 +62,8 @@ function startrepl() port = ApplicationsController.available_port() |> first @async serve_repl(port) |> errormonitor + @async GenieBuilder.Licensing.log(type = Actions.ACTION_START_REPL) |> errormonitor + port end @@ -99,22 +101,14 @@ function register_routes() # registers a new path as a GenieBuilder app route("$api_route/apps/register") do - @async GenieBuilder.Licensing.log("GenieBuilder.jl", - "apps-register", - Dict( - "name" => params(:name, ""), - "path" => params(:path, pwd()) - ) - ) |> errormonitor - register(params(:name, ""), params(:path, pwd())) end # creates the Genie app skeleton route("$api_route/apps/create") do - @async GenieBuilder.Licensing.log("GenieBuilder.jl", - "apps-create", - Dict( + @async GenieBuilder.Licensing.log(; + type = Actions.ACTION_CREATE_APP, + metadata = Dict( "name" => params(:name, ""), "path" => params(:path, pwd()) )) |> errormonitor @@ -166,14 +160,6 @@ function register_routes() ApplicationsController.save(params(:appid) |> ApplicationsController.get) end - route("$api_route$app_route/log") do - ApplicationsController.log(params(:appid) |> ApplicationsController.get) - end - - route("$api_route$app_route/errors") do - ApplicationsController.errors(params(:appid) |> ApplicationsController.get) - end - # returns the pages of an app route("$api_route$app_route/pages") do ApplicationsController.pages(params(:appid) |> ApplicationsController.get) diff --git a/src/Actions.jl b/src/Actions.jl new file mode 100644 index 0000000..9aad69d --- /dev/null +++ b/src/Actions.jl @@ -0,0 +1,15 @@ +module Actions + +ACTION_START_SESSION = "start_session" # start GB +ACTION_END_SESSION = "end_session" # stop GB +ACTION_REGISTER_APP = "register_app" # register app +ACTION_UNREGISTER_APP = "unregister_app" # unregister app +ACTION_CREATE_APP = "create_app" # create app +ACTION_START_APP = "start_app" # start app +ACTION_STOP_APP = "stop_app" # stop app +ACTION_START_REPL = "start_repl" # start REPL +ACTION_START_PKG_MNG = "start_pkg_mng" # start package manager +ACTION_STOP_PKG_MNG = "stop_pkg_mng" # stop package manager +ACTION_DOWNLOAD_APP = "download_app" # download app + +end \ No newline at end of file diff --git a/src/GenieBuilder.jl b/src/GenieBuilder.jl index e1404b2..4aa96b1 100755 --- a/src/GenieBuilder.jl +++ b/src/GenieBuilder.jl @@ -7,8 +7,10 @@ import Pkg include("Generators.jl") include("Licensing.jl") +include("Actions.jl") using .Generators using .Licensing +using .Actions const GBDIR = Ref{String}("") const DB_FOLDER = Ref{String}("") @@ -100,6 +102,14 @@ function _go(port) port = port == -1 ? Genie.config.server_port : port + @async Licensing.log( + type = Actions.ACTION_START_SESSION, + payload = Dict( + :port => port, + :version => get_version() + ) + ) |> errormonitor + Genie.up(port; async = false) catch ex @error ex @@ -150,6 +160,7 @@ function get_version() end function exit() + @async GenieBuilder.Licensing.log(type = Actions.ACTION_END_SESSION) |> errormonitor Genie.Server.down!() Base.exit() end diff --git a/src/Licensing.jl b/src/Licensing.jl index a643f8e..f5f81cf 100644 --- a/src/Licensing.jl +++ b/src/Licensing.jl @@ -3,20 +3,15 @@ Licensing integration for JuliaHub """ module Licensing -using HTTP, JSON, Logging, Random +using Base64, HTTP, JSON, Logging, Random -const LICENSE_API = get!(ENV, "GENIE_LICENSE_API", "https://genielicensing.hosting.genieframework.com/api/v1") # no trailing slash -# const LICENSE_API = get!(ENV, "GENIE_LICENSE_API", "http://127.0.0.1:3333/api/v1") # no trailing slash #TODO: change this to the production URL -const FAILED_SESSION_ID = "" +# const LICENSE_API = get!(ENV, "GENIE_LICENSE_API", "https://genielicensing.hosting.genieframework.com/api/v1") # no trailing slash +const LICENSE_API = get!(ENV, "GENIE_LICENSE_API", "http://127.0.0.1:3333/api/v1") # no trailing slash #TODO: change this to the production URL const GBFOLDER = joinpath(homedir(), ".geniebuilder") const GBPASSFILE = joinpath(GBFOLDER, "pass") const GBSESSIONFILE = joinpath(GBFOLDER, "sessionid") -@inline function is_failed_session() :: Bool - get(ENV, "GENIE_SESSION", FAILED_SESSION_ID) == FAILED_SESSION_ID -end - function isregistered() :: Bool isfile(GBPASSFILE) end @@ -180,9 +175,6 @@ function logoff() :: Nothing end function start_session() - subscription_info = fetch_subscription_info() # fetch JH subscription info TODO: - @async update_user_info(subscription_info) |> errormonitor # update user info in the GLS backend TODO: - if is_subscription_expired(subscription_info) @warn("Subscription expired") logoff() @@ -219,47 +211,41 @@ function start_session() ENV["GENIE_SESSION"] end -function headers() +function headers(; content_type::AbstractString = "application/json") Dict( "Authorization" => "Bearer " * ENV["GENIE_SESSION"], - "Content-Type" => "application/json" + "Content-Type" => content_type ) end -# TODO: implement this -function fetch_subscription_info() - # 1/ retrieve the subscription info from JuliaHub API and return it - - Dict() -end - -# TODO: implement this -function update_user_info(subscription_info) :: Nothing - # 1/ update the user info in the GLS backend - - nothing -end - function is_subscription_expired(subscription_info) :: Bool # 1/ check if the subscription is expired false end -function log(origin, type, payload::AbstractDict) - if is_failed_session() - @warn("No session found, skipping logging") +# log an action +function log(; + type::Union{AbstractString,Symbol}, + metadata::AbstractDict = Dict(), + origin::AbstractString = ORIGIN) + if ! isloggedin() + @warn("User not logged in, skipping logging") return end + payload = Dict( + "origin" => origin, + "type" => type, + "metadata" => metadata + ) + response = try - HTTP.post(LICENSE_API * "/action"; - body = Dict( - "origin" => origin, - "type" => type, - "metadata" => payload |> JSON.json - ), + HTTP.get(LICENSE_API * "/action?payload=$(JSON.json(payload) |> base64encode)"; headers = headers(), - status_exception = true + status_exception = true, + detect_content_type = true, + verbose = true, + cookies = false, ) catch ex @warn("Failed to log action: $ex") @@ -267,13 +253,12 @@ function log(origin, type, payload::AbstractDict) return end - @show response - nothing end function quotas() - if is_failed_session() + if ! isloggedin() + @warn("User not logged in, skipping quotas") return Dict() end @@ -302,10 +287,6 @@ function quotas() end function status() - if is_failed_session() - return Dict() - end - quotas_data = try HTTP.get(LICENSE_API * "/status"; status_exception = true, @@ -318,13 +299,14 @@ function status() end if quotas_data.status != 200 + @warn("Failed to get status: $(quotas_data.status)") return Dict() end try (quotas_data.body |> String |> JSON.parse) catch ex - # @warn("Failed to parse status data: $ex") + @warn("Failed to parse status data: $ex") return Dict() end end @@ -332,13 +314,13 @@ end function __init__() #TODO: uncouple this ENV["GENIE_USER_EMAIL"] = get(ENV, "JULIAHUB_USEREMAIL", "") ENV["GENIE_USER_FULL_NAME"] = get(ENV, "JULIAHUB_USER_FULL_NAME", "") - ENV["GENIE_ORIGIN"] = get(ENV, "JULIA_PKG_SERVER", "JULIAHUB") + ENV["GENIE_ORIGIN"] = get(ENV, "JULIA_PKG_SERVER", "local") ENV["GENIE_METADATA"] = get(ENV, "JULIAHUB_APP_URL", "") - @eval const USER_EMAIL = get!(ENV, "GENIE_USER_EMAIL", "") - @eval const USER_FULL_NAME = get!(ENV, "GENIE_USER_FULL_NAME", "") - @eval const ORIGIN = get!(ENV, "GENIE_ORIGIN", "JULIAHUB") - @eval const METADATA = get!(ENV, "GENIE_METADATA", "") + @eval const USER_EMAIL = ENV["GENIE_USER_EMAIL"] + @eval const USER_FULL_NAME = ENV["GENIE_USER_FULL_NAME"] + @eval const ORIGIN = ENV["GENIE_ORIGIN"] + @eval const METADATA = ENV["GENIE_METADATA"] end