Skip to content

Commit

Permalink
[Winlogbeat] Convert dashboards for Kibana 8.x (#37085) (#37118)
Browse files Browse the repository at this point in the history 
Migrate the saved dashboards to 8.x format.

The libbeat code that loads dashboards is hard-coded to look into the "7" directory. So I renamed the 8 dir back to 7.

Fix "Number of Events Over Time By Channel" visualization. One attribute related to the axis was giving an error. Editing and re-saving fixed it.

[git-generate]
cd x-pack/winlogbeat

export KIBANA_INSECURE=true
export KIBANA_URL="https://elastic:changeme@localhost:5601"

mage dashboards:import

for module in powershell security sysmon; do
  for id in $(jq -r .id module/$module/_meta/kibana/7/dashboard/*.json); do
    MODULE=$module ID=$id mage dashboards:export
  done
done

git add module/*/_meta/kibana/8
git rm -r module/*/_meta/kibana/7

cd ../../winlogbeat
go run ../dev-tools/cmd/dashboards/export_dashboards.go -kibana="https://elastic:changeme@localhost:5601" -folder . -insecure -dashboard Winlogbeat-Dashboard-ecs
git rm -r _meta/kibana/7

(cherry picked from commit 17d5bb1)

Co-authored-by: Andrew Kroh <andrew.kroh@elastic.co>
  • Loading branch information
@mergify @andrewkroh
mergify[bot] and andrewkroh authored Nov 21, 2023
1 parent f729442 commit c7d3709
Show file tree
Hide file tree
Showing 128 changed files with 1,362 additions and 783 deletions.
2 changes: 2 additions & 0 deletions CHANGELOG.next.asciidoc
View file
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,8 @@ https://github.com/elastic/beats/compare/v8.8.1\...main[Check the HEAD diff]

*Winlogbeat*

- Fix dashboards under Kibana 8.x. {issue}37080[37080] {pull}37085[37085]


*Elastic Logging Plugin*

Expand Down
7 changes: 6 additions & 1 deletion dev-tools/cmd/dashboards/export_dashboards.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ import (
"github.com/elastic/beats/v7/libbeat/version"
"github.com/elastic/elastic-agent-libs/kibana"
"github.com/elastic/elastic-agent-libs/transport/httpcommon"
"github.com/elastic/elastic-agent-libs/transport/tlscommon"
)

var (
Expand All @@ -42,6 +43,7 @@ const (

func main() {
kibanaURL := flag.String("kibana", "http://localhost:5601", "Kibana URL")
insecure := flag.Bool("insecure", false, "Disable TLS verification.")
spaceID := flag.String("space-id", "", "Space ID")
dashboard := flag.String("dashboard", "", "Dashboard ID")
fileOutput := flag.String("output", "", "Output NDJSON file, when exporting dashboards for Beats, please use -folder instead")
Expand Down Expand Up @@ -71,6 +73,9 @@ func main() {
}
transport := httpcommon.DefaultHTTPTransportSettings()
transport.Timeout = kibanaTimeout
if *insecure {
transport.TLS = &tlscommon.Config{VerificationMode: tlscommon.VerifyNone}
}

client, err := kibana.NewClientWithConfig(&kibana.ClientConfig{
Protocol: u.Scheme,
Expand Down Expand Up @@ -133,7 +138,7 @@ func exportDashboardsFromYML(client *kibana.Client, ymlFile string) error {
func exportSingleDashboard(client *kibana.Client, dashboard, folder string) error {
result, err := dashboards.Export(client, dashboard)
if err != nil {
return fmt.Errorf("failed to export the dashboard: %+v", err)
return fmt.Errorf("failed to export the dashboard: %w", err)
}
result = dashboards.DecodeExported(result)
return dashboards.SaveToFolder(result, folder, client.GetVersion())
Expand Down
7 changes: 7 additions & 0 deletions dev-tools/mage/dashboard.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,9 @@ func ExportDashboard() error {
if kibanaURL := EnvOr("KIBANA_URL", ""); kibanaURL != "" {
args = append(args, "-kibana", kibanaURL)
}
if kibanaInsecure, _ := strconv.ParseBool(os.Getenv("KIBANA_INSECURE")); kibanaInsecure {
args = append(args, "-insecure")
}

return dashboardCmd(args...)
}
Expand All @@ -65,6 +68,7 @@ func ExportDashboard() error {
//
// Optional environment variables:
// - KIBANA_URL: URL of Kibana
// - KIBANA_INSECURE: Disable TLS verification.
// - KIBANA_ALWAYS: Connect to Kibana without checking ES version. Default true.
// - ES_URL: URL of Elasticsearch (only used with KIBANA_ALWAYS=false).
func ImportDashboards(buildDep, dashboardDep interface{}) error {
Expand All @@ -83,6 +87,9 @@ func ImportDashboards(buildDep, dashboardDep interface{}) error {
if kibanaURL := EnvOr("KIBANA_URL", ""); kibanaURL != "" {
args = append(args, "-E", "setup.kibana.host="+kibanaURL)
}
if kibanaInsecure, _ := strconv.ParseBool(os.Getenv("KIBANA_INSECURE")); kibanaInsecure {
args = append(args, "-E", "setup.kibana.ssl.verification_mode=none")
}
if esURL := EnvOr("ES_URL", ""); !kibanaAlways && esURL != "" {
args = append(args, "-E", "setup.elasticsearch.host="+esURL)
}
Expand Down
2 changes: 2 additions & 0 deletions dev-tools/mage/target/dashboards/dashboards.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ type Dashboards mg.Namespace
//
// Optional environment variables:
// - KIBANA_URL: URL of Kibana
// - KIBANA_INSECURE: Disable TLS verification.
// - KIBANA_ALWAYS: Connect to Kibana without checking ES version. Default true.
// - ES_URL: URL of Elasticsearch (only used with KIBANA_ALWAYS=false).
func (Dashboards) Import() error {
Expand All @@ -59,6 +60,7 @@ func (Dashboards) Import() error {
//
// Required environment variables:
// - KIBANA_URL: URL of Kibana
// - KIBANA_INSECURE: Disable TLS verification.
// - MODULE: Name of the module
// - ID: Dashboard ID
func (Dashboards) Export() error {
Expand Down
110 changes: 101 additions & 9 deletions winlogbeat/_meta/kibana/7/dashboard/Winlogbeat-Dashboard-ecs.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,19 +3,110 @@
"description": "Overview of all Windows Event Logs.",
"hits": 0,
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"filter\":[],\"query\":{\"language\":\"kuery\",\"query\":\"\"}}"
"searchSourceJSON": {
"filter": [],
"query": {
"language": "kuery",
"query": ""
}
}
},
"optionsJSON": "{\"darkTheme\": false}",
"panelsJSON": "[{\"version\":\"7.0.0-SNAPSHOT\",\"gridData\":{\"h\":20,\"i\":\"1\",\"w\":36,\"x\":12,\"y\":0},\"panelIndex\":\"1\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_0\"},{\"version\":\"7.0.0-SNAPSHOT\",\"gridData\":{\"h\":20,\"i\":\"3\",\"w\":12,\"x\":0,\"y\":0},\"panelIndex\":\"3\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_1\"},{\"version\":\"7.0.0-SNAPSHOT\",\"gridData\":{\"h\":20,\"i\":\"4\",\"w\":16,\"x\":16,\"y\":20},\"panelIndex\":\"4\",\"embeddableConfig\":{\"vis\":{\"params\":{\"sort\":{\"columnIndex\":null,\"direction\":null}}},\"enhancements\":{}},\"panelRefName\":\"panel_2\"},{\"version\":\"7.0.0-SNAPSHOT\",\"gridData\":{\"h\":20,\"i\":\"5\",\"w\":16,\"x\":32,\"y\":20},\"panelIndex\":\"5\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_3\"},{\"version\":\"7.0.0-SNAPSHOT\",\"gridData\":{\"h\":20,\"i\":\"6\",\"w\":16,\"x\":0,\"y\":20},\"panelIndex\":\"6\",\"embeddableConfig\":{\"enhancements\":{}},\"panelRefName\":\"panel_4\"}]",
"optionsJSON": {
"darkTheme": false
},
"panelsJSON": [
{
"embeddableConfig": {
"enhancements": {}
},
"gridData": {
"h": 20,
"i": "1",
"w": 36,
"x": 12,
"y": 0
},
"panelIndex": "1",
"panelRefName": "panel_0",
"version": "7.0.0-SNAPSHOT"
},
{
"embeddableConfig": {
"enhancements": {}
},
"gridData": {
"h": 20,
"i": "3",
"w": 12,
"x": 0,
"y": 0
},
"panelIndex": "3",
"panelRefName": "panel_1",
"version": "7.0.0-SNAPSHOT"
},
{
"embeddableConfig": {
"enhancements": {},
"vis": {
"params": {
"sort": {
"columnIndex": null,
"direction": null
}
}
}
},
"gridData": {
"h": 20,
"i": "4",
"w": 16,
"x": 16,
"y": 20
},
"panelIndex": "4",
"panelRefName": "panel_2",
"version": "7.0.0-SNAPSHOT"
},
{
"embeddableConfig": {
"enhancements": {}
},
"gridData": {
"h": 20,
"i": "5",
"w": 16,
"x": 32,
"y": 20
},
"panelIndex": "5",
"panelRefName": "panel_3",
"version": "7.0.0-SNAPSHOT"
},
{
"embeddableConfig": {
"enhancements": {}
},
"gridData": {
"h": 20,
"i": "6",
"w": 16,
"x": 0,
"y": 20
},
"panelIndex": "6",
"panelRefName": "panel_4",
"version": "7.0.0-SNAPSHOT"
}
],
"timeRestore": false,
"title": "[Winlogbeat] Overview",
"version": 1
},
"coreMigrationVersion": "7.14.0",
"coreMigrationVersion": "8.8.0",
"created_at": "2023-11-10T17:45:36.656Z",
"id": "Winlogbeat-Dashboard-ecs",
"migrationVersion": {
"dashboard": "7.14.0"
},
"managed": false,
"references": [
{
"id": "Number-of-Events-Over-Time-By-Event-Log-ecs",
Expand Down Expand Up @@ -44,6 +135,7 @@
}
],
"type": "dashboard",
"updated_at": "2021-08-11T13:18:52.287Z",
"version": "WzIxNzMsMl0="
"typeMigrationVersion": "8.9.0",
"updated_at": "2023-11-10T17:45:36.656Z",
"version": "WzEwNiwxXQ=="
}
102 changes: 93 additions & 9 deletions winlogbeat/_meta/kibana/7/visualization/Event-Levels-ecs.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,18 +2,101 @@
"attributes": {
"description": "",
"kibanaSavedObjectMeta": {
"searchSourceJSON": "{\"filter\": [], \"indexRefName\": \"kibanaSavedObjectMeta.searchSourceJSON.index\", \"query\": {\"language\": \"kuery\", \"query\": \"\"}}"
"searchSourceJSON": {
"filter": [],
"indexRefName": "kibanaSavedObjectMeta.searchSourceJSON.index",
"query": {
"language": "kuery",
"query": ""
}
}
},
"title": "Event Levels [Winlogbeat Overview]",
"uiStateJSON": "{\"vis\": {\"params\": {\"sort\": {\"columnIndex\": null, \"direction\": null}}}}",
"uiStateJSON": {
"vis": {
"params": {
"sort": {
"columnIndex": null,
"direction": null
}
}
}
},
"version": 1,
"visState": "{\"aggs\":[{\"enabled\":true,\"id\":\"1\",\"params\":{},\"schema\":\"metric\",\"type\":\"count\"},{\"enabled\":true,\"id\":\"2\",\"params\":{\"customLabel\":\"Log Levels\",\"field\":\"log.level\",\"missingBucket\":false,\"missingBucketLabel\":\"Missing\",\"order\":\"desc\",\"orderBy\":\"1\",\"otherBucket\":false,\"otherBucketLabel\":\"Other\",\"size\":5},\"schema\":\"bucket\",\"type\":\"terms\"}],\"params\":{\"dimensions\":{\"buckets\":[{\"accessor\":0,\"aggType\":\"terms\",\"format\":{\"id\":\"terms\",\"params\":{\"id\":\"string\",\"missingBucketLabel\":\"Missing\",\"otherBucketLabel\":\"Other\"}},\"params\":{}}],\"metrics\":[{\"accessor\":1,\"aggType\":\"count\",\"format\":{\"id\":\"number\"},\"params\":{}}]},\"perPage\":10,\"showMetricsAtAllLevels\":false,\"showPartialRows\":false,\"showTotal\":false,\"sort\":{\"columnIndex\":null,\"direction\":null},\"totalFunc\":\"sum\",\"showToolbar\":true},\"title\":\"Event Levels [Winlogbeat Overview]\",\"type\":\"table\"}"
"visState": {
"aggs": [
{
"enabled": true,
"id": "1",
"params": {},
"schema": "metric",
"type": "count"
},
{
"enabled": true,
"id": "2",
"params": {
"customLabel": "Log Levels",
"field": "log.level",
"missingBucket": false,
"missingBucketLabel": "Missing",
"order": "desc",
"orderBy": "1",
"otherBucket": false,
"otherBucketLabel": "Other",
"size": 5
},
"schema": "bucket",
"type": "terms"
}
],
"params": {
"dimensions": {
"buckets": [
{
"accessor": 0,
"aggType": "terms",
"format": {
"id": "terms",
"params": {
"id": "string",
"missingBucketLabel": "Missing",
"otherBucketLabel": "Other"
}
},
"params": {}
}
],
"metrics": [
{
"accessor": 1,
"aggType": "count",
"format": {
"id": "number"
},
"params": {}
}
]
},
"perPage": 10,
"showMetricsAtAllLevels": false,
"showPartialRows": false,
"showToolbar": true,
"showTotal": false,
"sort": {
"columnIndex": null,
"direction": null
},
"totalFunc": "sum"
},
"title": "Event Levels [Winlogbeat Overview]",
"type": "table"
}
},
"coreMigrationVersion": "7.14.0",
"coreMigrationVersion": "8.8.0",
"created_at": "2023-11-10T17:45:36.656Z",
"id": "Event-Levels-ecs",
"migrationVersion": {
"visualization": "7.14.0"
},
"managed": false,
"references": [
{
"id": "winlogbeat-*",
Expand All @@ -22,6 +105,7 @@
}
],
"type": "visualization",
"updated_at": "2021-08-11T13:18:52.287Z",
"version": "WzIxNzcsMl0="
"typeMigrationVersion": "8.5.0",
"updated_at": "2023-11-10T17:45:36.656Z",
"version": "WzEwNCwxXQ=="
}
Loading

0 comments on commit c7d3709

Please sign in to comment.