Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add functionality for structured logs to render links in tlparse #36

Merged
merged 1 commit into from
May 16, 2024
Merged

Add functionality for structured logs to render links in tlparse #36

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,4 @@
/out
/tl_out
*.log
!/tests/inputs/*.log
3 changes: 2 additions & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,8 +33,9 @@ impl StructuredLogParser for MyCustomParser {
_rank: Option<u32>,
compile_id: &Option<CompileId>,
payload: &str
) -> anyhow::Result<ParseOutput> {
) -> anyhow::Result<ParserResult> {
// Use the metadata and payload however you'd like
// Return either a ParserOutput::File(filename, payload) or ParserOutput::Link(name, url)
}
}
```
23 changes: 18 additions & 5 deletions src/lib.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ use std::time::Instant;
use tinytemplate::TinyTemplate;

use crate::parsers::default_parsers;
use crate::parsers::ParserOutput;
use crate::templates::*;
use crate::types::*;
mod parsers;
Expand Down Expand Up @@ -101,7 +102,10 @@ pub fn parse_path(path: &PathBuf, config: ParseConfig) -> anyhow::Result<ParseOu

let mut expected_rank: Option<Option<u32>> = None;

let mut directory: FxIndexMap<Option<CompileId>, Vec<(PathBuf, i32)>> = FxIndexMap::default();
// Each entry is a compile id => (link, rendered name, output number)
// For files, link and rendered name are the same
// For links, you can specify a custom name for the link
let mut directory: FxIndexMap<Option<CompileId>, Vec<(String, String, i32)>> = FxIndexMap::default();

let mut metrics_index: CompilationMetricsIndex = FxIndexMap::default();
let stack_index: RefCell<StackIndex> = RefCell::new(FxHashMap::default());
Expand Down Expand Up @@ -234,10 +238,19 @@ pub fn parse_path(path: &PathBuf, config: ParseConfig) -> anyhow::Result<ParseOu
let results = parser.parse(lineno, md, e.rank, &e.compile_id, &payload);
match results {
Ok(results) => {
for (filename, out) in results {
output.push((filename.clone(), out));
compile_directory.push((filename, output_count));
output_count += 1;
for parser_result in results {
match parser_result {
ParserOutput::File(filename, out) => {
output.push((filename.clone(), out));
let filename_str = format!("{}", filename.to_string_lossy());
compile_directory.push((filename_str.clone(), filename_str, output_count));
output_count += 1;
}
ParserOutput::Link(name, url) => {
compile_directory.push((url, name, output_count));
output_count += 1;
}
}
}
}
Err(err) => match parser.name() {
Expand Down
60 changes: 49 additions & 11 deletions src/parsers.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,15 @@ use tinytemplate::TinyTemplate;

use syntect::highlighting::ThemeSet;
use syntect::parsing::SyntaxSet;

pub enum ParserOutput {
File(PathBuf, String), // File to be saved on disk
Link(String, String), // External href to (name, url) (linked in compile_directory, not returned)
}

// Each parser returns a list of files to save and links to render in compile directory
pub type ParserResults = Vec<ParserOutput>;

/**
* StructuredLogParser
* Parses a structured log and returns a vec of file outputs.
Expand All @@ -27,19 +36,20 @@ pub trait StructuredLogParser {
rank: Option<u32>, // Rank of the log
compile_id: &Option<CompileId>, // Compile ID of the envelope
payload: &str, // Payload from the log (empty string when None)
) -> anyhow::Result<ParseOutput>;
) -> anyhow::Result<ParserResults>;

// Name of the parser, for error logging
fn name(&self) -> &'static str;
}


// Takes a filename and a payload and writes that payload into a the file
fn simple_file_output(
filename: &str,
lineno: usize,
compile_id: &Option<CompileId>,
payload: &str,
) -> anyhow::Result<ParseOutput> {
) -> anyhow::Result<ParserResults> {
let compile_id_dir: PathBuf = compile_id
.as_ref()
.map_or(
Expand All @@ -53,7 +63,7 @@ fn simple_file_output(
.into();
let subdir = PathBuf::from(compile_id_dir);
let f = subdir.join(filename);
Ok(Vec::from([(f, String::from(payload))]))
Ok(Vec::from([ParserOutput::File(f, String::from(payload))]))
}

/**
Expand Down Expand Up @@ -88,7 +98,7 @@ impl StructuredLogParser for SentinelFileParser {
_rank: Option<u32>,
compile_id: &Option<CompileId>,
payload: &str,
) -> anyhow::Result<ParseOutput> {
) -> anyhow::Result<ParserResults> {
simple_file_output(
&format!("{}.txt", self.filename),
lineno,
Expand Down Expand Up @@ -116,7 +126,7 @@ impl StructuredLogParser for GraphDumpParser {
_rank: Option<u32>,
compile_id: &Option<CompileId>,
payload: &str,
) -> anyhow::Result<ParseOutput> {
) -> anyhow::Result<ParserResults> {
if let Metadata::GraphDump(metadata) = metadata {
let filename: PathBuf = {
let mut r = OsString::from(&metadata.name);
Expand Down Expand Up @@ -148,7 +158,7 @@ impl StructuredLogParser for DynamoOutputGraphParser {
_rank: Option<u32>,
compile_id: &Option<CompileId>,
payload: &str,
) -> anyhow::Result<ParseOutput> {
) -> anyhow::Result<ParserResults> {
simple_file_output("dynamo_output_graph.txt", lineno, compile_id, payload)
}
}
Expand All @@ -170,7 +180,7 @@ impl StructuredLogParser for DynamoGuardParser<'_> {
_rank: Option<u32>,
compile_id: &Option<CompileId>,
payload: &str,
) -> anyhow::Result<ParseOutput> {
) -> anyhow::Result<ParserResults> {
let filename = format!("{}.html", self.name());
let guards = serde_json::from_str::<Vec<DynamoGuard>>(payload)?;
let guards_context = DynamoGuardsContext { guards };
Expand All @@ -197,7 +207,7 @@ impl StructuredLogParser for InductorOutputCodeParser {
_rank: Option<u32>,
compile_id: &Option<CompileId>,
payload: &str,
) -> anyhow::Result<ParseOutput> {
) -> anyhow::Result<ParserResults> {
if let Metadata::InductorOutputCode(metadata) = metadata {
let filename = metadata
.filename
Expand Down Expand Up @@ -260,7 +270,7 @@ impl StructuredLogParser for OptimizeDdpSplitChildParser {
_rank: Option<u32>,
compile_id: &Option<CompileId>,
payload: &str,
) -> anyhow::Result<ParseOutput> {
) -> anyhow::Result<ParserResults> {
if let Metadata::OptimizeDdpSplitChild(m) = metadata {
let filename = format!("optimize_ddp_split_child_{}.txt", m.name);
simple_file_output(&filename, lineno, compile_id, payload)
Expand All @@ -270,6 +280,33 @@ impl StructuredLogParser for OptimizeDdpSplitChildParser {
}
}

pub struct LinkParser;
impl StructuredLogParser for LinkParser {
fn name(&self) -> &'static str {
"link_parser"
}
fn get_metadata<'e>(&self, e: &'e Envelope) -> Option<Metadata<'e>> {
e.link
.as_ref()
.map(|m| Metadata::Link(m))
}

fn parse<'e>(
&self,
_lineno: usize,
metadata: Metadata<'e>,
_rank: Option<u32>,
_compile_id: &Option<CompileId>,
_payload: &str,
) -> anyhow::Result<ParserResults> {
if let Metadata::Link(m) = metadata {
Ok(Vec::from([ParserOutput::Link(m.name.clone(), m.url.clone())]))
} else {
Err(anyhow::anyhow!("Expected Link Metadata"))
}
}
}

pub struct CompilationMetricsParser<'t> {
tt: &'t TinyTemplate<'t>,
stack_index: &'t RefCell<StackIndex>,
Expand All @@ -290,7 +327,7 @@ impl StructuredLogParser for CompilationMetricsParser<'_> {
_rank: Option<u32>,
compile_id: &Option<CompileId>,
_payload: &str,
) -> anyhow::Result<ParseOutput> {
) -> anyhow::Result<ParserResults> {
let filename = format!("{}.html", self.name());
if let Metadata::CompilationMetrics(m) = metrics {
let id = compile_id
Expand Down Expand Up @@ -342,7 +379,7 @@ impl StructuredLogParser for AOTAutogradBackwardCompilationMetricsParser<'_> {
_rank: Option<u32>,
compile_id: &Option<CompileId>,
_payload: &str,
) -> anyhow::Result<ParseOutput> {
) -> anyhow::Result<ParserResults> {
let filename = format!("{}.html", self.name());
if let Metadata::AOTAutogradBackwardCompilationMetrics(m) = metrics {
let id = compile_id
Expand Down Expand Up @@ -395,6 +432,7 @@ pub fn default_parsers<'t>(
Box::new(OptimizeDdpSplitChildParser),
Box::new(CompilationMetricsParser { tt, stack_index }), // TODO: use own tt instances
Box::new(AOTAutogradBackwardCompilationMetricsParser { tt }), // TODO: use own tt instances
Box::new(LinkParser),
];

result
Expand Down
2 changes: 1 addition & 1 deletion src/templates.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ Build products below:
<li><a id="{compile_directory.0}">{compile_directory.0}</a>
<ul>
{{ for path_idx in compile_directory.1 }}
<li><a href="{path_idx.0}">{path_idx.0}</a> ({path_idx.1})</li>
<li><a href="{path_idx.0}">{path_idx.1}</a> ({path_idx.2})</li>
{{ endfor }}
</ul>
</li>
Expand Down
12 changes: 11 additions & 1 deletion src/types.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,8 @@ use once_cell::sync::Lazy;
use serde::{Deserialize, Serialize};
use std::sync::Mutex;


// Main function returns a list of files to save
pub type ParseOutput = Vec<(PathBuf, String)>;
pub type CompilationMetricsIndex = FxIndexMap<Option<CompileId>, Vec<CompilationMetricsMetadata>>;
pub type StackIndex = FxHashMap<Option<CompileId>, StackSummary>; // NB: attempt is always 0 here
Expand Down Expand Up @@ -225,6 +227,12 @@ pub struct InductorOutputCodeMetadata {
pub filename: Option<PathBuf>,
}

#[derive(Debug, Deserialize)]
pub struct LinkMetadata {
pub name: String,
pub url: String,
}

#[derive(Debug, Deserialize, Serialize)]
pub struct CompilationMetricsMetadata {
// Other information like frame_key, co_name, etc. are already in envelope
Expand Down Expand Up @@ -316,6 +324,7 @@ pub struct RestartsAndFailuresContext {
#[derive(Debug)]
pub enum Metadata<'e> {
Empty(&'e EmptyMetadata),
Link(&'e LinkMetadata),
GraphDump(&'e GraphDumpMetadata),
DynamoOutputGraph(&'e DynamoOutputGraphMetadata),
#[allow(dead_code)]
Expand Down Expand Up @@ -351,6 +360,7 @@ pub struct Envelope {
pub aot_autograd_backward_compilation_metrics:
Option<AOTAutogradBackwardCompilationMetricsMetadata>,
pub graph_dump: Option<GraphDumpMetadata>,
pub link: Option<LinkMetadata>,
}

#[derive(Debug, Deserialize, Serialize)]
Expand All @@ -369,7 +379,7 @@ pub struct DynamoGuardsContext {
pub struct IndexContext {
pub css: &'static str,
pub javascript: &'static str,
pub directory: Vec<(String, Vec<(PathBuf, i32)>)>,
pub directory: Vec<(String, Vec<(String, String, i32)>)>,
pub stack_trie_html: String,
pub unknown_stack_trie_html: String,
pub has_unknown_stack_trie: bool,
Expand Down
Loading
Loading