Makefile.in

#=========================================================================
# Makefile
#=========================================================================
# SPDX-License-Identifier: MIT
# Author : Christopher Batten, NVIDIA
# Date   : May 20, 2024

#-------------------------------------------------------------------------
# Basic setup
#-------------------------------------------------------------------------

SHELL:=/bin/bash

# Remove all default implicit rules since they can cause subtle bugs
# and they just make things run slower
.SUFFIXES:
% : %,v
% : RCS/%,v
% : RCS/%
% : s.%
% : SCCS/s.%

# Default is to build the prereqs of the all target (defined at bottom)
default : all
.PHONY : default

src_dir      := @srcdir@
scripts_dir  := $(src_dir)/scripts

#-------------------------------------------------------------------------
# Configure
#-------------------------------------------------------------------------

include config.mk

#-------------------------------------------------------------------------
# Programs
#-------------------------------------------------------------------------

GENERATE=$(scripts_dir)/generate

GENERATE_FLAGS := "--model=$(cfg_model)"

num_examples := $(cfg_examples)
ifneq ($(num_examples),0)
  GENERATE_FLAGS += "--examples=$(num_examples)"
endif

GENERATE_VERILOG_FLAGS := $(GENERATE_FLAGS)
GENERATE_VERILOG_FLAGS += "--lang=verilog"

GENERATE_PYDSL_FLAGS := $(GENERATE_FLAGS)
pydsl := $(cfg_pydsl)
ifneq ($(pydsl),no)
  GENERATE_PYDSL_FLAGS += "--lang=$(pydsl)"
endif

IVERILOG_FLAGS   := -Wall -Winfloop -Wno-timescale -g2012
IVERILOG_COMPILE := @IVERILOG@ $(IVERILOG_FLAGS)

PYTEST := @PYTEST@

#-------------------------------------------------------------------------
# Verilog ref test
#-------------------------------------------------------------------------
# When this is enabled, we do not actually use an LLM to generate the
# Verilog samples. Instead we copy the Verilog reference and rename
# the module from TopModule to RefModel. This can help us test the
# Verilog reference implementations.

enable_ref_test := $(cfg_ref_test)

#-------------------------------------------------------------------------
# Dataset
#-------------------------------------------------------------------------

dataset_dir = $(cfg_dataset)
VPATH = ${dataset_dir}

num_samples = $(cfg_samples)

# Define prompt files

problem_prompts := $(patsubst %, %_prompt.txt, $(problems))

#-------------------------------------------------------------------------
# Progress indicator
#-------------------------------------------------------------------------
# Here is some neat code that enables a more compact output with a
# progress indication:
#
#  https://stackoverflow.com/questions/451413
#

REDIRECT_LOG= &>
REDIRECT_APPEND_LOG= &>>
VERBOSE:=0
QUIET=@
ifeq ($(VERBOSE),1)
  QUIET=
  REDIRECT_LOG= 2>&1 | tee
  REDIRECT_APPEND_LOG= 2>&1 | tee -a
endif

ifndef ECHO

# Do a dry run of make with with given targets and count number of
# times we see HIT_MARK which shows up for every instance of ECHO

HIT_TOTAL != $(MAKE) $(MAKECMDGOALS) --dry-run ECHO="HIT_MARK" | grep -c "HIT_MARK"

# Create a counter which will increment every instance of ECHO

HIT_COUNT = $(eval HIT_N != expr $(HIT_N) + 1)$(HIT_N)

# Create the output counter

ECHO = $(scripts_dir)/echo-progress \
         --nsteps=$(HIT_TOTAL) --stepno=$(HIT_COUNT) --verbose=$(VERBOSE)

endif

#-------------------------------------------------------------------------
# Template for per-problem rules
#-------------------------------------------------------------------------
# The template is instantiated for each of the problems.
#
# Arguments:
#  $(1) : real problem name (ie with underscores, no dashes)
#

define problem_template

# Figure out number of samples for this problem

sample_num_strs != seq --format "%02g" 1 $$(num_samples)

#-------------------------------------------------------------------------
# Generate verilog samples
#-------------------------------------------------------------------------

$(1)_sv_samples       := $$(patsubst %, $(1)/$(1)_sample%.v, $$(sample_num_strs))
$(1)_sv_generate_logs := $$(patsubst %, $(1)/$(1)_sample%-sv-generate.log, $$(sample_num_strs))

ifeq ($(enable_ref_test),yes)
$$($(1)_sv_samples) : %.v : $(1)_ref.v
	@$$(ECHO) Generating $$(notdir $$@) verilog
	$$(QUIET) mkdir -p $(1)
	$$(QUIET) sed -e's/RefModule/TopModule/' $$< > $$@
	$$(QUIET) echo "Copied reference to create sample" \
              $(REDIRECT_LOG) $$*-sv-generate.log
else
$$($(1)_sv_samples) : %.v : $(1)_prompt.txt
	@$$(ECHO) Generating $$(notdir $$@) for verilog
	$$(QUIET) mkdir -p $(1)
	$$(QUIET) $$(GENERATE) $$(GENERATE_VERILOG_FLAGS) --verbose \
              --output $$@ $$< \
              $(REDIRECT_LOG) $$*-sv-generate.log
endif

$(1)-sv-generate : $$($(1)_sv_samples)

ifneq ($(cfg_verilog),no)
pregen_files += $$($(1)_sv_samples)
pregen_files += $$($(1)_sv_generate_logs)
endif

sv_generate_targets += $$($(1)_sv_samples)

$(1)-sv-generate-clean :
	rm -rf $$($(1)_sv_samples)
	rm -rf $$($(1)_sv_generate_logs)

sv_generate_clean_targets += $(1)-sv-generate-clean

#-------------------------------------------------------------------------
# Test verilog samples
#-------------------------------------------------------------------------

$(1)_sv_test_bins = \
  $$(patsubst %.v, %, $$($(1)_sv_samples))

$(1)_sv_test_logs = \
  $$(patsubst %.v, %-sv-test.log, $$($(1)_sv_samples))

$$($(1)_sv_test_logs) : %-sv-test.log : %.v $(1)_test.v $(1)_ref.v
	@$$(ECHO)  Testing $$(notdir $$*) for verilog
	-$$(QUIET) $(IVERILOG_COMPILE) -s Top -I $(dataset_dir) -o $$* $$^ \
               $(REDIRECT_LOG) $$*-sv-test.log
	-$$(QUIET) timeout 45 \
               ./$$* +test-case=-1 $(REDIRECT_APPEND_LOG) $$@; \
             if [[ $$$${PIPESTATUS[0]} == 124 ]]; then \
               echo "TIMEOUT" $(REDIRECT_APPEND_LOG) $$@; \
             fi

$(1)-sv-test : $$($(1)_sv_test_logs)

sv_test_targets += $$($(1)_sv_test_logs)

$(1)-sv-test-clean :
	rm -rf $$($(1)_sv_test_bins)
	rm -rf $$($(1)_sv_test_logs)

sv_test_clean_targets += $(1)-sv-test-clean

#-------------------------------------------------------------------------
# Generate pydsl samples
#-------------------------------------------------------------------------

$(1)_py_samples       := $$(patsubst %, $(1)/$(1)_sample%.py, $$(sample_num_strs))
$(1)_py_generate_logs := $$(patsubst %, $(1)/$(1)_sample%-py-generate.log, $$(sample_num_strs))

$$($(1)_py_samples) : %.py : $(1)_prompt.txt
	@$$(ECHO) Generating $$(notdir $$@) for $$(pydsl)
	$$(QUIET) mkdir -p $(1)
	$$(QUIET) $$(GENERATE) $$(GENERATE_PYDSL_FLAGS) --verbose \
              --output $$@ $$< \
              $(REDIRECT_LOG) $$*-py-generate.log

$(1)-py-generate : $$($(1)_py_samples)

ifneq ($(cfg_pydsl),no)
pregen_files += $$($(1)_py_samples)
pregen_files += $$($(1)_py_generate_logs)
endif

py_generate_targets += $$($(1)_py_samples)

$(1)-py-generate-clean :
	rm -rf $$($(1)_py_samples)
	rm -rf $$($(1)_py_generate_logs)

py_generate_clean_targets += $(1)-py-generate-clean

#-------------------------------------------------------------------------
# Test pydsl samples with pytest
#-------------------------------------------------------------------------

$(1)_py_test_logs = \
  $$(patsubst %.py, %-py-test.log, $$($(1)_py_samples))

$$($(1)_py_test_logs) : %-py-test.log : $(1)_test.py %.py
	@$$(ECHO)  Testing $$(notdir $$*) for $$(pydsl)
	-$$(QUIET) cd $(1); \
             timeout 45 \
               $(PYTEST) ../$$< -sx --verbose --tb=short \
                 --$$(pydsl)=$$(notdir $$*).py \
                 $(REDIRECT_LOG) $$(notdir $$*)-py-test.log; \
             if [[ $$$${PIPESTATUS[0]} == 124 ]]; then  \
               echo "TIMEOUT" $(REDIRECT_APPEND_LOG) $$(notdir $$*)-py-test.log; \
             fi

$(1)-py-test : $$($(1)_py_test_logs)

py_test_targets += $$($(1)_py_test_logs)

$(1)-py-test-clean :
	rm -rf $$($(1)_py_test_logs)

py_test_clean_targets += $(1)-py-test-clean

#-------------------------------------------------------------------------
# Clean up
#-------------------------------------------------------------------------

$(1)-clean :
	rm -rf $(1)

# Add top-level to junk

junk += $(1)

# Phony targets

.PHONY : $(1)-sv-generate
.PHONY : $(1)-sv-test
.PHONY : $(1)-sv-test-clean

.PHONY : $(1)-py-generate
.PHONY : $(1)-py-test
.PHONY : $(1)-py-test-clean

endef

$(foreach problem, $(problems), \
   $(eval $(call problem_template,$(problem))))

#-------------------------------------------------------------------------
# Top level targets
#-------------------------------------------------------------------------

sv-generate       : $(sv_generate_targets)
sv-generate-clean : $(sv_generate_clean_targets)
sv-test           : $(sv_test_targets)
sv-test-clean     : $(sv_test_clean_targets)

py-generate       : $(py_generate_targets)
py-generate-clean : $(py_generate_clean_targets)
py-test           : $(py_test_targets)
py-test-clean     : $(py_test_clean_targets)

sv-analyze : $(sv_test_targets)
	@$(ECHO) Analyzing verilog results
	$(QUIET) $(scripts_dir)/analyze-verilog \
             --csv=sv-summary.csv $(problems) | tee sv-summary.txt

py-analyze : $(py_test_targets)
	@$(ECHO) Analyzing pydsl results
	$(QUIET) $(scripts_dir)/analyze-$(pydsl) \
             --csv=py-summary.csv $(problems) | tee py-summary.txt

junk += sv-summary.txt sv-summary.csv
junk += py-summary.txt py-summary.csv

#-------------------------------------------------------------------------
# pregen
#-------------------------------------------------------------------------
# Save the generated code and logs so that we can rerun experiments
# without having to regenerate all of the generated code with an LLM.
# We use the special $(file make function because the number of files
# can be so large that it might exceed the command line limit.

pregen_dir := $(cfg_pregen_dir)

extra_pregen_files = config.mk

ifneq ($(cfg_verilog),no)
extra_pregen_files += sv-summary.txt sv-summary.csv
endif

ifneq ($(cfg_pydsl),no)
extra_pregen_files += py-summary.txt py-summary.csv
endif

ifneq ($(pregen_dir),NOT_ENABLED)
pregen:
	$(file >  files-to-copy.txt, $(pregen_files))
	$(file >> files-to-copy.txt, $(extra_pregen_files))
	sed -i.bak -e 's/ \+/\n/g' files-to-copy.txt
	rsync --files-from=files-to-copy.txt . \
    $(pregen_dir)/$(shell date '+%Y-%m-%d-%H-%M')
	rm files-to-copy.txt files-to-copy.txt.bak
else
pregen:
	@echo "ERROR: pregen directory was not set with configure"
endif

#-------------------------------------------------------------------------
# configure information
#-------------------------------------------------------------------------

dist_junk += \
  config.status Makefile config.log \

#-------------------------------------------------------------------------
# Default
#-------------------------------------------------------------------------

# Only generating verilog

ifneq ($(cfg_verilog),no)
ifeq  ($(cfg_pydsl),no)

all : sv-analyze

endif
endif

# Only generating pydsl

ifeq  ($(cfg_verilog),no)
ifneq ($(cfg_pydsl),no)

all : py-analyze

endif
endif

# Generating both verilog and pydsl

ifneq ($(cfg_verilog),no)
ifneq ($(cfg_pydsl),no)

all : sv-analyze py-analyze
	$(scripts_dir)/collect-summaries
endif
endif

.PHONY : all

#-------------------------------------------------------------------------
# Makefile debugging
#-------------------------------------------------------------------------
# This handy rule will display the contents of any make variable by
# using the target debug-<varname>. So for example, make debug-junk will
# display the contents of the junk variable.

debug-% :
	@echo $* = $($*)

#-------------------------------------------------------------------------
# Clean up junk
#-------------------------------------------------------------------------

clean :
	rm -rf *~ \#* wave.vcd $(junk)

distclean :
	rm -rf *~ \#* wave.vcd $(junk) $(dist_junk)

.PHONY : clean