Merge branch 'llvm-ir'

.clang-tidy

@@ -5,7 +5,14 @@ modernize-*,\
 bugprone-*,\
 performance-*,\
 readability-*,\
+misc-const-correctness,\
+misc-use-internal-linkage,\
+misc-static-assert,\
+misc-redundant-expression,\
+misc-misleading-identifier,\
+misc-definitions-in-headers,\
 -clang-analyzer-cplusplus.NewDeleteLeaks,\
+-clang-analyzer-cplusplus.NewDelete,\
 -modernize-use-trailing-return-type,\
 -modernize-use-nodiscard,\
 -modernize-pass-by-value,\
@@ -22,8 +29,11 @@ readability-*,\
 -cppcoreguidelines-pro-type-static-cast-downcast,\
 -cppcoreguidelines-pro-type-const-cast,\
 -cppcoreguidelines-owning-memory,\
+-cppcoreguidelines-avoid-const-or-ref-data-members,\
+-cppcoreguidelines-rvalue-reference-param-not-moved,\
 -bugprone-easily-swappable-parameters,\
 -bugprone-reserved-identifier,\
+-bugprone-multi-level-implicit-pointer-conversion,\
 -readability-redundant-member-init,\
 -readability-implicit-bool-conversion,\
 -readability-magic-numbers,\
@@ -36,6 +46,8 @@ readability-*,\
 -readability-convert-member-functions-to-static,\
 -readability-isolate-declaration,\
 -readability-qualified-auto,\
+-readability-redundant-declaration,\
+-readability-use-anyofallof,\
 -performance-noexcept-move-constructor,\
 -performance-move-const-arg"
 

.github/actions/build/action.yml

@@ -17,7 +17,7 @@ runs:
     - if: runner.os == 'Linux'
       name: Install apt packages
       shell: bash
-      run: sudo apt-get install -y curl git zip build-essential entr libssl-dev libdouble-conversion-dev pkg-config ninja-build python3-pip cmake debhelper devscripts gnupg zlib1g-dev entr libffi-dev clang libjemalloc-dev libreadline-dev libzstd-dev
+      run: sudo apt-get install -y curl git git-lfs zip build-essential entr libssl-dev libdouble-conversion-dev pkg-config ninja-build python3-pip cmake debhelper devscripts gnupg zlib1g-dev entr libffi-dev clang libreadline-dev libzip-dev libbz2-dev doctest-dev libboost-all-dev gcc-14 g++-14
     - if: runner.os == 'macOS'
       name: Install brew packages
       shell: bash

.gitignore

@@ -1,5 +1,6 @@
 test.jank
 jank-generated.hpp
+.jank-repl-history
 a.out
 .jank-native-repl-history
 .jank-repl-history

compiler+runtime/CMakeLists.txt

@@ -64,6 +64,7 @@ set(
   -Wno-gnu-conditional-omitted-operand
   -Wno-implicit-fallthrough
   -Wno-covered-switch-default
+  -Wno-invalid-offsetof
   -fno-common
   -frtti
   -fexceptions
@@ -99,8 +100,7 @@ set(
   $<$<PLATFORM_ID:Darwin>:${CMAKE_BINARY_DIR}/llvm-install/usr/local/lib/libc++abi.a>
   )
 if(CMAKE_BUILD_TYPE STREQUAL "Debug")
-  list(APPEND jank_common_compiler_flags #-Werror
-    -DJANK_DEBUG)
+  list(APPEND jank_common_compiler_flags -Werror -DJANK_DEBUG)
 elseif(CMAKE_BUILD_TYPE STREQUAL "Release" OR CMAKE_BUILD_TYPE STREQUAL "RelWithDebInfo")
   list(APPEND jank_common_compiler_flags -DJANK_RELEASE)
 endif()
@@ -128,14 +128,11 @@ if(
   )
 endif()
 
-# TODO: Optimize
-llvm_map_components_to_libnames(jank_llvm_libs ${LLVM_TARGETS_TO_BUILD} support lineeditor orcjit native core)
-
 function(jank_hook_llvm target)
   set(tgt "${target}")
   target_include_directories("${tgt}" SYSTEM PRIVATE ${LLVM_INCLUDE_DIRS})
   target_compile_definitions("${tgt}" PRIVATE ${LLVM_DEFINITIONS})
-  target_link_libraries("${tgt}" PRIVATE ${jank_llvm_libs})
+  target_link_directories("${tgt}" PRIVATE ${CMAKE_BINARY_DIR}/llvm-install/usr/local/lib)
 endfunction()
 # ---- Clang/LLVM ----
 
@@ -147,6 +144,7 @@ endfunction()
 
 add_library(
   jank_lib STATIC
+  src/cpp/jank/c_api.cpp
   src/cpp/jank/hash.cpp
   src/cpp/jank/util/cli.cpp
   src/cpp/jank/util/sha256.cpp
@@ -175,6 +173,7 @@ add_library(
   src/cpp/jank/runtime/obj/number.cpp
   src/cpp/jank/runtime/obj/native_function_wrapper.cpp
   src/cpp/jank/runtime/obj/jit_function.cpp
+  src/cpp/jank/runtime/obj/jit_closure.cpp
   src/cpp/jank/runtime/obj/multi_function.cpp
   src/cpp/jank/runtime/obj/native_pointer_wrapper.cpp
   src/cpp/jank/runtime/obj/symbol.cpp
@@ -198,6 +197,7 @@ add_library(
   src/cpp/jank/runtime/obj/cons.cpp
   src/cpp/jank/runtime/obj/range.cpp
   src/cpp/jank/runtime/obj/repeat.cpp
+  src/cpp/jank/runtime/obj/ratio.cpp
   src/cpp/jank/runtime/obj/iterator.cpp
   src/cpp/jank/runtime/obj/lazy_sequence.cpp
   src/cpp/jank/runtime/obj/chunk_buffer.cpp
@@ -211,13 +211,17 @@ add_library(
   src/cpp/jank/runtime/obj/reduced.cpp
   src/cpp/jank/runtime/behavior/callable.cpp
   src/cpp/jank/runtime/behavior/metadatable.cpp
-  src/cpp/jank/runtime/behavior/derefable.cpp
   src/cpp/jank/analyze/processor.cpp
   src/cpp/jank/analyze/local_frame.cpp
   src/cpp/jank/analyze/step/force_boxed.cpp
   src/cpp/jank/evaluate.cpp
-  src/cpp/jank/codegen/processor.cpp
+  src/cpp/jank/codegen/llvm_processor.cpp
   src/cpp/jank/jit/processor.cpp
+
+  # Native module sources.
+  src/cpp/clojure/core_native.cpp
+  src/cpp/jank/compiler_native.cpp
+  src/cpp/jank/perf_native.cpp
 )
 
 target_include_directories(
@@ -239,10 +243,6 @@ target_include_directories(
   "$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/third-party/fmt/include>"
   "$<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/third-party/libzippp/src>"
 )
-target_precompile_headers(
-  jank_lib
-  PUBLIC ${PROJECT_SOURCE_DIR}/include/cpp/jank/prelude.hpp
-)
 
 set_property(TARGET jank_lib PROPERTY OUTPUT_NAME jank)
 
@@ -263,7 +263,9 @@ target_link_libraries(
   bdwgc::gc bdwgc::gccpp
   libzippp::libzippp
   readline
-  clangInterpreter
+  libclang.so
+  libclang-cpp.so
+  libLLVM.so
   Boost::filesystem
   OpenSSL::Crypto
 )
@@ -274,22 +276,6 @@ jank_hook_llvm(jank_lib)
 set(jank_jit_compile_flags_list ${jank_common_compiler_flags} ${jank_jit_compiler_flags})
 list(JOIN jank_jit_compile_flags_list " " jank_jit_compile_flags_str)
 
-# Build a string of all include flags for jank. This will be used for building the PCH at build
-# time as well as building the PCH on first run, after installation.
-get_target_property(jank_lib_includes_prop jank_lib INCLUDE_DIRECTORIES)
-set(jank_lib_includes "")
-foreach(dir ${jank_lib_includes_prop})
-  # System includes are just the path, but normal includes have this format:
-  # $<BUILD_INTERFACE:/home/jeaye/projects/jank/compiler+runtime/include/cpp>
-  # We need to strip all of that out.
-  if(${dir} MATCHES "BUILD_INTERFACE:")
-    string(REGEX REPLACE "\\$<BUILD_INTERFACE:(.*)>" "\\1" dir_path ${dir})
-    list(APPEND jank_lib_includes -I${dir_path})
-  else()
-    list(APPEND jank_lib_includes -isystem ${dir})
-  endif()
-endforeach()
-
 target_include_directories(
     jank_lib
     SYSTEM
@@ -443,7 +429,8 @@ if(jank_tests)
     test/cpp/jank/read/lex.cpp
     test/cpp/jank/read/parse.cpp
     test/cpp/jank/analyze/box.cpp
-    test/cpp/jank/runtime/detail/list_type.cpp
+    test/cpp/jank/runtime/detail/native_persistent_list.cpp
+    test/cpp/jank/runtime/obj/ratio.cpp
     test/cpp/jank/jit/processor.cpp
   )
   add_executable(jank::test_exe ALIAS jank_test_exe)
@@ -475,47 +462,6 @@ if(jank_tests)
 endif()
 # ---- Tests ----
 
-# ---- Incremental PCH ----
-# jank has two types of pre-compiled headers, since Clang's JIT compiler is picky. The first
-# PCH is handled normally via CMake using `target_precompile_headers` and it's used for AOT builts.
-# However, once we boot up jank, the first thing we do is load a PCH so that the JIT environment
-# can know all of the types and functions within the jank runtime. This second PCH is our
-# "incremental" PCH, named that way since it uses Clang's -fincremental-extensions flag.
-#
-# Here, we set up the incremental PCH to be built whenever the normal PCH is changed or the script
-# for building the incremental PCH changes. Any other changes don't matter.
-
-# Build a string of all of our AOT compilation flags so we can share them with our incremental PCH.
-set(jank_pch_compile_flags_list ${jank_common_compiler_flags} -w)
-list(JOIN jank_pch_compile_flags_list " " jank_pch_compile_flags_str)
-
-set(jank_incremental_pch_flag ${CMAKE_BINARY_DIR}/incremental.pch)
-
-separate_arguments(jank_incremental_pch_cxx_flags UNIX_COMMAND ${CMAKE_CXX_FLAGS})
-list(REMOVE_ITEM jank_incremental_pch_cxx_flags -fPIC)
-add_custom_command(
-  DEPENDS ${CMAKE_BINARY_DIR}/jank
-  OUTPUT ${jank_incremental_pch_flag}
-  COMMAND_EXPAND_LISTS
-  VERBATIM
-  COMMAND ${CMAKE_CXX_COMPILER}
-          ${jank_incremental_pch_cxx_flags}
-          ${jank_common_compiler_flags} ${jank_aot_compiler_flags}
-          ${jank_lib_includes}
-          -Xclang -fincremental-extensions
-          -Xclang -emit-pch
-          -x c++-header
-          -w
-          -c ${PROJECT_SOURCE_DIR}/include/cpp/jank/prelude.hpp
-          -o ${jank_incremental_pch_flag}
-)
-add_custom_target(
-  jank_incremental_pch
-  ALL
-  DEPENDS ${jank_incremental_pch_flag}
-)
-# ---- Incremental PCH ----
-
 # ---- Compiled Clojure libraries ----
 # We do a bit of a dance here, to have a custom command generate a file
 # which is a then a dependency of a custom target. This is because custom
@@ -527,11 +473,10 @@ add_custom_target(
 # change, or whenever the jank binary changes.
 set(jank_core_libraries_flag ${CMAKE_BINARY_DIR}/classes/core-libraries)
 add_custom_command(
-  # TODO: Depend on jank's actual PCH.
-  DEPENDS ${CMAKE_BINARY_DIR}/jank ${CMAKE_SOURCE_DIR}/src/jank/clojure/core.jank ${jank_incremental_pch_flag}
+  DEPENDS ${CMAKE_BINARY_DIR}/jank ${CMAKE_SOURCE_DIR}/src/jank/clojure/core.jank
   OUTPUT ${jank_core_libraries_flag}
   COMMAND ${CMAKE_BINARY_DIR}/jank compile clojure.core
-  COMMAND mkdir -p ${CMAKE_BINARY_DIR}/classes; touch ${jank_core_libraries_flag}
+  COMMAND touch ${jank_core_libraries_flag}
 )
 add_custom_target(
   jank_core_libraries

compiler+runtime/bin/build-clang

@@ -54,13 +54,16 @@ function build()
   cd "${srcdir}/llvm-build"
 
   cmake -DCMAKE_BUILD_TYPE=Release \
-        -DLLVM_ENABLE_RUNTIMES=all \
         -DCMAKE_CXX_STANDARD=20 \
+        -DLLVM_ENABLE_RUNTIMES=all \
+        -DLLVM_BUILD_LLVM_DYLIB=ON \
+        -DLLVM_LINK_LLVM_DYLIB=ON \
         -DLLVM_ENABLE_PROJECTS="clang;clang-tools-extra" \
         -DLLVM_TARGETS_TO_BUILD="host" \
-        -DLLVM_ENABLE_EH=OFF \
-        -DLLVM_ENABLE_RTTI=OFF \
+        -DLLVM_ENABLE_EH=ON \
+        -DLLVM_ENABLE_RTTI=ON \
         -DLLVM_INCLUDE_BENCHMARKS=OFF \
+        -DLLVM_ENABLE_BINDINGS=OFF \
         -DLLVM_INCLUDE_EXAMPLES=OFF \
         -DLLVM_INCLUDE_TESTS=OFF \
         -DLLVM_ENABLE_ZSTD=OFF \

compiler+runtime/bin/ci/test

@@ -19,16 +19,17 @@ export CXX="${llvm_dir}/usr/local/bin/clang++"
                 "$@"
 ./bin/compile
 
-if [[ "${CODECOV:-off}" == "on" ]];
-then
-  ./bin/configure -GNinja \
-                  -Djank_tests=on \
-                  -Djank_coverage=on \
-                  "$@"
-  LLVM_PROFILE_FILE=build/test.profraw ./bin/test
-  "${llvm_dir}/usr/local/bin/llvm-profdata" merge --sparse build/test.profraw -o build/test.profdata
-  "${llvm_dir}/usr/local/bin/llvm-cov" show ./build/jank-test --instr-profile build/test.profdata > coverage.txt
-  # This file will be read by a subsequent CodeCov Github action.
-else
-  ./bin/test
-fi
+# TODO: Enable these before merging to main.
+#if [[ "${CODECOV:-off}" == "on" ]];
+#then
+#  ./bin/configure -GNinja \
+#                  -Djank_tests=on \
+#                  -Djank_coverage=on \
+#                  "$@"
+#  LLVM_PROFILE_FILE=build/test.profraw ./bin/test
+#  "${llvm_dir}/usr/local/bin/llvm-profdata" merge --sparse build/test.profraw -o build/test.profdata
+#  "${llvm_dir}/usr/local/bin/llvm-cov" show ./build/jank-test --instr-profile build/test.profdata > coverage.txt
+#  # This file will be read by a subsequent CodeCov Github action.
+#else
+#  ./bin/test
+#fi

compiler+runtime/bin/format

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+git_root=$(git rev-parse --show-toplevel)
+
+for i in $(git status | grep -E "modified:" | sed 's/modified:\s*//'); do
+  "$git_root"/compiler+runtime/build/llvm-install/usr/local/bin/clang-format -i "$i"
+  echo "formatted" "$i"
+done

compiler+runtime/bin/watch

@@ -2,4 +2,7 @@
 
 set -euo pipefail
 
-git ls-files -cdmo --exclude-standard | entr bash -c "time $* || true";
+# We sleep for a bit before running to account for saving multiple files at a time.
+# Without the sleep, we may start compiling before all of them have saved, which will
+# then result in potentially false results for the compilation and hard to debug issues.
+git ls-files -cdmo --exclude-standard | entr bash -c "sleep 1; time $* || true";

compiler+runtime/cmake/dependency/bdwgc.cmake

@@ -1,9 +1,11 @@
 set(CMAKE_C_FLAGS_OLD "${CMAKE_C_FLAGS}")
 set(CMAKE_CXX_FLAGS_OLD "${CMAKE_CXX_FLAGS}")
 set(BUILD_SHARED_LIBS_OLD ${BUILD_SHARED_LIBS})
+set(CMAKE_CXX_CLANG_TIDY_OLD ${CMAKE_CXX_CLANG_TIDY})
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w")
   set(BUILD_SHARED_LIBS OFF)
+  set(CMAKE_CXX_CLANG_TIDY "")
   set(enable_cplusplus ON CACHE BOOL "Enable C++")
   set(build_cord OFF CACHE BOOL "Build cord")
   set(enable_docs OFF CACHE BOOL "Enable docs")
@@ -17,3 +19,4 @@ set(BUILD_SHARED_LIBS_OLD ${BUILD_SHARED_LIBS})
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS_OLD}")
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS_OLD}")
 set(BUILD_SHARED_LIBS ${BUILD_SHARED_LIBS_OLD})
+set(CMAKE_CXX_CLANG_TIDY ${CMAKE_CXX_CLANG_TIDY_OLD})

compiler+runtime/cmake/dependency/fmt.cmake

@@ -1,7 +1,10 @@
-set(FMT_DOC OFF CACHE BOOL "Generate docs")
-set(FMT_INSTALL OFF CACHE BOOL "Generate the install target.")
+set(CMAKE_CXX_CLANG_TIDY_OLD ${CMAKE_CXX_CLANG_TIDY})
+  set(CMAKE_CXX_CLANG_TIDY "")
+  set(FMT_DOC OFF CACHE BOOL "Generate docs")
+  set(FMT_INSTALL OFF CACHE BOOL "Generate the install target.")
 
-add_subdirectory(third-party/fmt)
+  add_subdirectory(third-party/fmt)
 
-unset(FMT_DOC)
-unset(FMT_INSTALL)
+  unset(FMT_DOC)
+  unset(FMT_INSTALL)
+set(CMAKE_CXX_CLANG_TIDY ${CMAKE_CXX_CLANG_TIDY_OLD})

compiler+runtime/cmake/dependency/libzippp.cmake

@@ -1,8 +1,11 @@
 set(CMAKE_C_FLAGS_OLD "${CMAKE_C_FLAGS}")
 set(CMAKE_CXX_FLAGS_OLD "${CMAKE_CXX_FLAGS}")
+set(CMAKE_CXX_CLANG_TIDY_OLD ${CMAKE_CXX_CLANG_TIDY})
   set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -w")
   set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -w")
+  set(CMAKE_CXX_CLANG_TIDY "")
 
   add_subdirectory(third-party/libzippp)
 set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS_OLD}")
 set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS_OLD}")
+set(CMAKE_CXX_CLANG_TIDY ${CMAKE_CXX_CLANG_TIDY_OLD})

compiler+runtime/include/cpp/clojure/core_native.hpp

@@ -0,0 +1,5 @@
+#pragma once
+
+#include <jank/c_api.h>
+
+jank_object_ptr jank_load_clojure_core_native();

compiler+runtime/include/cpp/jank/analyze/expr/call.hpp

@@ -12,8 +12,11 @@ namespace jank::analyze::expr
     native_box<E> source_expr{};
     runtime::obj::persistent_list_ptr args{};
     native_vector<native_box<E>> arg_exprs;
-    /* Do we recur through calling our own fn name? */
-    native_bool is_named_recur{};
+
+    void propagate_position(expression_position const pos)
+    {
+      position = pos;
+    }
 
     runtime::object_ptr to_runtime_data() const
     {

compiler+runtime/include/cpp/jank/analyze/expr/def.hpp

@@ -16,6 +16,11 @@ namespace jank::analyze::expr
     obj::symbol_ptr name{};
     option<native_box<E>> value;
 
+    void propagate_position(expression_position const pos)
+    {
+      position = pos;
+    }
+
     object_ptr to_runtime_data() const
     {
       return merge(static_cast<expression_base const *>(this)->to_runtime_data(),