From 47b423602bc41276229e463f17014e1cc75ada2e Mon Sep 17 00:00:00 2001 From: Eric Kilmer Date: Wed, 27 Sep 2023 15:04:33 -0400 Subject: [PATCH] LLVM-17 - macOS 13 - XCode 15 - Update vcpkg (#1032) * LLVM-17 * Skip building tools with LLVM 17 * Update to macOS 13 and XCode 15.0 * Remove LLVM pasta * Use newest commit on cxx-common branch vcpkg fork --- .github/workflows/vcpkg_ci_amd64.yml | 20 +- .github/workflows/vcpkg_ci_mac.yml | 31 +- ports/llvm-15/0002-Fix-DR-1734.patch | 17 - ports/llvm-15/0003-Fix-tools-path.patch | 16 - .../0004-Fix-compiler-rt-install-path.patch | 44 - .../llvm-15/0005-Fix-tools-install-path.patch | 207 -- ports/llvm-15/0007-Fix-install-bolt.patch | 21 - .../0023-clang-sys-include-dir-path.patch | 31 - ports/llvm-15/0025-PASTA-patches.patch | 2131 ------------- .../0027-unknown-attrs-as-annotations.patch | 180 -- ports/llvm-16/0025-PASTA-patches.patch | 2663 ----------------- ...es-to-clang-s-tablegen-of-attributes.patch | 34 - ...sAsAnnotate-and-AttributedType-Attrs.patch | 389 --- ports/llvm-16/portfile.cmake | 10 - ports/llvm-16/vcpkg.json | 3 - .../0001-Fix-install-paths.patch | 0 .../0006-Fix-libffi.patch | 0 .../0020-fix-FindZ3.cmake.patch | 0 .../0021-fix-find_dependency.patch | 24 +- .../0026-fix-prefix-path-calc.patch | 0 ...macro-expansion-on-invalid-sourceloc.patch | 28 + ports/{llvm-15 => llvm-17}/clang_usage | 0 ports/{llvm-15 => llvm-17}/flang_usage | 0 ports/{llvm-15 => llvm-17}/lld_usage | 0 ports/{llvm-15 => llvm-17}/llvm_usage | 0 ports/{llvm-15 => llvm-17}/mlir_usage | 0 ports/{llvm-15 => llvm-17}/portfile.cmake | 79 +- ports/{llvm-15 => llvm-17}/vcpkg.json | 80 +- vcpkg_info.txt | 2 +- 29 files changed, 157 insertions(+), 5853 deletions(-) delete mode 100644 ports/llvm-15/0002-Fix-DR-1734.patch delete mode 100644 ports/llvm-15/0003-Fix-tools-path.patch delete mode 100644 ports/llvm-15/0004-Fix-compiler-rt-install-path.patch delete mode 100644 ports/llvm-15/0005-Fix-tools-install-path.patch delete mode 100644 ports/llvm-15/0007-Fix-install-bolt.patch delete mode 100644 ports/llvm-15/0023-clang-sys-include-dir-path.patch delete mode 100644 ports/llvm-15/0025-PASTA-patches.patch delete mode 100644 ports/llvm-15/0027-unknown-attrs-as-annotations.patch delete mode 100644 ports/llvm-16/0025-PASTA-patches.patch delete mode 100644 ports/llvm-16/0028-Fixes-to-clang-s-tablegen-of-attributes.patch delete mode 100644 ports/llvm-16/0030-UnknownAttrsAsAnnotate-and-AttributedType-Attrs.patch rename ports/{llvm-15 => llvm-17}/0001-Fix-install-paths.patch (100%) rename ports/{llvm-15 => llvm-17}/0006-Fix-libffi.patch (100%) rename ports/{llvm-15 => llvm-17}/0020-fix-FindZ3.cmake.patch (100%) rename ports/{llvm-15 => llvm-17}/0021-fix-find_dependency.patch (67%) rename ports/{llvm-15 => llvm-17}/0026-fix-prefix-path-calc.patch (100%) create mode 100644 ports/llvm-17/0029-Do-not-attempt-macro-expansion-on-invalid-sourceloc.patch rename ports/{llvm-15 => llvm-17}/clang_usage (100%) rename ports/{llvm-15 => llvm-17}/flang_usage (100%) rename ports/{llvm-15 => llvm-17}/lld_usage (100%) rename ports/{llvm-15 => llvm-17}/llvm_usage (100%) rename ports/{llvm-15 => llvm-17}/mlir_usage (100%) rename ports/{llvm-15 => llvm-17}/portfile.cmake (87%) rename ports/{llvm-15 => llvm-17}/vcpkg.json (86%) diff --git a/.github/workflows/vcpkg_ci_amd64.yml b/.github/workflows/vcpkg_ci_amd64.yml index 6ecdea92..dadb5f9e 100644 --- a/.github/workflows/vcpkg_ci_amd64.yml +++ b/.github/workflows/vcpkg_ci_amd64.yml @@ -40,7 +40,10 @@ jobs: image: # 'name' is Docker image name whereas 'os' is more generic - { os: 'ubuntu', name: 'ubuntu-v2', tag: '22.04' } - llvm: [ 'llvm-16[pasta]', 'llvm-16' ] + llvm: [ + 'llvm-16', + 'llvm-17' + ] target_arch: [ 'x64', 'arm64' ] container: @@ -86,6 +89,7 @@ jobs: { read -r vcpkg_repo_url && read -r vcpkg_commit; } <./vcpkg_info.txt || exit 1 git clone "${vcpkg_repo_url}" git -C vcpkg checkout "${vcpkg_commit}" + export VCPKG_DISABLE_METRICS=1 ./vcpkg/bootstrap-vcpkg.sh echo "VCPKG_ROOT=$(pwd)/vcpkg" >> $GITHUB_ENV @@ -180,7 +184,7 @@ jobs: - name: 'Rellic build' shell: 'bash' working-directory: rellic - if: matrix.target_arch == 'x64' + if: matrix.target_arch == 'x64' && startswith(matrix.llvm, 'llvm-16') run: | # Does not compile with gcc export CC="$(which clang)" @@ -197,13 +201,13 @@ jobs: - name: 'Rellic test' shell: 'bash' working-directory: rellic - if: matrix.target_arch == 'x64' + if: matrix.target_arch == 'x64' && startswith(matrix.llvm, 'llvm-16') run: | cmake --build build --target test - name: 'Remill dependencies' shell: 'bash' - if: matrix.target_arch == 'x64' + if: matrix.target_arch == 'x64' && startswith(matrix.llvm, 'llvm-16') working-directory: remill run: | python3 -m pip install poetry @@ -213,7 +217,7 @@ jobs: - name: 'Remill build' shell: 'bash' - if: matrix.target_arch == 'x64' + if: matrix.target_arch == 'x64' && startswith(matrix.llvm, 'llvm-16') working-directory: remill run: | cmake -G Ninja \ @@ -229,7 +233,7 @@ jobs: - name: 'Remill test' shell: 'bash' working-directory: remill/build - if: matrix.target_arch == 'x64' + if: matrix.target_arch == 'x64' && startswith(matrix.llvm, 'llvm-16') run: | # Some tests fail on ubuntu 22.04 env CTEST_OUTPUT_ON_FAILURE=1 ctest . || true @@ -237,7 +241,7 @@ jobs: - name: 'Anvill build' shell: 'bash' working-directory: anvill - if: matrix.target_arch == 'x64' + if: matrix.target_arch == 'x64' && startswith(matrix.llvm, 'llvm-16') run: | # TODO: Does not support compilation with gcc export CC="$(which clang)" @@ -260,7 +264,7 @@ jobs: - name: 'Anvill test' shell: 'bash' working-directory: anvill - if: matrix.target_arch == 'x64' + if: matrix.target_arch == 'x64' && startswith(matrix.llvm, 'llvm-16') run: | cmake --build build --target test diff --git a/.github/workflows/vcpkg_ci_mac.yml b/.github/workflows/vcpkg_ci_mac.yml index c1329feb..6c70ce9f 100644 --- a/.github/workflows/vcpkg_ci_mac.yml +++ b/.github/workflows/vcpkg_ci_mac.yml @@ -36,8 +36,11 @@ jobs: fail-fast: false matrix: os: - - { runner: 'macos-12', xcode: '14.2' } - llvm: [ 'llvm-16[pasta]', 'llvm-16' ] + - { runner: 'macos-13', xcode: '15.0' } + llvm: [ + 'llvm-16', + 'llvm-17' + ] target_arch: [ 'x64', 'arm64' ] runs-on: ${{ matrix.os.runner }} @@ -78,21 +81,13 @@ jobs: echo "Selecting XCode Version ${{ matrix.os.xcode }}" sudo xcode-select -s /Applications/Xcode_${{ matrix.os.xcode }}.app/Contents/Developer - # Crosscompiling LLVM 16 requires a newer clang than available in XCode 14.2 - - name: Install latest LLVM - if: contains(matrix.llvm, 'llvm-16') && matrix.target_arch == 'arm64' - run: | - brew install llvm - echo "/usr/local/opt/llvm/bin" >> "$GITHUB_PATH" - echo "CC=/usr/local/opt/llvm/bin/clang" >> "$GITHUB_ENV" - echo "CXX=/usr/local/opt/llvm/bin/clang++" >> "$GITHUB_ENV" - - name: Initialize vcpkg shell: bash run: | { read -r vcpkg_repo_url && read -r vcpkg_commit; } <./vcpkg_info.txt || exit 1 git clone "${vcpkg_repo_url}" git -C vcpkg checkout "${vcpkg_commit}" + export VCPKG_DISABLE_METRICS=1 ./vcpkg/bootstrap-vcpkg.sh echo "VCPKG_ROOT=$(pwd)/vcpkg" >> $GITHUB_ENV @@ -177,7 +172,7 @@ jobs: - name: 'Rellic build' shell: 'bash' working-directory: rellic - if: matrix.target_arch == 'x64' + if: matrix.target_arch == 'x64' && startswith(matrix.llvm, 'llvm-16') run: | cmake -G Ninja \ -DCMAKE_VERBOSE_MAKEFILE=ON \ @@ -192,14 +187,14 @@ jobs: - name: 'Rellic test' shell: 'bash' working-directory: rellic - if: matrix.target_arch == 'x64' + if: matrix.target_arch == 'x64' && startswith(matrix.llvm, 'llvm-16') run: | cmake --build build --target test - name: 'Remill dependencies' shell: 'bash' working-directory: remill - if: matrix.target_arch == 'x64' + if: matrix.target_arch == 'x64' && startswith(matrix.llvm, 'llvm-16') run: | python3 -m pip install poetry python3 -m pip install --user ./scripts/diff_tester_export_insns @@ -209,7 +204,7 @@ jobs: - name: 'Remill build' shell: 'bash' working-directory: remill - if: matrix.target_arch == 'x64' + if: matrix.target_arch == 'x64' && startswith(matrix.llvm, 'llvm-16') run: | cmake -G Ninja \ -DCMAKE_VERBOSE_MAKEFILE=ON \ @@ -225,7 +220,7 @@ jobs: - name: 'Remill test' shell: 'bash' working-directory: remill/build - if: matrix.target_arch == 'x64' + if: matrix.target_arch == 'x64' && startswith(matrix.llvm, 'llvm-16') run: | # Only run test on x64 env CTEST_OUTPUT_ON_FAILURE=1 ctest . @@ -233,7 +228,7 @@ jobs: - name: 'Anvill build' shell: 'bash' working-directory: anvill - if: matrix.target_arch == 'x64' + if: matrix.target_arch == 'x64' && startswith(matrix.llvm, 'llvm-16') run: | cmake -G Ninja \ -DCMAKE_VERBOSE_MAKEFILE=ON \ @@ -253,7 +248,7 @@ jobs: - name: 'Anvill test' shell: 'bash' working-directory: anvill - if: matrix.target_arch == 'x64' + if: matrix.target_arch == 'x64' && startswith(matrix.llvm, 'llvm-16') run: | cmake --build build --target test diff --git a/ports/llvm-15/0002-Fix-DR-1734.patch b/ports/llvm-15/0002-Fix-DR-1734.patch deleted file mode 100644 index 3a2f4a28..00000000 --- a/ports/llvm-15/0002-Fix-DR-1734.patch +++ /dev/null @@ -1,17 +0,0 @@ - llvm/include/llvm/Support/type_traits.h | 3 ++- - 1 file changed, 2 insertions(+), 1 deletion(-) - -diff --git a/llvm/include/llvm/Support/type_traits.h b/llvm/include/llvm/Support/type_traits.h -index 7b7d5d991f3f..469b681deea3 100644 ---- a/llvm/include/llvm/Support/type_traits.h -+++ b/llvm/include/llvm/Support/type_traits.h -@@ -176,7 +176,8 @@ class is_trivially_copyable { - (has_deleted_copy_assign || has_trivial_copy_assign) && - (has_deleted_copy_constructor || has_trivial_copy_constructor); - --#ifdef HAVE_STD_IS_TRIVIALLY_COPYABLE -+// due to DR 1734, a type can be std::is_trivially_copyable but not llvm::is_trivially_copyable -+#if 0 - static_assert(value == std::is_trivially_copyable::value, - "inconsistent behavior between llvm:: and std:: implementation of is_trivially_copyable"); - #endif diff --git a/ports/llvm-15/0003-Fix-tools-path.patch b/ports/llvm-15/0003-Fix-tools-path.patch deleted file mode 100644 index 66fec67d..00000000 --- a/ports/llvm-15/0003-Fix-tools-path.patch +++ /dev/null @@ -1,16 +0,0 @@ - llvm/tools/llvm-config/llvm-config.cpp | 2 +- - 1 file changed, 1 insertion(+), 1 deletion(-) - -diff --git a/llvm/tools/llvm-config/llvm-config.cpp b/llvm/tools/llvm-config/llvm-config.cpp -index 2c6c55f89d38..f2b581559991 100644 ---- a/llvm/tools/llvm-config/llvm-config.cpp -+++ b/llvm/tools/llvm-config/llvm-config.cpp -@@ -307,7 +307,7 @@ int main(int argc, char **argv) { - // bin dir). - sys::fs::make_absolute(CurrentPath); - CurrentExecPrefix = -- sys::path::parent_path(sys::path::parent_path(CurrentPath)).str(); -+ sys::path::parent_path(sys::path::parent_path(sys::path::parent_path(CurrentPath))).str(); - - // Check to see if we are inside a development tree by comparing to possible - // locations (prefix style or CMake style). diff --git a/ports/llvm-15/0004-Fix-compiler-rt-install-path.patch b/ports/llvm-15/0004-Fix-compiler-rt-install-path.patch deleted file mode 100644 index b08c1215..00000000 --- a/ports/llvm-15/0004-Fix-compiler-rt-install-path.patch +++ /dev/null @@ -1,44 +0,0 @@ - clang/lib/Headers/CMakeLists.txt | 2 +- - clang/runtime/CMakeLists.txt | 2 +- - compiler-rt/cmake/base-config-ix.cmake | 2 +- - 3 files changed, 3 insertions(+), 3 deletions(-) - -diff --git a/clang/lib/Headers/CMakeLists.txt b/clang/lib/Headers/CMakeLists.txt -index 6e2060991b92..94ced7feda9b 100644 ---- a/clang/lib/Headers/CMakeLists.txt -+++ b/clang/lib/Headers/CMakeLists.txt -@@ -420,7 +420,7 @@ add_header_target("openmp-resource-headers" ${openmp_wrapper_files}) - add_header_target("windows-resource-headers" ${windows_only_files}) - add_header_target("utility-resource-headers" ${utility_files}) - --set(header_install_dir lib${LLVM_LIBDIR_SUFFIX}/clang/${CLANG_VERSION}/include) -+set(header_install_dir tools/llvm/lib/clang/${CLANG_VERSION}/include) - - ############################################################# - # Install rules for the catch-all clang-resource-headers target -diff --git a/clang/runtime/CMakeLists.txt b/clang/runtime/CMakeLists.txt -index 9f4633bc85b1..6d7b70ee0dea 100644 ---- a/clang/runtime/CMakeLists.txt -+++ b/clang/runtime/CMakeLists.txt -@@ -84,7 +84,7 @@ if(LLVM_BUILD_EXTERNAL_COMPILER_RT AND EXISTS ${COMPILER_RT_SRC_ROOT}/) - -DLLVM_LIT_ARGS=${LLVM_LIT_ARGS} - -DCOMPILER_RT_OUTPUT_DIR=${LLVM_LIBRARY_OUTPUT_INTDIR}/clang/${CLANG_VERSION} - -DCOMPILER_RT_EXEC_OUTPUT_DIR=${LLVM_RUNTIME_OUTPUT_INTDIR} -- -DCOMPILER_RT_INSTALL_PATH:PATH=lib${LLVM_LIBDIR_SUFFIX}/clang/${CLANG_VERSION} -+ -DCOMPILER_RT_INSTALL_PATH:PATH=tools/llvm/lib/clang/${CLANG_VERSION} - -DCOMPILER_RT_INCLUDE_TESTS=${LLVM_INCLUDE_TESTS} - -DCMAKE_INSTALL_PREFIX=${CMAKE_INSTALL_PREFIX} - -DLLVM_LIBDIR_SUFFIX=${LLVM_LIBDIR_SUFFIX} -diff --git a/compiler-rt/cmake/base-config-ix.cmake b/compiler-rt/cmake/base-config-ix.cmake -index 8a6219568b3f..f9c9f6478280 100644 ---- a/compiler-rt/cmake/base-config-ix.cmake -+++ b/compiler-rt/cmake/base-config-ix.cmake -@@ -45,7 +45,7 @@ if (LLVM_TREE_AVAILABLE) - # Setup the paths where compiler-rt runtimes and headers should be stored. - set(COMPILER_RT_OUTPUT_DIR ${LLVM_LIBRARY_OUTPUT_INTDIR}/clang/${CLANG_VERSION}) - set(COMPILER_RT_EXEC_OUTPUT_DIR ${LLVM_RUNTIME_OUTPUT_INTDIR}) -- set(COMPILER_RT_INSTALL_PATH lib${LLVM_LIBDIR_SUFFIX}/clang/${CLANG_VERSION}) -+ set(COMPILER_RT_INSTALL_PATH tools/llvm/lib/clang/${CLANG_VERSION}) - option(COMPILER_RT_INCLUDE_TESTS "Generate and build compiler-rt unit tests." - ${LLVM_INCLUDE_TESTS}) - option(COMPILER_RT_ENABLE_WERROR "Fail and stop if warning is triggered" diff --git a/ports/llvm-15/0005-Fix-tools-install-path.patch b/ports/llvm-15/0005-Fix-tools-install-path.patch deleted file mode 100644 index 22bd804f..00000000 --- a/ports/llvm-15/0005-Fix-tools-install-path.patch +++ /dev/null @@ -1,207 +0,0 @@ - clang-tools-extra/clang-tidy/tool/CMakeLists.txt | 2 +- - clang-tools-extra/modularize/CMakeLists.txt | 2 +- - clang/cmake/modules/AddClang.cmake | 2 +- - clang/tools/c-index-test/CMakeLists.txt | 2 +- - clang/tools/clang-format/CMakeLists.txt | 2 +- - clang/tools/clang-linker-wrapper/CMakeLists.txt | 2 +- - clang/tools/clang-nvlink-wrapper/CMakeLists.txt | 2 +- - clang/tools/scan-build-py/CMakeLists.txt | 4 ++-- - clang/tools/scan-build/CMakeLists.txt | 2 +- - clang/tools/scan-view/CMakeLists.txt | 2 +- - flang/cmake/modules/AddFlang.cmake | 2 +- - flang/tools/f18/CMakeLists.txt | 2 +- - flang/tools/flang-driver/CMakeLists.txt | 2 +- - lld/cmake/modules/AddLLD.cmake | 2 +- - lldb/cmake/modules/AddLLDB.cmake | 2 +- - 15 files changed, 16 insertions(+), 16 deletions(-) - -diff --git a/clang-tools-extra/clang-tidy/tool/CMakeLists.txt b/clang-tools-extra/clang-tidy/tool/CMakeLists.txt -index 3ce552872015..e09b917ae5f8 100644 ---- a/clang-tools-extra/clang-tidy/tool/CMakeLists.txt -+++ b/clang-tools-extra/clang-tidy/tool/CMakeLists.txt -@@ -64,6 +64,6 @@ install(PROGRAMS clang-tidy-diff.py - DESTINATION "${CMAKE_INSTALL_DATADIR}/clang" - COMPONENT clang-tidy) - install(PROGRAMS run-clang-tidy.py -- DESTINATION "${CMAKE_INSTALL_BINDIR}" -+ DESTINATION "${LLVM_TOOLS_INSTALL_DIR}" - COMPONENT clang-tidy - RENAME run-clang-tidy) -diff --git a/clang-tools-extra/modularize/CMakeLists.txt b/clang-tools-extra/modularize/CMakeLists.txt -index fb17e353c39f..4b409e47446a 100644 ---- a/clang-tools-extra/modularize/CMakeLists.txt -+++ b/clang-tools-extra/modularize/CMakeLists.txt -@@ -23,5 +23,5 @@ clang_target_link_libraries(modularize - ) - - install(TARGETS modularize -- RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}" -+ RUNTIME DESTINATION "${LLVM_TOOLS_INSTALL_DIR}" - COMPONENT clang-extras) -diff --git a/clang/cmake/modules/AddClang.cmake b/clang/cmake/modules/AddClang.cmake -index 21ac332e4f5f..1aaf785bdc99 100644 ---- a/clang/cmake/modules/AddClang.cmake -+++ b/clang/cmake/modules/AddClang.cmake -@@ -166,7 +166,7 @@ macro(add_clang_tool name) - get_target_export_arg(${name} Clang export_to_clangtargets) - install(TARGETS ${name} - ${export_to_clangtargets} -- RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}" -+ RUNTIME DESTINATION "${LLVM_TOOLS_INSTALL_DIR}" - COMPONENT ${name}) - - if(NOT LLVM_ENABLE_IDE) -diff --git a/clang/tools/c-index-test/CMakeLists.txt b/clang/tools/c-index-test/CMakeLists.txt -index 0ae1b4e55244..e8a34e136194 100644 ---- a/clang/tools/c-index-test/CMakeLists.txt -+++ b/clang/tools/c-index-test/CMakeLists.txt -@@ -49,7 +49,7 @@ if (NOT LLVM_INSTALL_TOOLCHAIN_ONLY) - set_property(TARGET c-index-test APPEND PROPERTY INSTALL_RPATH - "@executable_path/../../lib") - else() -- set(INSTALL_DESTINATION "${CMAKE_INSTALL_BINDIR}") -+ set(INSTALL_DESTINATION "${LLVM_TOOLS_INSTALL_DIR}") - endif() - - install(TARGETS c-index-test -diff --git a/clang/tools/clang-format/CMakeLists.txt b/clang/tools/clang-format/CMakeLists.txt -index bbdef93b576b..8744e414da0a 100644 ---- a/clang/tools/clang-format/CMakeLists.txt -+++ b/clang/tools/clang-format/CMakeLists.txt -@@ -36,5 +36,5 @@ install(PROGRAMS clang-format.py - DESTINATION "${CMAKE_INSTALL_DATADIR}/clang" - COMPONENT clang-format) - install(PROGRAMS git-clang-format -- DESTINATION "${CMAKE_INSTALL_BINDIR}" -+ DESTINATION "${LLVM_TOOLS_INSTALL_DIR}" - COMPONENT clang-format) -diff --git a/clang/tools/clang-linker-wrapper/CMakeLists.txt b/clang/tools/clang-linker-wrapper/CMakeLists.txt -index 88c19cad7b53..b50c9ff90586 100644 ---- a/clang/tools/clang-linker-wrapper/CMakeLists.txt -+++ b/clang/tools/clang-linker-wrapper/CMakeLists.txt -@@ -44,4 +44,4 @@ target_link_libraries(clang-linker-wrapper - ${CLANG_LINKER_WRAPPER_LIB_DEPS} - ) - --install(TARGETS clang-linker-wrapper RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}") -+install(TARGETS clang-linker-wrapper RUNTIME DESTINATION "${LLVM_TOOLS_INSTALL_DIR}") -diff --git a/clang/tools/clang-nvlink-wrapper/CMakeLists.txt b/clang/tools/clang-nvlink-wrapper/CMakeLists.txt -index 2c979e509795..f22b801fe19a 100644 ---- a/clang/tools/clang-nvlink-wrapper/CMakeLists.txt -+++ b/clang/tools/clang-nvlink-wrapper/CMakeLists.txt -@@ -22,4 +22,4 @@ target_link_libraries(clang-nvlink-wrapper - ${CLANG_NVLINK_WRAPPER_LIB_DEPS} - ) - --install(TARGETS clang-nvlink-wrapper RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}") -+install(TARGETS clang-nvlink-wrapper RUNTIME DESTINATION "${LLVM_TOOLS_INSTALL_DIR}") -diff --git a/clang/tools/scan-build-py/CMakeLists.txt b/clang/tools/scan-build-py/CMakeLists.txt -index 061dc7ef4dd9..91499600693b 100644 ---- a/clang/tools/scan-build-py/CMakeLists.txt -+++ b/clang/tools/scan-build-py/CMakeLists.txt -@@ -43,7 +43,7 @@ foreach(BinFile ${BinFiles}) - ${CMAKE_BINARY_DIR}/bin/scan-build-py - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/bin/scan-build) - install (PROGRAMS "bin/scan-build" -- DESTINATION "${CMAKE_INSTALL_BINDIR}" -+ DESTINATION "${LLVM_TOOLS_INSTALL_DIR}" - RENAME scan-build-py - COMPONENT scan-build-py) - list(APPEND Depends ${CMAKE_BINARY_DIR}/bin/scan-build-py) -@@ -56,7 +56,7 @@ foreach(BinFile ${BinFiles}) - ${CMAKE_BINARY_DIR}/bin/ - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/bin/${BinFile}) - install(PROGRAMS bin/${BinFile} -- DESTINATION "${CMAKE_INSTALL_BINDIR}" -+ DESTINATION "${LLVM_TOOLS_INSTALL_DIR}" - COMPONENT scan-build-py) - list(APPEND Depends ${CMAKE_BINARY_DIR}/bin/${BinFile}) - endif() -diff --git a/clang/tools/scan-build/CMakeLists.txt b/clang/tools/scan-build/CMakeLists.txt -index ef687b0e90a1..a52af70443c3 100644 ---- a/clang/tools/scan-build/CMakeLists.txt -+++ b/clang/tools/scan-build/CMakeLists.txt -@@ -47,7 +47,7 @@ if(CLANG_INSTALL_SCANBUILD) - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/bin/${BinFile}) - list(APPEND Depends ${CMAKE_BINARY_DIR}/bin/${BinFile}) - install(PROGRAMS bin/${BinFile} -- DESTINATION "${CMAKE_INSTALL_BINDIR}" -+ DESTINATION "${LLVM_TOOLS_INSTALL_DIR}" - COMPONENT scan-build) - endforeach() - -diff --git a/clang/tools/scan-view/CMakeLists.txt b/clang/tools/scan-view/CMakeLists.txt -index 07aec76ee66f..ca6a3380ad0c 100644 ---- a/clang/tools/scan-view/CMakeLists.txt -+++ b/clang/tools/scan-view/CMakeLists.txt -@@ -20,7 +20,7 @@ if(CLANG_INSTALL_SCANVIEW) - DEPENDS ${CMAKE_CURRENT_SOURCE_DIR}/bin/${BinFile}) - list(APPEND Depends ${CMAKE_BINARY_DIR}/bin/${BinFile}) - install(PROGRAMS bin/${BinFile} -- DESTINATION "${CMAKE_INSTALL_BINDIR}" -+ DESTINATION "${LLVM_TOOLS_INSTALL_DIR}" - COMPONENT scan-view) - endforeach() - -diff --git a/flang/cmake/modules/AddFlang.cmake b/flang/cmake/modules/AddFlang.cmake -index d516ca31b51f..4a0d4ce80168 100644 ---- a/flang/cmake/modules/AddFlang.cmake -+++ b/flang/cmake/modules/AddFlang.cmake -@@ -110,7 +110,7 @@ macro(add_flang_tool name) - get_target_export_arg(${name} Flang export_to_flangtargets) - install(TARGETS ${name} - ${export_to_flangtargets} -- RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}" -+ RUNTIME DESTINATION "${LLVM_TOOLS_INSTALL_DIR}" - COMPONENT ${name}) - - if(NOT LLVM_ENABLE_IDE) -diff --git a/flang/tools/f18/CMakeLists.txt b/flang/tools/f18/CMakeLists.txt -index dd0898730e2e..d01c2f8076e3 100644 ---- a/flang/tools/f18/CMakeLists.txt -+++ b/flang/tools/f18/CMakeLists.txt -@@ -56,7 +56,7 @@ if (NOT WIN32) - @ONLY - ) - add_custom_target(flang-to-external-fc ALL DEPENDS ${CMAKE_BINARY_DIR}/bin/flang-to-external-fc) -- install(PROGRAMS ${CMAKE_BINARY_DIR}/bin/flang-to-external-fc DESTINATION "${CMAKE_INSTALL_BINDIR}") -+ install(PROGRAMS ${CMAKE_BINARY_DIR}/bin/flang-to-external-fc DESTINATION "${LLVM_TOOLS_INSTALL_DIR}") - endif() - - # TODO Move this to a more suitable location -diff --git a/flang/tools/flang-driver/CMakeLists.txt b/flang/tools/flang-driver/CMakeLists.txt -index 94c8ce6d58f1..466d41c58b1d 100644 ---- a/flang/tools/flang-driver/CMakeLists.txt -+++ b/flang/tools/flang-driver/CMakeLists.txt -@@ -42,4 +42,4 @@ if(FLANG_PLUGIN_SUPPORT) - export_executable_symbols_for_plugins(flang-new) - endif() - --install(TARGETS flang-new DESTINATION "${CMAKE_INSTALL_BINDIR}") -+install(TARGETS flang-new DESTINATION "${LLVM_TOOLS_INSTALL_DIR}") -diff --git a/lld/cmake/modules/AddLLD.cmake b/lld/cmake/modules/AddLLD.cmake -index d3924f7243d4..f328b62ad8c8 100644 ---- a/lld/cmake/modules/AddLLD.cmake -+++ b/lld/cmake/modules/AddLLD.cmake -@@ -47,7 +47,7 @@ macro(add_lld_tool name) - get_target_export_arg(${name} LLD export_to_lldtargets) - install(TARGETS ${name} - ${export_to_lldtargets} -- RUNTIME DESTINATION "${CMAKE_INSTALL_BINDIR}" -+ RUNTIME DESTINATION "${LLVM_TOOLS_INSTALL_DIR}" - COMPONENT ${name}) - - if(NOT CMAKE_CONFIGURATION_TYPES) -diff --git a/lldb/cmake/modules/AddLLDB.cmake b/lldb/cmake/modules/AddLLDB.cmake -index 3291a7c808e1..9fe9b9a7940d 100644 ---- a/lldb/cmake/modules/AddLLDB.cmake -+++ b/lldb/cmake/modules/AddLLDB.cmake -@@ -189,7 +189,7 @@ function(add_lldb_executable name) - endif() - - if(ARG_GENERATE_INSTALL) -- set(install_dest bin) -+ set(install_dest "${LLVM_TOOLS_INSTALL_DIR}") - if(ARG_INSTALL_PREFIX) - set(install_dest ${ARG_INSTALL_PREFIX}) - endif() diff --git a/ports/llvm-15/0007-Fix-install-bolt.patch b/ports/llvm-15/0007-Fix-install-bolt.patch deleted file mode 100644 index 361f58e7..00000000 --- a/ports/llvm-15/0007-Fix-install-bolt.patch +++ /dev/null @@ -1,21 +0,0 @@ - bolt/tools/driver/CMakeLists.txt | 7 ------- - 1 file changed, 7 deletions(-) - -diff --git a/bolt/tools/driver/CMakeLists.txt b/bolt/tools/driver/CMakeLists.txt -index e56be15dbcff..85b078e2e761 100644 ---- a/bolt/tools/driver/CMakeLists.txt -+++ b/bolt/tools/driver/CMakeLists.txt -@@ -35,13 +35,6 @@ set(BOLT_DEPENDS - ) - - add_custom_target(bolt DEPENDS ${BOLT_DEPENDS}) --install(PROGRAMS -- ${CMAKE_BINARY_DIR}/bin/llvm-bolt -- ${CMAKE_BINARY_DIR}/bin/perf2bolt -- ${CMAKE_BINARY_DIR}/bin/llvm-boltdiff -- DESTINATION ${CMAKE_INSTALL_BINDIR} -- COMPONENT bolt -- ) - add_llvm_install_targets(install-bolt DEPENDS bolt COMPONENT bolt) - set_target_properties(bolt PROPERTIES FOLDER "BOLT") - set_target_properties(install-bolt PROPERTIES FOLDER "BOLT") diff --git a/ports/llvm-15/0023-clang-sys-include-dir-path.patch b/ports/llvm-15/0023-clang-sys-include-dir-path.patch deleted file mode 100644 index 9a4dc6ad..00000000 --- a/ports/llvm-15/0023-clang-sys-include-dir-path.patch +++ /dev/null @@ -1,31 +0,0 @@ -diff --git a/clang/lib/Driver/ToolChains/Darwin.cpp b/clang/lib/Driver/ToolChains/Darwin.cpp -index 261f522f6c49..1aca52642169 100644 ---- a/clang/lib/Driver/ToolChains/Darwin.cpp -+++ b/clang/lib/Driver/ToolChains/Darwin.cpp -@@ -2126,11 +2126,12 @@ void DarwinClang::AddClangCXXStdlibIncludeArgs( - // include_next could break). - - // Check for (1) -- // Get from '/bin' to '/include/c++/v1'. -+ // Get from '/tools/llvm' to '/include/c++/v1'. - // Note that InstallBin can be relative, so we use '..' instead of - // parent_path. - llvm::SmallString<128> InstallBin = - llvm::StringRef(getDriver().getInstalledDir()); // /bin -+ llvm::sys::path::append(InstallBin, ".."); - llvm::sys::path::append(InstallBin, "..", "include", "c++", "v1"); - if (getVFS().exists(InstallBin)) { - addSystemInclude(DriverArgs, CC1Args, InstallBin); -diff --git a/clang/lib/Driver/ToolChains/Gnu.cpp b/clang/lib/Driver/ToolChains/Gnu.cpp -index da39f29e4619..acf46ec11ed3 100644 ---- a/clang/lib/Driver/ToolChains/Gnu.cpp -+++ b/clang/lib/Driver/ToolChains/Gnu.cpp -@@ -2922,7 +2922,7 @@ Generic_GCC::addLibCxxIncludePaths(const llvm::opt::ArgList &DriverArgs, - // Android never uses the libc++ headers installed alongside the toolchain, - // which are generally incompatible with the NDK libraries anyway. - if (!getTriple().isAndroid()) -- if (AddIncludePath(getDriver().Dir + "/../include")) -+ if (AddIncludePath(getDriver().Dir + "/../../include")) - return; - // If this is a development, non-installed, clang, libcxx will - // not be found at ../include/c++ but it likely to be found at diff --git a/ports/llvm-15/0025-PASTA-patches.patch b/ports/llvm-15/0025-PASTA-patches.patch deleted file mode 100644 index 67fe0492..00000000 --- a/ports/llvm-15/0025-PASTA-patches.patch +++ /dev/null @@ -1,2131 +0,0 @@ -From c521bb0dde19bcacaaed085c7d9202d658b87da9 Mon Sep 17 00:00:00 2001 -From: Peter Goodman -Date: Mon, 14 Nov 2022 14:51:27 -0500 -Subject: [PATCH] Patches for PASTA - ---- - clang/include/clang/Basic/TokenKinds.def | 7 + - clang/include/clang/Lex/PPCallbacks.h | 120 ++++++++++ - clang/include/clang/Lex/Preprocessor.h | 46 ++-- - clang/include/clang/Lex/TokenLexer.h | 7 +- - clang/lib/AST/ASTContext.cpp | 124 ++++++++--- - clang/lib/AST/ExprConstant.cpp | 9 + - clang/lib/AST/ItaniumMangle.cpp | 8 + - clang/lib/AST/Type.cpp | 9 +- - clang/lib/Lex/Lexer.cpp | 8 + - clang/lib/Lex/PPDirectives.cpp | 147 +++++++++---- - clang/lib/Lex/PPExpressions.cpp | 28 +++ - clang/lib/Lex/PPLexerChange.cpp | 38 ++++ - clang/lib/Lex/PPMacroExpansion.cpp | 267 ++++++++++++++++++++++- - clang/lib/Lex/Pragma.cpp | 59 +++++ - clang/lib/Lex/Preprocessor.cpp | 43 +++- - clang/lib/Lex/TokenLexer.cpp | 39 +++- - clang/lib/Parse/ParseExpr.cpp | 33 ++- - clang/lib/Parse/ParseTemplate.cpp | 7 + - clang/lib/Sema/SemaDecl.cpp | 12 + - clang/lib/Sema/SemaExpr.cpp | 32 ++- - clang/lib/Sema/SemaExprCXX.cpp | 1 + - clang/lib/Sema/TreeTransform.h | 24 ++ - 22 files changed, 953 insertions(+), 115 deletions(-) - -diff --git a/clang/include/clang/Basic/TokenKinds.def b/clang/include/clang/Basic/TokenKinds.def -index 84fc0893c..102e20a92 100644 ---- a/clang/include/clang/Basic/TokenKinds.def -+++ b/clang/include/clang/Basic/TokenKinds.def -@@ -554,6 +554,13 @@ ALIAS("__is_same_as", __is_same, KEYCXX) - KEYWORD(__private_extern__ , KEYALL) - KEYWORD(__module_private__ , KEYALL) - -+// PASTA PATCH: Apple-specific XNU extensions. -+UNARY_EXPR_OR_TYPE_TRAIT(__builtin_ptrauth_type_discriminator, PtrAuthTypeDiscriminator, KEYALL) -+UNARY_EXPR_OR_TYPE_TRAIT(__builtin_xnu_type_signature, XNUTypeSignature, KEYALL) -+UNARY_EXPR_OR_TYPE_TRAIT(__builtin_xnu_type_summary, XNUTypeSummary, KEYALL) -+UNARY_EXPR_OR_TYPE_TRAIT(__builtin_tmo_type_get_metadata, TMOTypeGetMetadata, KEYALL) -+TYPE_TRAIT_2(__builtin_xnu_types_compatible, XNUTypeCompatible, KEYALL) -+ - // Extension that will be enabled for Microsoft, Borland and PS4, but can be - // disabled via '-fno-declspec'. - KEYWORD(__declspec , 0) -diff --git a/clang/include/clang/Lex/PPCallbacks.h b/clang/include/clang/Lex/PPCallbacks.h -index 045df8711..cba195cff 100644 ---- a/clang/include/clang/Lex/PPCallbacks.h -+++ b/clang/include/clang/Lex/PPCallbacks.h -@@ -40,6 +40,120 @@ public: - EnterFile, ExitFile, SystemHeaderPragma, RenameFile - }; - -+ // PASTA PATCH: Add in an event that lets us get better visibility into the -+ // behavior of the preprocessor, as things are happening. -+#define LLVM_CLANG_HAS_PASTA_EVENTS 20221107L -+ enum EventKind { -+ // Tell us just after one of the lexers has lexed a token. -+ // -+ // `Tok` is the token generated from one of the underlying lexers. -+ // `Data` is zero or it is a raw source location for where the lexer was -+ // invoked. -+ TokenFromLexer, -+ TokenFromTokenLexer, -+ TokenFromCachingLexer, -+ TokenFromAfterModuleImportLexer, -+ -+ // Tell the listener that the parser has split a token. This happens in C++ -+ // code for templates, e.g. `constant>>0`, where the `>>>` is first -+ // treated as one token, but then where the parser realizes that it is -+ // really `constant> > 0`. -+ BeginSplitToken, -+ EndSplitToken, -+ -+ // Tell the listener that we've just lexed the hash token that should start -+ // off a directive. -+ // -+ // `Tok` is the `#`. -+ BeginDirective, -+ -+ // Ends with an `EndDirective`. -+ // -+ // `Tok` is the `#`. -+ BeginSkippedArea, -+ -+ // Tell the listener that we're in a named directive, e.g. `if` or `define`. -+ // -+ // `Tok` is the `#`. -+ // `Data` is a `Token *` of the token lexed after the `#`. -+ SetNamedDirective, -+ -+ // Tell the listener that we're in an unnamed directive, e.g. GNU line -+ // numbers, such as `# 1`. -+ // -+ // `Tok` is the `#`. -+ // `Data` is a `const Token *` of the token lexed after the `#`. -+ SetUnnamedDirective, -+ -+ // End a directive. -+ // -+ // `Tok` is the `tok::eod` token. -+ EndDirective, -+ -+ // We thought something was a directive, but it wasn't, e.g. due to us -+ // parsing a .S file. -+ EndNonDirective, -+ -+ // `Tok` is the name of the macro being expanded. -+ // `Data` is the `MacroInfo *`. For built-in macros, this may be `nullptr`. -+ BeginMacroExpansion, -+ SwitchToExpansion, -+ BeginPreArgumentExpansion, -+ EndPreArgumentExpansion, -+ PrepareToCancelExpansion, // E.g. `_Pragma` in macro argument pre-exansion. -+ CancelExpansion, // E.g. `_Pragma` in a macro parameter. -+ EndMacroExpansion, -+ -+ // `Tok` is the name of the macro being expanded. -+ // `Data` is a `MacroInfo *`. -+ BeginMacroCallArgumentList, -+ -+ // `Tok` is the token that terminated the argument list, i.e. a `)`. -+ // `Data` is a `MacroArgs *`. For built-in macros, this may be `nullptr`. -+ EndMacroCallArgumentList, -+ -+ // `Tok` is the token just before the first token of the argument, e.g. -+ // `(` or `,`. -+ // `Data` is a `Token *` of the macro name. -+ BeginMacroCallArgument, -+ -+ // `Tok` is the token just before the first token of the argument. -+ // `Data` is a `Token *` just after the last token of the argument, e.g. a -+ // `)` or a `,`. -+ EndMacroCallArgument, -+ -+ // `Tok` is the token just before the first token of the variadic arguments, -+ // e.g. a `(` or a `,`. -+ // `Data` is a `Token *` of the macro name. -+ BeginVariadicCallArgumentList, -+ -+ // `Tok` is the token just before the first token of the variadic arguments -+ // `Data` is a `Token *` just after the last token of the arguments, e.g. a -+ // `)` or a `,`. -+ EndVariadicCallArgumentList, -+ -+ // `Tok` is the token which begins the substitution. -+ // `Data` is `nullptr`. -+ BeginSubstitution, -+ -+ // `Tok` is the token (previously visible via another event) which we want -+ // to say begins the substitution. -+ // `Data` is `nullptr`. -+ BeginDelayedSubstitution, -+ -+ // `Tok` is the last token before the substituted tokens will begin being -+ // outputted. -+ // `Data` is a `Token *` of the first token of the substitution. -+ SwitchToSubstitution, -+ -+ // `Tok` is the last substituted token. -+ // `Data` is a `Token *` of the first token of the substitution. -+ EndSubstitution, -+ }; -+ -+ // PASTA PATCH: -+ virtual void Event(const Token &Tok, EventKind Kind, uintptr_t Data) {} -+ - /// Callback invoked whenever a source file is entered or exited. - /// - /// \param Loc Indicates the new location. -@@ -435,6 +549,12 @@ public: - - ~PPChainedCallbacks() override; - -+ // PASTA PATCH -+ void Event(const Token &Tok, EventKind Kind, uintptr_t Data) override { -+ First->Event(Tok, Kind, Data); -+ Second->Event(Tok, Kind, Data); -+ } -+ - void FileChanged(SourceLocation Loc, FileChangeReason Reason, - SrcMgr::CharacteristicKind FileType, - FileID PrevFID) override { -diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h -index 7c5df0506..643f7216f 100644 ---- a/clang/include/clang/Lex/Preprocessor.h -+++ b/clang/include/clang/Lex/Preprocessor.h -@@ -658,6 +658,9 @@ private: - /// encountered (e.g. a file is \#included, etc). - std::unique_ptr Callbacks; - -+ // PASTA PATCH: Add post-lex action. -+ std::function PostLexAction; -+ - struct MacroExpandsInfo { - Token Tok; - MacroDefinition MD; -@@ -1480,8 +1483,8 @@ public: - - /// Process directives while skipping until the through header or - /// #pragma hdrstop is found. -- void HandleSkippedDirectiveWhileUsingPCH(Token &Result, -- SourceLocation HashLoc); -+ void HandleSkippedDirectiveWhileUsingPCH(Token &Result, // PASTA PATCH -+ const Token &SavedHash); // PASTA PATCH - - /// Enter the specified FileID as the main source file, - /// which implicitly adds the builtin defines etc. -@@ -2261,15 +2264,7 @@ private: - CurPPLexer = nullptr; - } - -- void PopIncludeMacroStack() { -- CurLexer = std::move(IncludeMacroStack.back().TheLexer); -- CurPPLexer = IncludeMacroStack.back().ThePPLexer; -- CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer); -- CurDirLookup = IncludeMacroStack.back().TheDirLookup; -- CurLexerSubmodule = IncludeMacroStack.back().TheSubmodule; -- CurLexerKind = IncludeMacroStack.back().CurLexerKind; -- IncludeMacroStack.pop_back(); -- } -+ void PopIncludeMacroStack(); // PASTA PATCH - - void PropagateLineStartLeadingSpaceInfo(Token &Result); - -@@ -2333,7 +2328,7 @@ private: - /// \p FoundElse is false, then \#else directives are ok, if not, then we have - /// already seen one so a \#else directive is a duplicate. When this returns, - /// the caller can lex the first valid token. -- void SkipExcludedConditionalBlock(SourceLocation HashTokenLoc, -+ void SkipExcludedConditionalBlock(const Token &HashToken, // PASTA PATCH - SourceLocation IfTokenLoc, - bool FoundNonSkipPortion, bool FoundElse, - SourceLocation ElseLoc = SourceLocation()); -@@ -2401,7 +2396,7 @@ private: - /// After reading "MACRO(", this method is invoked to read all of the formal - /// arguments specified for the macro invocation. Returns null on error. - MacroArgs *ReadMacroCallArgumentList(Token &MacroName, MacroInfo *MI, -- SourceLocation &MacroEnd); -+ Token &MacroEndTok); // PASTA PATCH - - /// If an identifier token is read that is to be expanded - /// as a builtin macro, handle it and return the next token as 'Tok'. -@@ -2468,12 +2463,12 @@ private: - /// Handle*Directive - implement the various preprocessor directives. These - /// should side-effect the current preprocessor object so that the next call - /// to Lex() will return the appropriate token next. -- void HandleLineDirective(); -- void HandleDigitDirective(Token &Tok); -- void HandleUserDiagnosticDirective(Token &Tok, bool isWarning); -- void HandleIdentSCCSDirective(Token &Tok); -- void HandleMacroPublicDirective(Token &Tok); -- void HandleMacroPrivateDirective(); -+ void HandleLineDirective(const Token &SavedHash); // PASTA PATCH -+ void HandleDigitDirective(const Token &SavedHash, Token &Tok); // PASTA PATCH -+ void HandleUserDiagnosticDirective(const Token &SavedHash, Token &Tok, bool isWarning); // PASTA PATCH -+ void HandleIdentSCCSDirective(const Token &SavedHash, Token &Tok); // PASTA PATCH -+ void HandleMacroPublicDirective(const Token &SavedHash, Token &Tok); // PASTA PATCH -+ void HandleMacroPrivateDirective(const Token &SavedHash); // PASTA PATCH - - /// An additional notification that can be produced by a header inclusion or - /// import to tell the parser what happened. -@@ -2505,7 +2500,7 @@ private: - ModuleMap::KnownHeader &SuggestedModule, bool isAngled); - - // File inclusion. -- void HandleIncludeDirective(SourceLocation HashLoc, Token &Tok, -+ void HandleIncludeDirective(const Token &SavedHash, Token &Tok, // PASTA PATCH - ConstSearchDirIterator LookupFrom = nullptr, - const FileEntry *LookupFromFile = nullptr); - ImportAction -@@ -2513,9 +2508,9 @@ private: - Token &FilenameTok, SourceLocation EndLoc, - ConstSearchDirIterator LookupFrom = nullptr, - const FileEntry *LookupFromFile = nullptr); -- void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok); -- void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok); -- void HandleImportDirective(SourceLocation HashLoc, Token &Tok); -+ void HandleIncludeNextDirective(const Token &SavedHash, Token &Tok); // PASTA PATCH -+ void HandleIncludeMacrosDirective(const Token &SavedHash, Token &Tok); // PASTA PATCH -+ void HandleImportDirective(const Token &SavedHash, Token &Tok); // PASTA PATCH - void HandleMicrosoftImportDirective(Token &Tok); - - public: -@@ -2582,8 +2577,9 @@ private: - void replayPreambleConditionalStack(); - - // Macro handling. -- void HandleDefineDirective(Token &Tok, bool ImmediatelyAfterHeaderGuard); -- void HandleUndefDirective(); -+ void HandleDefineDirective(const Token &SavedHash, Token &Tok, // PASTA PATCH -+ bool ImmediatelyAfterHeaderGuard); // PASTA PATCH -+ void HandleUndefDirective(const Token &SavedHash); // PASTA PATCH - - // Conditional Inclusion. - void HandleIfdefDirective(Token &Result, const Token &HashToken, -diff --git a/clang/include/clang/Lex/TokenLexer.h b/clang/include/clang/Lex/TokenLexer.h -index 4d229ae61..923107b84 100644 ---- a/clang/include/clang/Lex/TokenLexer.h -+++ b/clang/include/clang/Lex/TokenLexer.h -@@ -13,7 +13,7 @@ - #ifndef LLVM_CLANG_LEX_TOKENLEXER_H - #define LLVM_CLANG_LEX_TOKENLEXER_H - --#include "clang/Basic/SourceLocation.h" -+#include "clang/Lex/Token.h" // PASTA PATCH - #include "llvm/ADT/ArrayRef.h" - - namespace clang { -@@ -40,6 +40,9 @@ class TokenLexer { - /// The current preprocessor object we are expanding for. - Preprocessor &PP; - -+ // PASTA PATCH: The name token of the macro. -+ Token MacroNameTok; -+ - /// This is the pointer to an array of tokens that the macro is - /// defined to, with arguments expanded for function-like macros. If this is - /// a token stream, these are the tokens we are returning. This points into -@@ -98,7 +101,7 @@ class TokenLexer { - - /// When true, the produced tokens have Token::IsReinjected flag set. - /// See the flag documentation for details. -- bool IsReinject : 1; -+ bool IsReinject; // PASTA PATCH - - public: - /// Create a TokenLexer for the specified macro with the specified actual -diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp -index cfd7bf604..fb20b33cd 100644 ---- a/clang/lib/AST/ASTContext.cpp -+++ b/clang/lib/AST/ASTContext.cpp -@@ -3519,48 +3519,94 @@ QualType ASTContext::getConstantArrayType(QualType EltTy, - EltTy->isIncompleteType() || EltTy->isConstantSizeType()) && - "Constant array of VLAs is illegal!"); - -- // We only need the size as part of the type if it's instantiation-dependent. -- if (SizeExpr && !SizeExpr->isInstantiationDependent()) -- SizeExpr = nullptr; -- - // Convert the array size into a canonical width matching the pointer size for - // the target. - llvm::APInt ArySize(ArySizeIn); - ArySize = ArySize.zextOrTrunc(Target->getMaxPointerWidth()); - -+ // PASTA PATCH: Make it always retain `SizeExpr` so that we can see token -+ // provenance for arrays whose types are the result of some -+ // expression. -+ const Expr *OrigSizeExpr = SizeExpr; -+ if (SizeExpr && !SizeExpr->isInstantiationDependent() && -+ !isa(SizeExpr) && !isa(SizeExpr)) { -+ llvm::APSInt ArySizeInt(ArySizeIn, !ArySizeIn.isNegative()); -+ OrigSizeExpr = ConstantExpr::Create( -+ *this, const_cast(SizeExpr), clang::APValue(ArySizeInt)); -+ } -+ -+ // We only need the size as part of the type if it's instantiation-dependent. -+ if (SizeExpr && !SizeExpr->isInstantiationDependent()) -+ SizeExpr = nullptr; -+ - llvm::FoldingSetNodeID ID; - ConstantArrayType::Profile(ID, *this, EltTy, ArySize, SizeExpr, ASM, - IndexTypeQuals); - - void *InsertPos = nullptr; -- if (ConstantArrayType *ATP = -- ConstantArrayTypes.FindNodeOrInsertPos(ID, InsertPos)) -+ ConstantArrayType *ATP = -+ ConstantArrayTypes.FindNodeOrInsertPos(ID, InsertPos); -+ -+ // PASTA PATCH: Profile the original one. In practice, we should probably -+ // never have a cache hit. -+ llvm::FoldingSetNodeID OrigID; -+ void *OrigInsertPos = nullptr; -+ ConstantArrayType *OrigATP = nullptr; -+ if (OrigSizeExpr) { -+ ConstantArrayType::Profile(OrigID, *this, EltTy, ArySize, OrigSizeExpr, ASM, -+ IndexTypeQuals); -+ OrigATP = ConstantArrayTypes.FindNodeOrInsertPos(OrigID, OrigInsertPos); -+ } -+ -+ if (ATP && OrigATP) -+ return getAdjustedType(QualType(OrigATP, 0), QualType(ATP, 0)); -+ -+ if (!OrigSizeExpr && ATP) - return QualType(ATP, 0); - -- // If the element type isn't canonical or has qualifiers, or the array bound -- // is instantiation-dependent, this won't be a canonical type either, so fill -- // in the canonical type field. -- QualType Canon; -- if (!EltTy.isCanonical() || EltTy.hasLocalQualifiers() || SizeExpr) { -- SplitQualType canonSplit = getCanonicalType(EltTy).split(); -- Canon = getConstantArrayType(QualType(canonSplit.Ty, 0), ArySize, nullptr, -- ASM, IndexTypeQuals); -- Canon = getQualifiedType(Canon, canonSplit.Quals); -+ if (!ATP) { -+ // If the element type isn't canonical or has qualifiers, or the array bound -+ // is instantiation-dependent, this won't be a canonical type either, so fill -+ // in the canonical type field. -+ QualType Canon; -+ if (!EltTy.isCanonical() || EltTy.hasLocalQualifiers() || SizeExpr) { -+ SplitQualType canonSplit = getCanonicalType(EltTy).split(); -+ Canon = getConstantArrayType(QualType(canonSplit.Ty, 0), ArySize, nullptr, -+ ASM, IndexTypeQuals); -+ Canon = getQualifiedType(Canon, canonSplit.Quals); - -- // Get the new insert position for the node we care about. -- ConstantArrayType *NewIP = -- ConstantArrayTypes.FindNodeOrInsertPos(ID, InsertPos); -- assert(!NewIP && "Shouldn't be in the map!"); (void)NewIP; -+ // Get the new insert position for the node we care about. -+ ConstantArrayType *NewIP = -+ ConstantArrayTypes.FindNodeOrInsertPos(ID, InsertPos); -+ assert(!NewIP && "Shouldn't be in the map!"); (void)NewIP; -+ -+ // TODO(pag): Calculate a new insert position for orig expressions? -+ } -+ -+ void *Mem = Allocate( -+ ConstantArrayType::totalSizeToAlloc(SizeExpr ? 1 : 0), -+ TypeAlignment); -+ ATP = new (Mem) -+ ConstantArrayType(EltTy, Canon, ArySize, SizeExpr, ASM, IndexTypeQuals); -+ ConstantArrayTypes.InsertNode(ATP, InsertPos); -+ Types.push_back(ATP); - } - -- void *Mem = Allocate( -- ConstantArrayType::totalSizeToAlloc(SizeExpr ? 1 : 0), -- TypeAlignment); -- auto *New = new (Mem) -- ConstantArrayType(EltTy, Canon, ArySize, SizeExpr, ASM, IndexTypeQuals); -- ConstantArrayTypes.InsertNode(New, InsertPos); -- Types.push_back(New); -- return QualType(New, 0); -+ if (OrigSizeExpr && !OrigATP) { -+ void *OrigMem = Allocate( -+ ConstantArrayType::totalSizeToAlloc(OrigSizeExpr ? 1 : 0), -+ TypeAlignment); -+ OrigATP = new (OrigMem) -+ ConstantArrayType(EltTy, ATP->getCanonicalTypeInternal(), -+ ArySize, OrigSizeExpr, ASM, IndexTypeQuals); -+ ConstantArrayTypes.InsertNode(OrigATP, OrigInsertPos); -+ Types.push_back(OrigATP); -+ } -+ -+ if (ATP && OrigATP) -+ return getAdjustedType(QualType(OrigATP, 0), QualType(ATP, 0)); -+ -+ return QualType(ATP, 0); - } - - /// getVariableArrayDecayedType - Turns the given type, which may be -@@ -3661,6 +3707,12 @@ QualType ASTContext::getVariableArrayDecayedType(QualType type) const { - cat->getSizeExpr(), - cat->getSizeModifier(), - cat->getIndexTypeCVRQualifiers()); -+ -+ // PASTA PATCH: If we returned an adjusted type above, then we want to -+ // desugar it to the internal type. -+ if (auto AT = dyn_cast(result.getTypePtr())) { -+ result = AT->getAdjustedType(); -+ } - break; - } - -@@ -6792,11 +6844,23 @@ const ArrayType *ASTContext::getAsArrayType(QualType T) const { - // qualifiers into the array element type and return a new array type. - QualType NewEltTy = getQualifiedType(ATy->getElementType(), qs); - -- if (const auto *CAT = dyn_cast(ATy)) -- return cast(getConstantArrayType(NewEltTy, CAT->getSize(), -+ if (const auto *CAT = dyn_cast(ATy)) { -+ -+ // PASTA PATCH: `getConstantArrayType` might return an adjusted type. -+ QualType result = getConstantArrayType(NewEltTy, CAT->getSize(), - CAT->getSizeExpr(), - CAT->getSizeModifier(), -- CAT->getIndexTypeCVRQualifiers())); -+ CAT->getIndexTypeCVRQualifiers()); -+ -+ // PASTA PATCH: If we returned an adjusted type above, then we want to -+ // desugar it to the internal type. -+ if (auto AT = dyn_cast(result.getTypePtr())) { -+ result = AT->getAdjustedType(); -+ } -+ -+ return cast(result); -+ } -+ - if (const auto *IAT = dyn_cast(ATy)) - return cast(getIncompleteArrayType(NewEltTy, - IAT->getSizeModifier(), -diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp -index 9d92c848c..50d445f49 100644 ---- a/clang/lib/AST/ExprConstant.cpp -+++ b/clang/lib/AST/ExprConstant.cpp -@@ -13287,6 +13287,15 @@ bool IntExprEvaluator::VisitUnaryExprOrTypeTraitExpr( - Info.Ctx.getOpenMPDefaultSimdAlign(E->getArgumentType())) - .getQuantity(), - E); -+ -+ // PASTA PATCH: Fake these XNU-specific type traits. -+ case UETT_PtrAuthTypeDiscriminator: -+ case UETT_XNUTypeSummary: -+ case UETT_TMOTypeGetMetadata: -+ return Success(0, E); -+ -+ case UETT_XNUTypeSignature: -+ return false; - } - - llvm_unreachable("unknown expr/type trait"); -diff --git a/clang/lib/AST/ItaniumMangle.cpp b/clang/lib/AST/ItaniumMangle.cpp -index 91f41778e..08a6fdd15 100644 ---- a/clang/lib/AST/ItaniumMangle.cpp -+++ b/clang/lib/AST/ItaniumMangle.cpp -@@ -4653,6 +4653,14 @@ recurse: - }; - - switch(SAE->getKind()) { -+ -+ // PASTA PATCH: Sort of support these. -+ case UETT_PtrAuthTypeDiscriminator: -+ case UETT_XNUTypeSummary: -+ case UETT_TMOTypeGetMetadata: -+ case UETT_XNUTypeSignature: -+ break; -+ - case UETT_SizeOf: - Out << 's'; - MangleAlignofSizeofArg(); -diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp -index 0f168a518..cd5497be5 100644 ---- a/clang/lib/AST/Type.cpp -+++ b/clang/lib/AST/Type.cpp -@@ -3994,9 +3994,14 @@ bool Type::hasUnnamedOrLocalType() const { - LinkageInfo LinkageComputer::computeTypeLinkageInfo(const Type *T) { - switch (T->getTypeClass()) { - #define TYPE(Class,Base) --#define NON_CANONICAL_TYPE(Class,Base) case Type::Class: -+ -+// PASTA PATCH: For Multiplier Issue #130, we use AdjustedTypes for arrays -+// and so we go and handle non-canonical types via desugarding -+// where Clang normally does not. -+#define NON_CANONICAL_TYPE(Class,Base) \ -+ case Type::Class: \ -+ return computeTypeLinkageInfo(cast(T)->desugar()); - #include "clang/AST/TypeNodes.inc" -- llvm_unreachable("didn't expect a non-canonical type here"); - - #define TYPE(Class,Base) - #define DEPENDENT_TYPE(Class,Base) case Type::Class: -diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp -index a4cff403e..33f17b148 100644 ---- a/clang/lib/Lex/Lexer.cpp -+++ b/clang/lib/Lex/Lexer.cpp -@@ -2921,6 +2921,14 @@ void Lexer::ReadToEndOfLine(SmallVectorImpl *Result) { - } - assert(Tmp.is(tok::eod) && "Unexpected token!"); - -+ // PASTA PATCH: Visibility into all tokens. -+ if (PP) { -+ if (auto Callbacks = PP->getPPCallbacks()) { -+ Callbacks->Event(Tmp, PPCallbacks::TokenFromTokenLexer, 0u); -+ Callbacks->Event(Tmp, PPCallbacks::EndDirective, 0); -+ } -+ } -+ - // Finally, we're done; - return; - } -diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp -index 9a8fd4391..b20144c12 100644 ---- a/clang/lib/Lex/PPDirectives.cpp -+++ b/clang/lib/Lex/PPDirectives.cpp -@@ -91,6 +91,12 @@ SourceRange Preprocessor::DiscardUntilEndOfDirective() { - LexUnexpandedToken(Tmp); - } - Res.setEnd(Tmp.getLocation()); -+ -+// // PASTA PATCH: -+// if (Callbacks) { -+// Callbacks->Event(Tmp, PPCallbacks::EndDirective, 0); -+// } -+ - return Res; - } - -@@ -477,7 +483,7 @@ void Preprocessor::SuggestTypoedDirective(const Token &Tok, - /// If ElseOk is true, then \#else directives are ok, if not, then we have - /// already seen one so a \#else directive is a duplicate. When this returns, - /// the caller can lex the first valid token. --void Preprocessor::SkipExcludedConditionalBlock(SourceLocation HashTokenLoc, -+void Preprocessor::SkipExcludedConditionalBlock(const Token &HashToken, // PASTA PATCH - SourceLocation IfTokenLoc, - bool FoundNonSkipPortion, - bool FoundElse, -@@ -504,6 +510,12 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation HashTokenLoc, - CurPPLexer->pushConditionalLevel(IfTokenLoc, /*isSkipping*/ false, - FoundNonSkipPortion, FoundElse); - -+ // PASTA PATCH -+ SourceLocation HashTokenLoc = HashToken.getLocation(); -+ if (Callbacks) { -+ Callbacks->Event(HashToken, PPCallbacks::BeginSkippedArea, 0); -+ } -+ - // Enter raw mode to disable identifier lookup (and thus macro expansion), - // disabling warnings, etc. - CurPPLexer->LexingRawMode = true; -@@ -651,6 +663,12 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation HashTokenLoc, - if (Sub.empty() || // "if" - Sub == "def" || // "ifdef" - Sub == "ndef") { // "ifndef" -+ -+ // PASTA PATCH: -+ if (Callbacks) { -+ Callbacks->Event(Tok, PPCallbacks::BeginSkippedArea, 0); -+ } -+ - // We know the entire #if/#ifdef/#ifndef block will be skipped, don't - // bother parsing the condition. - DiscardUntilEndOfDirective(); -@@ -661,6 +679,12 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation HashTokenLoc, - SuggestTypoedDirective(Tok, Directive); - } - } else if (Directive[0] == 'e') { -+ -+ // PASTA PATCH: -+ if (Callbacks) { -+ Callbacks->Event(Tok, PPCallbacks::BeginSkippedArea, 0); -+ } -+ - StringRef Sub = Directive.substr(1); - if (Sub == "ndif") { // "endif" - PPConditionalInfo CondInfo; -@@ -1102,16 +1126,16 @@ private: - /// #define (to warn about macros that don't match the PCH) - /// #pragma (to check for pragma hdrstop). - /// All other directives are completely discarded. --void Preprocessor::HandleSkippedDirectiveWhileUsingPCH(Token &Result, -- SourceLocation HashLoc) { -+void Preprocessor::HandleSkippedDirectiveWhileUsingPCH( // PASTA PATCH -+ Token &Result, const Token &SavedHash) { // PASTA PATCH - if (const IdentifierInfo *II = Result.getIdentifierInfo()) { - if (II->getPPKeywordID() == tok::pp_define) { -- return HandleDefineDirective(Result, -- /*ImmediatelyAfterHeaderGuard=*/false); -+ return HandleDefineDirective(SavedHash, Result, // PASTA PATCH -+ /*ImmediatelyAfterHeaderGuard=*/false); // PASTA PATCH - } - if (SkippingUntilPCHThroughHeader && - II->getPPKeywordID() == tok::pp_include) { -- return HandleIncludeDirective(HashLoc, Result); -+ return HandleIncludeDirective(SavedHash, Result); // PASTA PATCH - } - if (SkippingUntilPragmaHdrStop && II->getPPKeywordID() == tok::pp_pragma) { - Lex(Result); -@@ -1150,10 +1174,36 @@ void Preprocessor::HandleDirective(Token &Result) { - // Save the '#' token in case we need to return it later. - Token SavedHash = Result; - -+ // PASTA PATCH: Tell us when we're about to start a directive. -+ if (Callbacks) { -+ Callbacks->Event(SavedHash, PPCallbacks::BeginDirective, 0); -+ } -+ - // Read the next token, the directive flavor. This isn't expanded due to - // C99 6.10.3p8. - LexUnexpandedToken(Result); - -+ // PASTA PATCH: Tell us when we're about to start a directive. -+ if (Callbacks) { -+ switch (Result.getKind()) { -+ case tok::identifier: -+ case tok::raw_identifier: -+ case tok::kw_if: -+ case tok::kw_else: -+ Callbacks->Event(SavedHash, PPCallbacks::SetNamedDirective, -+ reinterpret_cast(&Result)); -+ break; -+ case tok::eod: -+ case tok::code_completion: -+ case tok::numeric_constant: -+ case tok::string_literal: -+ default: -+ Callbacks->Event(SavedHash, PPCallbacks::SetUnnamedDirective, -+ reinterpret_cast(&Result)); -+ break; -+ } -+ } -+ - // C99 6.10.3p11: Is this preprocessor directive in macro invocation? e.g.: - // #define A(x) #x - // A(abc -@@ -1187,7 +1237,7 @@ void Preprocessor::HandleDirective(Token &Result) { - ResetMacroExpansionHelper helper(this); - - if (SkippingUntilPCHThroughHeader || SkippingUntilPragmaHdrStop) -- return HandleSkippedDirectiveWhileUsingPCH(Result, SavedHash.getLocation()); -+ return HandleSkippedDirectiveWhileUsingPCH(Result, SavedHash); // PASTA PATCH - - switch (Result.getKind()) { - case tok::eod: -@@ -1201,7 +1251,7 @@ void Preprocessor::HandleDirective(Token &Result) { - case tok::numeric_constant: // # 7 GNU line marker directive. - if (getLangOpts().AsmPreprocessor) - break; // # 4 is not a preprocessor directive in .S files. -- return HandleDigitDirective(Result); -+ return HandleDigitDirective(SavedHash, Result); // PASTA PATCH - default: - IdentifierInfo *II = Result.getIdentifierInfo(); - if (!II) break; // Not an identifier. -@@ -1231,24 +1281,25 @@ void Preprocessor::HandleDirective(Token &Result) { - // C99 6.10.2 - Source File Inclusion. - case tok::pp_include: - // Handle #include. -- return HandleIncludeDirective(SavedHash.getLocation(), Result); -+ return HandleIncludeDirective(SavedHash, Result); // PASTA PATCH - case tok::pp___include_macros: - // Handle -imacros. -- return HandleIncludeMacrosDirective(SavedHash.getLocation(), Result); -+ return HandleIncludeMacrosDirective(SavedHash, Result); // PASTA PATCH - - // C99 6.10.3 - Macro Replacement. - case tok::pp_define: -- return HandleDefineDirective(Result, ImmediatelyAfterTopLevelIfndef); -+ return HandleDefineDirective(SavedHash, Result, // PASTA PATCH -+ ImmediatelyAfterTopLevelIfndef); // PASTA PATCH - case tok::pp_undef: -- return HandleUndefDirective(); -+ return HandleUndefDirective(SavedHash); // PASTA PATCH - - // C99 6.10.4 - Line Control. - case tok::pp_line: -- return HandleLineDirective(); -+ return HandleLineDirective(SavedHash); // PASTA PATCH - - // C99 6.10.5 - Error Directive. - case tok::pp_error: -- return HandleUserDiagnosticDirective(Result, false); -+ return HandleUserDiagnosticDirective(SavedHash, Result, false); // PASTA PATCH - - // C99 6.10.6 - Pragma Directive. - case tok::pp_pragma: -@@ -1256,9 +1307,9 @@ void Preprocessor::HandleDirective(Token &Result) { - - // GNU Extensions. - case tok::pp_import: -- return HandleImportDirective(SavedHash.getLocation(), Result); -+ return HandleImportDirective(SavedHash, Result); // PASTA PATCH - case tok::pp_include_next: -- return HandleIncludeNextDirective(SavedHash.getLocation(), Result); -+ return HandleIncludeNextDirective(SavedHash, Result); // PASTA PATCH - - case tok::pp_warning: - if (LangOpts.CPlusPlus) -@@ -1271,11 +1322,11 @@ void Preprocessor::HandleDirective(Token &Result) { - : diag::ext_pp_warning_directive) - << /*C2x*/ 0; - -- return HandleUserDiagnosticDirective(Result, true); -+ return HandleUserDiagnosticDirective(SavedHash, Result, true); // PASTA PATCH - case tok::pp_ident: -- return HandleIdentSCCSDirective(Result); -+ return HandleIdentSCCSDirective(SavedHash, Result); // PASTA PATCH - case tok::pp_sccs: -- return HandleIdentSCCSDirective(Result); -+ return HandleIdentSCCSDirective(SavedHash, Result); // PASTA PATCH - case tok::pp_assert: - //isExtension = true; // FIXME: implement #assert - break; -@@ -1285,12 +1336,12 @@ void Preprocessor::HandleDirective(Token &Result) { - - case tok::pp___public_macro: - if (getLangOpts().Modules || getLangOpts().ModulesLocalVisibility) -- return HandleMacroPublicDirective(Result); -+ return HandleMacroPublicDirective(SavedHash, Result); // PASTA PATCH - break; - - case tok::pp___private_macro: - if (getLangOpts().Modules || getLangOpts().ModulesLocalVisibility) -- return HandleMacroPrivateDirective(); -+ return HandleMacroPrivateDirective(SavedHash); // PASTA PATCH - break; - } - break; -@@ -1301,6 +1352,11 @@ void Preprocessor::HandleDirective(Token &Result) { - // various pseudo-ops. Just return the # token and push back the following - // token to be lexed next time. - if (getLangOpts().AsmPreprocessor) { -+ // PASTA PATCH: Get visibility on end of macro directives. -+ if (Callbacks) { -+ Callbacks->Event(SavedHash, PPCallbacks::EndNonDirective, 0); -+ } -+ - auto Toks = std::make_unique(2); - // Return the # and the token after it. - Toks[0] = SavedHash; -@@ -1389,7 +1445,7 @@ static bool GetLineValue(Token &DigitTok, unsigned &Val, - /// # line digit-sequence - /// # line digit-sequence "s-char-sequence" - /// \endverbatim --void Preprocessor::HandleLineDirective() { -+void Preprocessor::HandleLineDirective(const Token &SavedHash) { // PASTA PATCH - // Read the line # and string argument. Per C99 6.10.4p5, these tokens are - // expanded. - Token DigitTok; -@@ -1551,7 +1607,8 @@ static bool ReadLineMarkerFlags(bool &IsFileEntry, bool &IsFileExit, - /// # 42 "file" ('1' | '2')? - /// # 42 "file" ('1' | '2')? '3' '4'? - /// --void Preprocessor::HandleDigitDirective(Token &DigitTok) { -+void Preprocessor::HandleDigitDirective( // PASTA PATCH -+ const Token &SavedHash, Token &DigitTok) { // PASTA PATCH - // Validate the number and convert it to an unsigned. GNU does not have a - // line # limit other than it fit in 32-bits. - unsigned LineNo; -@@ -1627,8 +1684,8 @@ void Preprocessor::HandleDigitDirective(Token &DigitTok) { - - /// HandleUserDiagnosticDirective - Handle a #warning or #error directive. - /// --void Preprocessor::HandleUserDiagnosticDirective(Token &Tok, -- bool isWarning) { -+void Preprocessor::HandleUserDiagnosticDirective(const Token &SavedHash, // PASTA PATCH -+ Token &Tok, bool isWarning) { // PASTA PATCH - // Read the rest of the line raw. We do this because we don't want macros - // to be expanded and we don't require that the tokens be valid preprocessing - // tokens. For example, this is allowed: "#warning ` 'foo". GCC does -@@ -1649,7 +1706,8 @@ void Preprocessor::HandleUserDiagnosticDirective(Token &Tok, - - /// HandleIdentSCCSDirective - Handle a #ident/#sccs directive. - /// --void Preprocessor::HandleIdentSCCSDirective(Token &Tok) { -+void Preprocessor::HandleIdentSCCSDirective(const Token &SavedHash, // PASTA PATCH -+ Token &Tok) { // PASTA PATCH - // Yes, this directive is an extension. - Diag(Tok, diag::ext_pp_ident_directive); - -@@ -1684,7 +1742,8 @@ void Preprocessor::HandleIdentSCCSDirective(Token &Tok) { - } - - /// Handle a #public directive. --void Preprocessor::HandleMacroPublicDirective(Token &Tok) { -+void Preprocessor::HandleMacroPublicDirective(const Token &SavedHash, // PASTA PATCH -+ Token &Tok) { // PASTA PATCH - Token MacroNameTok; - ReadMacroName(MacroNameTok, MU_Undef); - -@@ -1711,7 +1770,7 @@ void Preprocessor::HandleMacroPublicDirective(Token &Tok) { - } - - /// Handle a #private directive. --void Preprocessor::HandleMacroPrivateDirective() { -+void Preprocessor::HandleMacroPrivateDirective(const Token &SavedHash) { // PASTA PATCH - Token MacroNameTok; - ReadMacroName(MacroNameTok, MU_Undef); - -@@ -1948,10 +2007,11 @@ Preprocessor::getIncludeNextStart(const Token &IncludeNextTok) const { - /// routine with functionality shared between \#include, \#include_next and - /// \#import. LookupFrom is set when this is a \#include_next directive, it - /// specifies the file to start searching from. --void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc, -+void Preprocessor::HandleIncludeDirective(const Token &SavedHash, // PASTA PATCH - Token &IncludeTok, - ConstSearchDirIterator LookupFrom, - const FileEntry *LookupFromFile) { -+ SourceLocation HashLoc = SavedHash.getLocation(); // PASTA PATCH - Token FilenameTok; - if (LexHeaderName(FilenameTok)) - return; -@@ -2574,7 +2634,7 @@ Preprocessor::ImportAction Preprocessor::HandleHeaderIncludeOrImport( - - /// HandleIncludeNextDirective - Implements \#include_next. - /// --void Preprocessor::HandleIncludeNextDirective(SourceLocation HashLoc, -+void Preprocessor::HandleIncludeNextDirective(const Token &SavedHash, // PASTA PATCH - Token &IncludeNextTok) { - Diag(IncludeNextTok, diag::ext_pp_include_next_directive); - -@@ -2582,7 +2642,7 @@ void Preprocessor::HandleIncludeNextDirective(SourceLocation HashLoc, - const FileEntry *LookupFromFile; - std::tie(Lookup, LookupFromFile) = getIncludeNextStart(IncludeNextTok); - -- return HandleIncludeDirective(HashLoc, IncludeNextTok, Lookup, -+ return HandleIncludeDirective(SavedHash, IncludeNextTok, Lookup, - LookupFromFile); - } - -@@ -2602,21 +2662,21 @@ void Preprocessor::HandleMicrosoftImportDirective(Token &Tok) { - - /// HandleImportDirective - Implements \#import. - /// --void Preprocessor::HandleImportDirective(SourceLocation HashLoc, -- Token &ImportTok) { -+void Preprocessor::HandleImportDirective(const Token &SavedHash, // PASTA PATCH -+ Token &ImportTok) { // PASTA PATCH - if (!LangOpts.ObjC) { // #import is standard for ObjC. - if (LangOpts.MSVCCompat) - return HandleMicrosoftImportDirective(ImportTok); - Diag(ImportTok, diag::ext_pp_import_directive); - } -- return HandleIncludeDirective(HashLoc, ImportTok); -+ return HandleIncludeDirective(SavedHash, ImportTok); // PASTA PATCH - } - - /// HandleIncludeMacrosDirective - The -imacros command line option turns into a - /// pseudo directive in the predefines buffer. This handles it by sucking all - /// tokens through the preprocessor and discarding them (only keeping the side - /// effects on the preprocessor). --void Preprocessor::HandleIncludeMacrosDirective(SourceLocation HashLoc, -+void Preprocessor::HandleIncludeMacrosDirective(const Token &SavedHash, // PASTA PATCH - Token &IncludeMacrosTok) { - // This directive should only occur in the predefines buffer. If not, emit an - // error and reject it. -@@ -2630,7 +2690,7 @@ void Preprocessor::HandleIncludeMacrosDirective(SourceLocation HashLoc, - - // Treat this as a normal #include for checking purposes. If this is - // successful, it will push a new lexer onto the include stack. -- HandleIncludeDirective(HashLoc, IncludeMacrosTok); -+ HandleIncludeDirective(SavedHash, IncludeMacrosTok); // PASTA PATCH - - Token TmpTok; - do { -@@ -3019,7 +3079,8 @@ MacroInfo *Preprocessor::ReadOptionalMacroParameterListAndBody( - /// HandleDefineDirective - Implements \#define. This consumes the entire macro - /// line then lets the caller lex the next real token. - void Preprocessor::HandleDefineDirective( -- Token &DefineTok, const bool ImmediatelyAfterHeaderGuard) { -+ const Token &SavedHash, Token &DefineTok, // PASTA PATCH -+ const bool ImmediatelyAfterHeaderGuard) { // PASTA PATCH - ++NumDefined; - - Token MacroNameTok; -@@ -3173,7 +3234,7 @@ void Preprocessor::HandleDefineDirective( - - /// HandleUndefDirective - Implements \#undef. - /// --void Preprocessor::HandleUndefDirective() { -+void Preprocessor::HandleUndefDirective(const Token &SavedHash) { // PASTA PATCH - ++NumUndefined; - - Token MacroNameTok; -@@ -3237,7 +3298,7 @@ void Preprocessor::HandleIfdefDirective(Token &Result, - if (MacroNameTok.is(tok::eod)) { - // Skip code until we get to #endif. This helps with recovery by not - // emitting an error when the #endif is reached. -- SkipExcludedConditionalBlock(HashToken.getLocation(), -+ SkipExcludedConditionalBlock(HashToken, // PASTA PATCH - DirectiveTok.getLocation(), - /*Foundnonskip*/ false, /*FoundElse*/ false); - return; -@@ -3292,7 +3353,7 @@ void Preprocessor::HandleIfdefDirective(Token &Result, - /*foundelse*/false); - } else { - // No, skip the contents of this block. -- SkipExcludedConditionalBlock(HashToken.getLocation(), -+ SkipExcludedConditionalBlock(HashToken, // PASTA PATCH - DirectiveTok.getLocation(), - /*Foundnonskip*/ false, - /*FoundElse*/ false); -@@ -3345,7 +3406,7 @@ void Preprocessor::HandleIfDirective(Token &IfToken, - /*foundnonskip*/true, /*foundelse*/false); - } else { - // No, skip the contents of this block. -- SkipExcludedConditionalBlock(HashToken.getLocation(), IfToken.getLocation(), -+ SkipExcludedConditionalBlock(HashToken, IfToken.getLocation(), // PASTA PATCH - /*Foundnonskip*/ false, - /*FoundElse*/ false); - } -@@ -3413,7 +3474,7 @@ void Preprocessor::HandleElseDirective(Token &Result, const Token &HashToken) { - } - - // Finally, skip the rest of the contents of this block. -- SkipExcludedConditionalBlock(HashToken.getLocation(), CI.IfLoc, -+ SkipExcludedConditionalBlock(HashToken, CI.IfLoc, // PASTA PATCH - /*Foundnonskip*/ true, - /*FoundElse*/ true, Result.getLocation()); - } -@@ -3494,6 +3555,6 @@ void Preprocessor::HandleElifFamilyDirective(Token &ElifToken, - - // Finally, skip the rest of the contents of this block. - SkipExcludedConditionalBlock( -- HashToken.getLocation(), CI.IfLoc, /*Foundnonskip*/ true, -+ HashToken, CI.IfLoc, /*Foundnonskip*/ true, // PASTA PATCH - /*FoundElse*/ CI.FoundElse, ElifToken.getLocation()); - } -diff --git a/clang/lib/Lex/PPExpressions.cpp b/clang/lib/Lex/PPExpressions.cpp -index bd35689f1..dd97bb3e2 100644 ---- a/clang/lib/Lex/PPExpressions.cpp -+++ b/clang/lib/Lex/PPExpressions.cpp -@@ -101,6 +101,32 @@ struct DefinedTracker { - /// EvaluateDefined - Process a 'defined(sym)' expression. - static bool EvaluateDefined(PPValue &Result, Token &PeekTok, DefinedTracker &DT, - bool ValueLive, Preprocessor &PP) { -+ // PASTA PATCH -+ PPCallbacks *Callbacks = PP.getPPCallbacks(); -+ Token SavedStart = PeekTok; -+ if (Callbacks) { -+ Callbacks->Event(SavedStart, PPCallbacks::BeginDelayedSubstitution, 0); -+ } -+ auto Expand = [&] (void) { -+ if (Callbacks) { -+ Token ResultTok; -+ ResultTok.startToken(); -+ ResultTok.setKind(tok::numeric_constant); -+ const char *ResultStr = Result.Val.getExtValue() ? "1" : "0"; -+ PP.CreateString(ResultStr, ResultTok, SavedStart.getLocation(), -+ PeekTok.getEndLoc()); -+ -+ Callbacks->Event(PeekTok, PPCallbacks::SwitchToSubstitution, -+ reinterpret_cast(&SavedStart)); -+ -+ Callbacks->Event(ResultTok, PPCallbacks::TokenFromTokenLexer, -+ SavedStart.getLocation().getRawEncoding()); -+ -+ Callbacks->Event(SavedStart, PPCallbacks::EndSubstitution, -+ reinterpret_cast(&SavedStart)); -+ } -+ }; -+ - SourceLocation beginLoc(PeekTok.getLocation()); - Result.setBegin(beginLoc); - -@@ -154,12 +180,14 @@ static bool EvaluateDefined(PPValue &Result, Token &PeekTok, DefinedTracker &DT, - PP.Diag(LParenLoc, diag::note_matching) << tok::l_paren; - return true; - } -+ Expand(); // PASTA PATCH - // Consume the ). - PP.LexNonComment(PeekTok); - Result.setEnd(PeekTok.getLocation()); - } else { - // Consume identifier. - Result.setEnd(PeekTok.getLocation()); -+ Expand(); // PASTA PATCH - PP.LexNonComment(PeekTok); - } - -diff --git a/clang/lib/Lex/PPLexerChange.cpp b/clang/lib/Lex/PPLexerChange.cpp -index 36d3aa59b..19697799e 100644 ---- a/clang/lib/Lex/PPLexerChange.cpp -+++ b/clang/lib/Lex/PPLexerChange.cpp -@@ -29,6 +29,36 @@ using namespace clang; - // Miscellaneous Methods. - //===----------------------------------------------------------------------===// - -+void Preprocessor::PopIncludeMacroStack() { -+ -+ // PASTA PATCH: Tell us about the end of macro expansions. -+ if (Callbacks && CurTokenLexer && CurTokenLexer->Macro) { -+ Callbacks->Event(CurTokenLexer->MacroNameTok, -+ PPCallbacks::EndMacroExpansion, -+ reinterpret_cast(CurTokenLexer->Macro)); -+ } -+ -+ // PASTA PATCH: Make us aware of the end of `_Pragma` handling. -+ if (Callbacks && CurPPLexer && CurLexer.get() == CurPPLexer && -+ CurLexer->isPragmaLexer()) { -+ Token PragmaTok; -+ PragmaTok.setKind(tok::raw_identifier); -+ PragmaTok.setLocation(SourceMgr.getExpansionLoc(CurLexer->getFileLoc())); -+ PragmaTok.setLength(7u); -+ PragmaTok.setRawIdentifierData( -+ SourceMgr.getCharacterData(PragmaTok.getLocation(), nullptr)); -+ Callbacks->Event(PragmaTok, PPCallbacks::EndMacroExpansion, 0); -+ } -+ -+ CurLexer = std::move(IncludeMacroStack.back().TheLexer); -+ CurPPLexer = IncludeMacroStack.back().ThePPLexer; -+ CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer); -+ CurDirLookup = IncludeMacroStack.back().TheDirLookup; -+ CurLexerSubmodule = IncludeMacroStack.back().TheSubmodule; -+ CurLexerKind = IncludeMacroStack.back().CurLexerKind; -+ IncludeMacroStack.pop_back(); -+} -+ - /// isInPrimaryFile - Return true if we're in the top-level file, not in a - /// \#include. This looks through macro expansions and active _Pragma lexers. - bool Preprocessor::isInPrimaryFile() const { -@@ -587,6 +617,14 @@ bool Preprocessor::HandleEndOfTokenLexer(Token &Result) { - assert(CurTokenLexer && !CurPPLexer && - "Ending a macro when currently in a #include file!"); - -+ // PASTA PATCH: Tell us about the end of macro expansions. -+ if (CurTokenLexer && CurTokenLexer->Macro) { -+ Callbacks->Event(CurTokenLexer->MacroNameTok, -+ PPCallbacks::EndMacroExpansion, -+ reinterpret_cast(CurTokenLexer->Macro)); -+ CurTokenLexer->Macro = nullptr; -+ } -+ - if (!MacroExpandingLexersStack.empty() && - MacroExpandingLexersStack.back().first == CurTokenLexer.get()) - removeCachedMacroExpandedTokensOfLastLexer(); -diff --git a/clang/lib/Lex/PPMacroExpansion.cpp b/clang/lib/Lex/PPMacroExpansion.cpp -index c56f41c44..5f61b2476 100644 ---- a/clang/lib/Lex/PPMacroExpansion.cpp -+++ b/clang/lib/Lex/PPMacroExpansion.cpp -@@ -60,6 +60,58 @@ - - using namespace clang; - -+namespace { -+ -+// PASTA PATCH: Tell us about the end of macro expansions. We have to add -+// this as a post-lex action, so that we can observe what is actually lexed -+// into `Identifier`. -+struct DeferExpansionEnd { -+ std::function &PostLexAction; -+ PPCallbacks * const Callbacks; -+ Token Identifier; -+ MacroInfo * const MI; -+ -+ DeferExpansionEnd(std::function &PostLexAction_, -+ PPCallbacks *Callbacks_, Token Identifier_, MacroInfo *MI_) -+ : PostLexAction(PostLexAction_), -+ Callbacks(Callbacks_), -+ Identifier(std::move(Identifier_)), -+ MI(MI_) {} -+ -+ ~DeferExpansionEnd(void) { -+ if (Callbacks) { -+ PostLexAction = [CB = Callbacks, Ident = std::move(Identifier), Info = MI, -+ PrevPostLexAction = std::move(PostLexAction)] -+ (const Token &Tok) { -+ CB->Event(Ident, PPCallbacks::EndMacroExpansion, -+ reinterpret_cast(Info)); -+ PrevPostLexAction(Tok); -+ }; -+ } -+ } -+}; -+ -+struct DeferExpansionCancellation { -+ std::function Action; -+ bool DoCancel{true}; -+ -+ template -+ DeferExpansionCancellation(CB Action_) -+ : Action(std::move(Action_)) {} -+ -+ ~DeferExpansionCancellation(void) { -+ if (DoCancel) { -+ Action(); -+ } -+ } -+ -+ void DisableCancellation(void) { -+ DoCancel = false; -+ } -+}; -+ -+} // namespace -+ - MacroDirective * - Preprocessor::getLocalMacroDirectiveHistory(const IdentifierInfo *II) const { - if (!II->hadMacroDefinition()) -@@ -478,6 +530,19 @@ bool Preprocessor::HandleMacroExpandedIdentifier(Token &Identifier, - - MacroInfo *MI = M.getMacroInfo(); - -+ // PASTA PATCH: Visibility into macro expansion. -+ Token SavedIdentifier = Identifier; -+ if (Callbacks) { -+ Callbacks->Event(SavedIdentifier, PPCallbacks::BeginMacroExpansion, -+ reinterpret_cast(MI)); -+ } -+ DeferExpansionCancellation CancelExpansion([&, this] (void) { -+ if (Callbacks) { -+ Callbacks->Event(SavedIdentifier, PPCallbacks::CancelExpansion, -+ reinterpret_cast(MI)); -+ } -+ }); -+ - // If this is a macro expansion in the "#if !defined(x)" line for the file, - // then the macro could expand to different things in other contexts, we need - // to disable the optimization in this case. -@@ -488,6 +553,8 @@ bool Preprocessor::HandleMacroExpandedIdentifier(Token &Identifier, - if (Callbacks) - Callbacks->MacroExpands(Identifier, M, Identifier.getLocation(), - /*Args=*/nullptr); -+ -+ CancelExpansion.DisableCancellation(); // PASTA PATCH - ExpandBuiltinMacro(Identifier); - return true; - } -@@ -509,7 +576,22 @@ bool Preprocessor::HandleMacroExpandedIdentifier(Token &Identifier, - InMacroArgs = true; - ArgMacro = &Identifier; - -- Args = ReadMacroCallArgumentList(Identifier, MI, ExpansionEnd); -+ // PASTA PATCH: Visibility into macro expansion. -+ if (Callbacks) { -+ Callbacks->Event(SavedIdentifier, PPCallbacks::BeginMacroCallArgumentList, -+ reinterpret_cast(MI)); -+ } -+ -+ // PASTA PATCH: Visibility to last token in argument list. -+ Token ExpansionEndTok; -+ Args = ReadMacroCallArgumentList(Identifier, MI, ExpansionEndTok); -+ ExpansionEnd = ExpansionEndTok.getLocation(); -+ -+ // PASTA PATCH: Visibility into macro expansion. -+ if (Callbacks) { -+ Callbacks->Event(ExpansionEndTok, PPCallbacks::EndMacroCallArgumentList, -+ reinterpret_cast(Args)); -+ } - - // Finished parsing args. - InMacroArgs = false; -@@ -568,7 +650,7 @@ bool Preprocessor::HandleMacroExpandedIdentifier(Token &Identifier, - - // If this macro expands to no tokens, don't bother to push it onto the - // expansion stack, only to take it right back off. -- if (MI->getNumTokens() == 0) { -+ if (false && MI->getNumTokens() == 0) { // PASTA PATCH - // No need for arg info. - if (Args) Args->destroy(*this); - -@@ -578,7 +660,7 @@ bool Preprocessor::HandleMacroExpandedIdentifier(Token &Identifier, - PropagateLineStartLeadingSpaceInfo(Identifier); - ++NumFastMacroExpanded; - return false; -- } else if (MI->getNumTokens() == 1 && -+ } else if (false && MI->getNumTokens() == 1 && // PASTA PATCH - isTrivialSingleTokenExpansion(MI, Identifier.getIdentifierInfo(), - *this)) { - // Otherwise, if this macro expands into a single trivially-expanded -@@ -626,6 +708,13 @@ bool Preprocessor::HandleMacroExpandedIdentifier(Token &Identifier, - return true; - } - -+ // PASTA PATCH: Switch state to now start expanding. -+ CancelExpansion.DisableCancellation(); -+ if (Callbacks) { -+ Callbacks->Event(SavedIdentifier, PPCallbacks::SwitchToExpansion, -+ reinterpret_cast(MI)); -+ } -+ - // Start expanding the macro. - EnterMacro(Identifier, ExpansionEnd, MI, Args); - return false; -@@ -765,13 +854,13 @@ static bool GenerateNewArgTokens(Preprocessor &PP, - /// error. - MacroArgs *Preprocessor::ReadMacroCallArgumentList(Token &MacroName, - MacroInfo *MI, -- SourceLocation &MacroEnd) { -+ Token &Tok) { // PASTA PATCH - // The number of fixed arguments to parse. - unsigned NumFixedArgsLeft = MI->getNumParams(); - bool isVariadic = MI->isVariadic(); - - // Outer loop, while there are more arguments, keep reading them. -- Token Tok; -+ // Token Tok; // PASTA PATCH: Taken from argument. - - // Read arguments as unexpanded tokens. This avoids issues, e.g., where - // an argument value in a macro could expand to ',' or '(' or ')'. -@@ -784,6 +873,9 @@ MacroArgs *Preprocessor::ReadMacroCallArgumentList(Token &MacroName, - SmallVector ArgTokens; - bool ContainsCodeCompletionTok = false; - bool FoundElidedComma = false; -+ bool InVariadicSection = false; // PASTA PATCH -+ bool InArgument = false; // PASTA PATCH -+ Token ArgStartTok; // PASTA PATCH - - SourceLocation TooManyArgsLoc; - -@@ -798,6 +890,21 @@ MacroArgs *Preprocessor::ReadMacroCallArgumentList(Token &MacroName, - size_t ArgTokenStart = ArgTokens.size(); - SourceLocation ArgStartLoc = Tok.getLocation(); - -+ // PASTA PATCH: Visibility into macro expansion. -+ ArgStartTok = Tok; -+ if (Callbacks) { -+ if (isVariadic && !NumFixedArgsLeft) { -+ Callbacks->Event(ArgStartTok, PPCallbacks::BeginVariadicCallArgumentList, -+ reinterpret_cast(&MacroName)); -+ InVariadicSection = true; -+ } -+ if (NumFixedArgsLeft) { -+ Callbacks->Event(ArgStartTok, PPCallbacks::BeginMacroCallArgument, -+ reinterpret_cast(&MacroName)); -+ InArgument = true; -+ } -+ } -+ - // C99 6.10.3p11: Keep track of the number of l_parens we have seen. Note - // that we already consumed the first one. - unsigned NumParens = 0; -@@ -824,7 +931,7 @@ MacroArgs *Preprocessor::ReadMacroCallArgumentList(Token &MacroName, - } else if (Tok.is(tok::r_paren)) { - // If we found the ) token, the macro arg list is done. - if (NumParens-- == 0) { -- MacroEnd = Tok.getLocation(); -+ // MacroEnd = Tok.getLocation(); // PASTA PATCH - if (!ArgTokens.empty() && - ArgTokens.back().commaAfterElided()) { - FoundElidedComma = true; -@@ -852,6 +959,27 @@ MacroArgs *Preprocessor::ReadMacroCallArgumentList(Token &MacroName, - break; - if (NumFixedArgsLeft > 1) - break; -+ -+ // PASTA PATCH: Visibility into variadic macro expansion. -+ if (Callbacks) { -+ if (InArgument) { -+ Callbacks->Event(ArgStartTok, PPCallbacks::EndMacroCallArgument, -+ reinterpret_cast(&Tok)); -+ InArgument = false; -+ } -+ ArgStartTok = Tok; -+ -+ if (!InVariadicSection) { -+ Callbacks->Event( -+ ArgStartTok, PPCallbacks::BeginVariadicCallArgumentList, -+ reinterpret_cast(&MacroName)); -+ InVariadicSection = true; -+ } -+ -+ Callbacks->Event(ArgStartTok, PPCallbacks::BeginMacroCallArgument, -+ reinterpret_cast(&MacroName)); -+ InArgument = true; -+ } - } - } else if (Tok.is(tok::comment) && !KeepMacroComments) { - // If this is a comment token in the argument list and we're just in -@@ -901,6 +1029,15 @@ MacroArgs *Preprocessor::ReadMacroCallArgumentList(Token &MacroName, - ? diag::warn_cxx98_compat_empty_fnmacro_arg - : diag::ext_empty_fnmacro_arg); - -+ // PASTA PATCH: -+ if (Callbacks) { -+ if (InArgument) { -+ InArgument = false; -+ Callbacks->Event(ArgStartTok, PPCallbacks::EndMacroCallArgument, -+ reinterpret_cast(&Tok)); -+ } -+ } -+ - // Add a marker EOF token to the end of the token list for this argument. - Token EOFTok; - EOFTok.startToken(); -@@ -913,6 +1050,18 @@ MacroArgs *Preprocessor::ReadMacroCallArgumentList(Token &MacroName, - --NumFixedArgsLeft; - } - -+ if (Callbacks) { -+ if (InArgument) { -+ InArgument = false; -+ Callbacks->Event(ArgStartTok, PPCallbacks::EndMacroCallArgument, -+ reinterpret_cast(&Tok)); -+ } -+ if (InVariadicSection) { -+ Callbacks->Event(ArgStartTok, PPCallbacks::EndVariadicCallArgumentList, -+ reinterpret_cast(&Tok)); -+ } -+ } -+ - // Okay, we either found the r_paren. Check to see if we parsed too few - // arguments. - unsigned MinArgsExpected = MI->getNumParams(); -@@ -1167,6 +1316,17 @@ static bool EvaluateHasIncludeCommon(Token &Tok, IdentifierInfo *II, - Preprocessor &PP, - ConstSearchDirIterator LookupFrom, - const FileEntry *LookupFromFile) { -+ // PASTA PATCH: Visibility into macro expansion. -+ PPCallbacks *Callbacks = PP.getPPCallbacks(); -+ Token SavedIdentifier = Tok; -+ SavedIdentifier.setIdentifierInfo(II); -+ MacroDefinition MD = PP.getMacroDefinition(II); -+ MacroInfo *MI = MD.getMacroInfo(); -+ if (Callbacks) { -+ Callbacks->Event(SavedIdentifier, PPCallbacks::BeginMacroCallArgumentList, -+ reinterpret_cast(MI)); -+ } -+ - // Save the location of the current token. If a '(' is later found, use - // that location. If not, use the end of this location instead. - SourceLocation LParenLoc = Tok.getLocation(); -@@ -1186,6 +1346,8 @@ static bool EvaluateHasIncludeCommon(Token &Tok, IdentifierInfo *II, - return false; - } while (Tok.getKind() == tok::comment); - -+ Token LParenTok = Tok; // PASTA PATCH -+ - // Ensure we have a '('. - if (Tok.isNot(tok::l_paren)) { - // No '(', use end of last token. -@@ -1196,6 +1358,10 @@ static bool EvaluateHasIncludeCommon(Token &Tok, IdentifierInfo *II, - if (Tok.isNot(tok::header_name)) - return false; - } else { -+ if (Callbacks) { // PASTA PATCH -+ Callbacks->Event(LParenTok, PPCallbacks::BeginMacroCallArgument, -+ reinterpret_cast(&SavedIdentifier)); -+ } - // Save '(' location for possible missing ')' message. - LParenLoc = Tok.getLocation(); - if (PP.LexHeaderName(Tok)) -@@ -1227,6 +1393,16 @@ static bool EvaluateHasIncludeCommon(Token &Tok, IdentifierInfo *II, - return false; - } - -+ // PASTA PATCH -+ if (Callbacks) { -+ Callbacks->Event(LParenTok, PPCallbacks::EndMacroCallArgument, -+ reinterpret_cast(&Tok)); -+ Callbacks->Event(LParenTok, PPCallbacks::EndMacroCallArgumentList, -+ 0); -+ Callbacks->Event(SavedIdentifier, PPCallbacks::SwitchToExpansion, -+ reinterpret_cast(MI)); -+ } -+ - bool isAngled = PP.GetIncludeFilenameSpelling(Tok.getLocation(), Filename); - // If GetIncludeFilenameSpelling set the start ptr to null, there was an - // error. -@@ -1270,6 +1446,13 @@ static void EvaluateFeatureLikeBuiltinMacro(llvm::raw_svector_ostream& OS, - llvm::function_ref< - int(Token &Tok, - bool &HasLexedNextTok)> Op) { -+ // PASTA PATCH: Visibility into macro expansion. -+ PPCallbacks *Callbacks = PP.getPPCallbacks(); -+ Token SavedIdentifier = Tok; -+ SavedIdentifier.setIdentifierInfo(II); -+ MacroDefinition MD = PP.getMacroDefinition(II); -+ MacroInfo *MI = MD.getMacroInfo(); -+ - // Parse the initial '('. - PP.LexUnexpandedToken(Tok); - if (Tok.isNot(tok::l_paren)) { -@@ -1288,6 +1471,15 @@ static void EvaluateFeatureLikeBuiltinMacro(llvm::raw_svector_ostream& OS, - SourceLocation LParenLoc = Tok.getLocation(); - llvm::Optional Result; - -+ // PASTA PATCH -+ clang::Token ArgSepTok = Tok; -+ if (Callbacks) { -+ Callbacks->Event(SavedIdentifier, PPCallbacks::BeginMacroCallArgumentList, -+ reinterpret_cast(MI)); -+ Callbacks->Event(ArgSepTok, PPCallbacks::BeginMacroCallArgument, -+ reinterpret_cast(&SavedIdentifier)); -+ } -+ - Token ResultTok; - bool SuppressDiagnostic = false; - while (true) { -@@ -1307,6 +1499,15 @@ already_lexed: - return; - - case tok::comma: -+ // PASTA PATCH -+ if (ParenDepth == 1 && Callbacks) { -+ Callbacks->Event(ArgSepTok, PPCallbacks::EndMacroCallArgument, -+ reinterpret_cast(&Tok)); -+ ArgSepTok = Tok; -+ Callbacks->Event(ArgSepTok, PPCallbacks::BeginMacroCallArgument, -+ reinterpret_cast(&SavedIdentifier)); -+ } -+ - if (!SuppressDiagnostic) { - PP.Diag(Tok.getLocation(), diag::err_too_many_args_in_macro_invoc); - SuppressDiagnostic = true; -@@ -1327,6 +1528,15 @@ already_lexed: - if (--ParenDepth > 0) - continue; - -+ // PASTA PATCH -+ if (Callbacks) { -+ Callbacks->Event(ArgSepTok, PPCallbacks::EndMacroCallArgument, -+ reinterpret_cast(&Tok)); -+ Callbacks->Event(ArgSepTok, PPCallbacks::EndMacroCallArgumentList, 0); -+ Callbacks->Event(SavedIdentifier, PPCallbacks::SwitchToExpansion, -+ reinterpret_cast(MI)); -+ } -+ - // The last ')' has been reached; return the value if one found or - // a diagnostic and a dummy value. - if (Result) { -@@ -1493,7 +1703,22 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) { - bool IsAtStartOfLine = Tok.isAtStartOfLine(); - bool HasLeadingSpace = Tok.hasLeadingSpace(); - -+ // PASTA PATCH: -+ Token SavedIdentifier = Tok; -+ SavedIdentifier.setIdentifierInfo(II); -+ MacroDefinition MD = getMacroDefinition(II); -+ MacroInfo *MI = MD.getMacroInfo(); -+ DeferExpansionEnd NotifyMacroEnd(PostLexAction, Callbacks.get(), -+ SavedIdentifier, MI); -+ auto SwitchToExpansion = [&] (void) { -+ if (Callbacks) { -+ Callbacks->Event(SavedIdentifier, PPCallbacks::SwitchToExpansion, -+ reinterpret_cast(MI)); -+ } -+ }; -+ - if (II == Ident__LINE__) { -+ SwitchToExpansion(); // PASTA PATCH - // C99 6.10.8: "__LINE__: The presumed line number (within the current - // source file) of the current source line (an integer constant)". This can - // be affected by #line. -@@ -1516,6 +1741,7 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) { - Tok.setKind(tok::numeric_constant); - } else if (II == Ident__FILE__ || II == Ident__BASE_FILE__ || - II == Ident__FILE_NAME__) { -+ SwitchToExpansion(); // PASTA PATCH - // C99 6.10.8: "__FILE__: The presumed name of the current source file (a - // character string literal)". This can be affected by #line. - PresumedLoc PLoc = SourceMgr.getPresumedLoc(Tok.getLocation()); -@@ -1555,6 +1781,7 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) { - } - Tok.setKind(tok::string_literal); - } else if (II == Ident__DATE__) { -+ SwitchToExpansion(); // PASTA PATCH - Diag(Tok.getLocation(), diag::warn_pp_date_time); - if (!DATELoc.isValid()) - ComputeDATE_TIME(DATELoc, TIMELoc, *this); -@@ -1565,6 +1792,7 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) { - Tok.getLength())); - return; - } else if (II == Ident__TIME__) { -+ SwitchToExpansion(); // PASTA PATCH - Diag(Tok.getLocation(), diag::warn_pp_date_time); - if (!TIMELoc.isValid()) - ComputeDATE_TIME(DATELoc, TIMELoc, *this); -@@ -1575,6 +1803,7 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) { - Tok.getLength())); - return; - } else if (II == Ident__INCLUDE_LEVEL__) { -+ SwitchToExpansion(); // PASTA PATCH - // Compute the presumed include depth of this token. This can be affected - // by GNU line markers. - unsigned Depth = 0; -@@ -1590,6 +1819,7 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) { - OS << Depth; - Tok.setKind(tok::numeric_constant); - } else if (II == Ident__TIMESTAMP__) { -+ SwitchToExpansion(); // PASTA PATCH - Diag(Tok.getLocation(), diag::warn_pp_date_time); - // MSVC, ICC, GCC, VisualAge C++ extension. The generated string should be - // of the form "Ddd Mmm dd hh::mm::ss yyyy", which is returned by asctime. -@@ -1614,6 +1844,7 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) { - OS << '"' << StringRef(Result).drop_back() << '"'; - Tok.setKind(tok::string_literal); - } else if (II == Ident__FLT_EVAL_METHOD__) { -+ SwitchToExpansion(); // PASTA PATCH - // __FLT_EVAL_METHOD__ is set to the default value. - if (getTUFPEvalMethod() == - LangOptions::FPEvalMethodKind::FEM_Indeterminable) { -@@ -1646,6 +1877,7 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) { - } - } - } else if (II == Ident__COUNTER__) { -+ SwitchToExpansion(); // PASTA PATCH - // __COUNTER__ expands to a simple numeric value. - OS << CounterValue++; - Tok.setKind(tok::numeric_constant); -@@ -1834,6 +2066,7 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) { - (II->getName() == getLangOpts().CurrentModule); - }); - } else if (II == Ident__MODULE__) { -+ SwitchToExpansion(); // PASTA PATCH - // The current module as an identifier. - OS << getLangOpts().CurrentModule; - IdentifierInfo *ModuleII = getIdentifierInfo(getLangOpts().CurrentModule); -@@ -1842,6 +2075,12 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) { - } else if (II == Ident__identifier) { - SourceLocation Loc = Tok.getLocation(); - -+ // PASTA PATCH -+ if (Callbacks) { -+ Callbacks->Event(SavedIdentifier, PPCallbacks::BeginMacroCallArgumentList, -+ 0); -+ } -+ - // We're expecting '__identifier' '(' identifier ')'. Try to recover - // if the parens are missing. - LexNonComment(Tok); -@@ -1855,6 +2094,13 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) { - return; - } - -+ // PASTA PATCH -+ Token LParenTok = Tok; -+ if (Callbacks) { -+ Callbacks->Event(LParenTok, PPCallbacks::BeginMacroCallArgument, -+ reinterpret_cast(&SavedIdentifier)); -+ } -+ - SourceLocation LParenLoc = Tok.getLocation(); - LexNonComment(Tok); - -@@ -1883,6 +2129,15 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) { - << Tok.getKind() << tok::r_paren; - Diag(LParenLoc, diag::note_matching) << tok::l_paren; - } -+ -+ // PASTA PATCH -+ if (Callbacks) { -+ Callbacks->Event(LParenTok, PPCallbacks::EndMacroCallArgument, -+ reinterpret_cast(&Tok)); -+ Callbacks->Event(Tok, PPCallbacks::EndMacroCallArgumentList, 0); -+ } -+ SwitchToExpansion(); // PASTA PATCH -+ - return; - } else if (II == Ident__is_target_arch) { - EvaluateFeatureLikeBuiltinMacro( -diff --git a/clang/lib/Lex/Pragma.cpp b/clang/lib/Lex/Pragma.cpp -index fb4f2dc45..320b44466 100644 ---- a/clang/lib/Lex/Pragma.cpp -+++ b/clang/lib/Lex/Pragma.cpp -@@ -206,10 +206,27 @@ void Preprocessor::Handle_Pragma(Token &Tok) { - // In Case #2, we check the syntax now, but then put the tokens back into the - // token stream for later consumption. - -+ // PASTA PATCH -+ // If we're expanding a macro argument, let the checking and subsequent lexing -+ // happen, but tell the listener that we're going to be cancelling this. -+ Token SavedIdentifier = Tok; -+ MacroDefinition MD = getMacroDefinition(Tok.getIdentifierInfo()); -+ MacroInfo *MI = MD.getMacroInfo(); -+ if (Callbacks) { -+ if (InMacroArgPreExpansion) { -+ Callbacks->Event(SavedIdentifier, PPCallbacks::PrepareToCancelExpansion, -+ reinterpret_cast(MI)); -+ } else { -+ Callbacks->Event(SavedIdentifier, PPCallbacks::BeginMacroCallArgumentList, -+ reinterpret_cast(MI)); -+ } -+ } -+ - TokenCollector Toks = {*this, InMacroArgPreExpansion, {}, Tok}; - - // Remember the pragma token location. - SourceLocation PragmaLoc = Tok.getLocation(); -+ SourceLocation PragmaEndLoc = Tok.getEndLoc(); // PASTA PATCH - - // Read the '('. - Toks.lex(); -@@ -218,6 +235,13 @@ void Preprocessor::Handle_Pragma(Token &Tok) { - return; - } - -+ // PASTA PATCH -+ Token LParenTok = Tok; -+ if (Callbacks && !InMacroArgPreExpansion) { -+ Callbacks->Event(LParenTok, PPCallbacks::BeginMacroCallArgument, -+ reinterpret_cast(&SavedIdentifier)); -+ } -+ - // Read the '"..."'. - Toks.lex(); - if (!tok::isStringLiteral(Tok.getKind())) { -@@ -253,8 +277,43 @@ void Preprocessor::Handle_Pragma(Token &Tok) { - return; - } - -+ // PASTA PATCH: Go and simulate the directive events that would have happened -+ // if this had been an actual `#pragma` directive. -+ Token HashTok; -+ Token PragmaDirectiveTok; -+ if (Callbacks && !InMacroArgPreExpansion) { -+ Callbacks->Event(LParenTok, PPCallbacks::EndMacroCallArgument, -+ reinterpret_cast(&Tok)); -+ Callbacks->Event(LParenTok, PPCallbacks::EndMacroCallArgumentList, 0); -+ Callbacks->Event(SavedIdentifier, PPCallbacks::SwitchToExpansion, -+ reinterpret_cast(MI)); -+ -+ HashTok.startToken(); -+ HashTok.setKind(tok::hash); -+ CreateString("#", HashTok, PragmaLoc, -+ PragmaLoc.getLocWithOffset(1)); -+ HashTok.setLength(1u); -+ Callbacks->Event(HashTok, PPCallbacks::BeginDirective, 0); -+ -+ PragmaDirectiveTok.startToken(); -+ PragmaDirectiveTok.setKind(tok::raw_identifier); -+ CreateString("pragma", PragmaDirectiveTok, PragmaLoc.getLocWithOffset(1), -+ PragmaEndLoc); -+ Callbacks->Event(PragmaDirectiveTok, PPCallbacks::TokenFromLexer, 0); -+ Callbacks->Event(HashTok, PPCallbacks::SetNamedDirective, -+ reinterpret_cast(&PragmaDirectiveTok)); -+ } -+ - // If we're expanding a macro argument, put the tokens back. - if (InMacroArgPreExpansion) { -+ if (Callbacks) { // PASTA PATCH -+// Token EOD; -+// EOD.startToken(); -+// EOD.setKind(tok::eod); -+// Callbacks->Event(EOD, PPCallbacks::EndNonDirective, 0); -+ Callbacks->Event(SavedIdentifier, PPCallbacks::CancelExpansion, -+ reinterpret_cast(MI)); -+ } - Toks.revert(); - return; - } -diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp -index 5310db3c8..31f31736a 100644 ---- a/clang/lib/Lex/Preprocessor.cpp -+++ b/clang/lib/Lex/Preprocessor.cpp -@@ -92,6 +92,7 @@ Preprocessor::Preprocessor(std::shared_ptr PPOpts, - Identifiers(IILookup), PragmaHandlers(new PragmaNamespace(StringRef())), - TUKind(TUKind), SkipMainFilePreamble(0, true), - CurSubmoduleState(&NullSubmoduleState) { -+ PostLexAction = [] (const Token &) {}; // PASTA PATCH - OwnsHeaderSearch = OwnsHeaders; - - // Default to discarding comments. -@@ -686,9 +687,17 @@ void Preprocessor::replayPreambleConditionalStack() { - "CurPPLexer is null when calling replayPreambleConditionalStack."); - CurPPLexer->setConditionalLevels(PreambleConditionalStack.getStack()); - PreambleConditionalStack.doneReplaying(); -+ -+ // PASTA PATCH -+ Token HashToken; -+ HashToken.startToken(); -+ HashToken.setKind(tok::hash); -+ HashToken.setLocation(PreambleConditionalStack.SkipInfo->HashTokenLoc); -+ HashToken.setLength(1u); -+ - if (PreambleConditionalStack.reachedEOFWhileSkipping()) - SkipExcludedConditionalBlock( -- PreambleConditionalStack.SkipInfo->HashTokenLoc, -+ HashToken, // PASTA PATCH - PreambleConditionalStack.SkipInfo->IfTokenLoc, - PreambleConditionalStack.SkipInfo->FoundNonSkipPortion, - PreambleConditionalStack.SkipInfo->FoundElse, -@@ -904,27 +913,59 @@ bool Preprocessor::HandleIdentifier(Token &Identifier) { - void Preprocessor::Lex(Token &Result) { - ++LexLevel; - -+ auto InputRawLoc = Result.getLocation().getRawEncoding(); // PASTA PATCH -+ - // We loop here until a lex function returns a token; this avoids recursion. - bool ReturnedToken; - do { - switch (CurLexerKind) { - case CLK_Lexer: - ReturnedToken = CurLexer->Lex(Result); -+ -+ // PASTA PATCH: Visibility into all tokens. -+ if (ReturnedToken && Callbacks) { -+ Callbacks->Event(Result, PPCallbacks::TokenFromLexer, InputRawLoc); -+ } - break; - case CLK_TokenLexer: - ReturnedToken = CurTokenLexer->Lex(Result); -+ -+ // PASTA PATCH: Visibility into all tokens. -+ if (ReturnedToken && Callbacks) { -+ Callbacks->Event(Result, PPCallbacks::TokenFromTokenLexer, InputRawLoc); -+ } - break; - case CLK_CachingLexer: - CachingLex(Result); - ReturnedToken = true; -+ -+ // PASTA PATCH: Visibility into all tokens. -+ if (ReturnedToken && Callbacks) { -+ Callbacks->Event(Result, PPCallbacks::TokenFromCachingLexer, -+ InputRawLoc); -+ } - break; - case CLK_DependencyDirectivesLexer: - ReturnedToken = CurLexer->LexDependencyDirectiveToken(Result); - break; - case CLK_LexAfterModuleImport: - ReturnedToken = LexAfterModuleImport(Result); -+ -+ // PASTA PATCH: Visibility into all tokens. -+ if (ReturnedToken && Callbacks) { -+ Callbacks->Event(Result, PPCallbacks::TokenFromAfterModuleImportLexer, -+ InputRawLoc); -+ } - break; - } -+ -+ // PASTA PATCH -+ if (ReturnedToken && Callbacks && Result.is(tok::eod)) { -+ Callbacks->Event(Result, PPCallbacks::EndDirective, 0); -+ } -+ PostLexAction(Result); -+ PostLexAction = [] (const Token &) {}; -+ - } while (!ReturnedToken); - - if (Result.is(tok::unknown) && TheModuleLoader.HadFatalFailure) -diff --git a/clang/lib/Lex/TokenLexer.cpp b/clang/lib/Lex/TokenLexer.cpp -index efda6d004..4e8c9b78d 100644 ---- a/clang/lib/Lex/TokenLexer.cpp -+++ b/clang/lib/Lex/TokenLexer.cpp -@@ -41,6 +41,8 @@ void TokenLexer::Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI, - // associated with it. - destroy(); - -+ MacroNameTok = Tok; // PASTA PATCH -+ - Macro = MI; - ActualArgs = Actuals; - CurTokenIdx = 0; -@@ -80,8 +82,33 @@ void TokenLexer::Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI, - - // If this is a function-like macro, expand the arguments and change - // Tokens to point to the expanded tokens. -- if (Macro->isFunctionLike() && Macro->getNumParams()) -+ if (Macro->isFunctionLike() && Macro->getNumParams()) { // PASTA PATCH -+ auto Callbacks = PP.getPPCallbacks(); -+ if (ActualArgs && Callbacks) { -+ unsigned NumArgs = ActualArgs->getNumMacroArguments(); -+ if (NumArgs && Macro->isVariadic() && ActualArgs->isVarargsElidedUse()) { -+ NumArgs -= 1u; // Last argument is `VariadicArgIndex`. -+ } -+ -+ // In theory we only need to pre-expand what needs pre-expansion. In -+ // practice, Clang goes and sometimes requests pre-expansion for the -+ // sake of figuring out __VA_OPT__ stuff, via the -+ // `MacroArgs::invokedWithVariadicArgument` function. -+ Callbacks->Event(MacroNameTok, PPCallbacks::BeginPreArgumentExpansion, -+ reinterpret_cast(Macro)); -+ -+ // Pre expand each argument, which internally caches the results. -+ for (unsigned ArgNo = 0; ArgNo < NumArgs; ++ArgNo) { -+ Callbacks->Event(MacroNameTok, PPCallbacks::BeginMacroCallArgument, 0); -+ (void) ActualArgs->getPreExpArgument(ArgNo, PP); -+ Callbacks->Event(MacroNameTok, PPCallbacks::EndMacroCallArgument, 0); -+ } -+ -+ Callbacks->Event(MacroNameTok, PPCallbacks::EndPreArgumentExpansion, -+ reinterpret_cast(Macro)); -+ } - ExpandFunctionArguments(); -+ } // PASTA PATCH - - // Mark the macro as currently disabled, so that it is not recursively - // expanded. The macro must be disabled only after argument pre-expansion of -@@ -251,6 +278,8 @@ void TokenLexer::ExpandFunctionArguments() { - - VAOptExpansionContext VCtx(PP); - -+ auto Callbacks = PP.getPPCallbacks(); // PASTA PATCH -+ - for (unsigned I = 0, E = NumTokens; I != E; ++I) { - const Token &CurTok = Tokens[I]; - // We don't want a space for the next token after a paste -@@ -451,7 +480,7 @@ void TokenLexer::ExpandFunctionArguments() { - // Only preexpand the argument if it could possibly need it. This - // avoids some work in common cases. - const Token *ArgTok = ActualArgs->getUnexpArgument(ArgNo); -- if (ActualArgs->ArgNeedsPreexpansion(ArgTok, PP)) -+ if (Callbacks || ActualArgs->ArgNeedsPreexpansion(ArgTok, PP)) // PASTA PATCH - ResultArgToks = &ActualArgs->getPreExpArgument(ArgNo, PP)[0]; - else - ResultArgToks = ArgTok; // Use non-preexpanded tokens. -@@ -625,6 +654,12 @@ bool TokenLexer::Lex(Token &Tok) { - // that it is no longer being expanded. - if (Macro) Macro->EnableMacro(); - -+// // PASTA PATCH -+// if (Macro && PP.Callbacks) { -+// PP.Callbacks->Event(MacroNameTok, PPCallbacks::EndMacroExpansion, -+// reinterpret_cast(Macro)); -+// } -+ - Tok.startToken(); - Tok.setFlagValue(Token::StartOfLine , AtStartOfLine); - Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace || NextTokGetsSpace); -diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp -index a6a946d7f..72dc2a006 100644 ---- a/clang/lib/Parse/ParseExpr.cpp -+++ b/clang/lib/Parse/ParseExpr.cpp -@@ -1420,6 +1420,13 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind, - case tok::kw_sizeof: // unary-expression: 'sizeof' unary-expression - // unary-expression: 'sizeof' '(' type-name ')' - case tok::kw_vec_step: // unary-expression: OpenCL 'vec_step' expression -+ -+ // PASTA PATCH: XNU-specific UETTs. -+ case tok::kw___builtin_ptrauth_type_discriminator: -+ case tok::kw___builtin_xnu_type_signature: -+ case tok::kw___builtin_xnu_type_summary: -+ case tok::kw___builtin_tmo_type_get_metadata: -+ - // unary-expression: '__builtin_omp_required_simd_align' '(' type-name ')' - case tok::kw___builtin_omp_required_simd_align: - if (NotPrimaryExpression) -@@ -2284,7 +2291,13 @@ Parser::ParseExprAfterUnaryExprOrTypeTrait(const Token &OpTok, - - assert(OpTok.isOneOf(tok::kw_typeof, tok::kw_sizeof, tok::kw___alignof, - tok::kw_alignof, tok::kw__Alignof, tok::kw_vec_step, -- tok::kw___builtin_omp_required_simd_align) && -+ tok::kw___builtin_omp_required_simd_align, -+ -+ // PASTA PATCH: XNU-specific UETTs. -+ tok::kw___builtin_ptrauth_type_discriminator, -+ tok::kw___builtin_xnu_type_signature, -+ tok::kw___builtin_xnu_type_summary, -+ tok::kw___builtin_tmo_type_get_metadata) && - "Not a typeof/sizeof/alignof/vec_step expression!"); - - ExprResult Operand; -@@ -2403,7 +2416,13 @@ ExprResult Parser::ParseSYCLUniqueStableNameExpression() { - ExprResult Parser::ParseUnaryExprOrTypeTraitExpression() { - assert(Tok.isOneOf(tok::kw_sizeof, tok::kw___alignof, tok::kw_alignof, - tok::kw__Alignof, tok::kw_vec_step, -- tok::kw___builtin_omp_required_simd_align) && -+ tok::kw___builtin_omp_required_simd_align, -+ -+ // PASTA PATCH: XNU-specific UETTs. -+ tok::kw___builtin_ptrauth_type_discriminator, -+ tok::kw___builtin_xnu_type_signature, -+ tok::kw___builtin_xnu_type_summary, -+ tok::kw___builtin_tmo_type_get_metadata) && - "Not a sizeof/alignof/vec_step expression!"); - Token OpTok = Tok; - ConsumeToken(); -@@ -2480,6 +2499,16 @@ ExprResult Parser::ParseUnaryExprOrTypeTraitExpression() { - else if (OpTok.is(tok::kw___builtin_omp_required_simd_align)) - ExprKind = UETT_OpenMPRequiredSimdAlign; - -+ // PASTA PATCH: XNU-specific things. -+ else if (OpTok.is(tok::kw___builtin_ptrauth_type_discriminator)) -+ ExprKind = UETT_PtrAuthTypeDiscriminator; -+ else if (OpTok.is(tok::kw___builtin_xnu_type_signature)) -+ ExprKind = UETT_XNUTypeSignature; -+ else if (OpTok.is(tok::kw___builtin_xnu_type_summary)) -+ ExprKind = UETT_XNUTypeSummary; -+ else if (OpTok.is(tok::kw___builtin_tmo_type_get_metadata)) -+ ExprKind = UETT_TMOTypeGetMetadata; -+ - if (isCastExpr) - return Actions.ActOnUnaryExprOrTypeTraitExpr(OpTok.getLocation(), - ExprKind, -diff --git a/clang/lib/Parse/ParseTemplate.cpp b/clang/lib/Parse/ParseTemplate.cpp -index e32ea6003..942c71526 100644 ---- a/clang/lib/Parse/ParseTemplate.cpp -+++ b/clang/lib/Parse/ParseTemplate.cpp -@@ -1209,6 +1209,13 @@ bool Parser::ParseGreaterThanInTemplateList(SourceLocation LAngleLoc, - AfterGreaterLoc = PP.SplitToken(AfterGreaterLoc, Tok.getLength()); - Tok.setLocation(AfterGreaterLoc); - -+ // PASTA PATCH: We want to observe when tokens are split up so that we can -+ // have this reflected in PASTA's token lists. -+ if (PPCallbacks *Callbacks = PP.getPPCallbacks()) { -+ Callbacks->Event(Greater, PPCallbacks::BeginSplitToken, 0); -+ Callbacks->Event(Tok, PPCallbacks::EndSplitToken, 0); -+ } -+ - // Update the token cache to match what we just did if necessary. - if (CachingTokens) { - // If the previous cached token is being merged, delete it. -diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp -index 75ffa8b0a..27cea9cdf 100644 ---- a/clang/lib/Sema/SemaDecl.cpp -+++ b/clang/lib/Sema/SemaDecl.cpp -@@ -6261,6 +6261,11 @@ NamedDecl *Sema::HandleDeclarator(Scope *S, Declarator &D, - Previous.setRedeclarationKind(ForExternalRedeclaration); - } - -+ // PASTA PATCH: Disable auto-creation of builtins from things like `atoi`; -+ // it ends up leading to a lot of source location information -+ // being dropped. -+ CreateBuiltins = CreateBuiltins ? false : false; -+ - LookupName(Previous, S, CreateBuiltins); - } else { // Something like "int foo::x;" - LookupQualifiedName(Previous, DC); -@@ -6455,6 +6460,13 @@ FixInvalidVariablyModifiedTypeLoc(TypeLoc SrcTL, TypeLoc DstTL) { - DstPTL.setRParenLoc(SrcPTL.getRParenLoc()); - return; - } -+ -+ // PASTA PATCH: Might have an AdjustedTypeLoc for DstTL due to multiplier -+ // patches for issue #130. -+ if (AdjustedTypeLoc AdjTL = DstTL.getAs()) { -+ DstTL = AdjTL.getOriginalLoc(); -+ } -+ - ArrayTypeLoc SrcATL = SrcTL.castAs(); - ArrayTypeLoc DstATL = DstTL.castAs(); - TypeLoc SrcElemTL = SrcATL.getElementLoc(); -diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp -index 83081bbf0..c9c19b9d8 100644 ---- a/clang/lib/Sema/SemaExpr.cpp -+++ b/clang/lib/Sema/SemaExpr.cpp -@@ -4264,6 +4264,15 @@ bool Sema::CheckUnaryExprOrTypeTraitOperand(Expr *E, - bool IsUnevaluatedOperand = - (ExprKind == UETT_SizeOf || ExprKind == UETT_AlignOf || - ExprKind == UETT_PreferredAlignOf || ExprKind == UETT_VecStep); -+ -+ // PASTA PATCH: Add in XNU-specific unary expr/type traits. -+ IsUnevaluatedOperand = -+ IsUnevaluatedOperand || -+ ExprKind == UETT_PtrAuthTypeDiscriminator || -+ ExprKind == UETT_XNUTypeSignature || -+ ExprKind == UETT_XNUTypeSummary || -+ ExprKind == UETT_TMOTypeGetMetadata; -+ - if (IsUnevaluatedOperand) { - ExprResult Result = CheckUnevaluatedOperand(E); - if (Result.isInvalid()) -@@ -4632,8 +4641,14 @@ Sema::CreateUnaryExprOrTypeTraitExpr(TypeSourceInfo *TInfo, - TInfo->getType()->isVariablyModifiedType()) - TInfo = TransformToPotentiallyEvaluated(TInfo); - -+ // PASTA PATCH: This returns a string literal that is a type signature. -+ QualType RetTy = Context.getSizeType(); -+ if (ExprKind == UETT_XNUTypeSignature) { -+ RetTy = Context.getStringLiteralArrayType(Context.CharTy, 0u); -+ } -+ - return new (Context) UnaryExprOrTypeTraitExpr( -- ExprKind, TInfo, Context.getSizeType(), OpLoc, R.getEnd()); -+ ExprKind, TInfo, RetTy, OpLoc, R.getEnd()); - } - - /// Build a sizeof or alignof expression given an expression -@@ -4647,6 +4662,9 @@ Sema::CreateUnaryExprOrTypeTraitExpr(Expr *E, SourceLocation OpLoc, - - E = PE.get(); - -+ // PASTA PATCH: We want a string literal array return type for the signature. -+ QualType RetTy = Context.getSizeType(); -+ - // Verify that the operand is valid. - bool isInvalid = false; - if (E->isTypeDependent()) { -@@ -4661,6 +4679,16 @@ Sema::CreateUnaryExprOrTypeTraitExpr(Expr *E, SourceLocation OpLoc, - } else if (E->refersToBitField()) { // C99 6.5.3.4p1. - Diag(E->getExprLoc(), diag::err_sizeof_alignof_typeof_bitfield) << 0; - isInvalid = true; -+ -+ // PASTA PATCH: Let these through. -+ } else if (ExprKind == UETT_PtrAuthTypeDiscriminator || -+ ExprKind == UETT_XNUTypeSummary || -+ ExprKind == UETT_TMOTypeGetMetadata) { -+ -+ // PASTA PATCH: This returns a string literal that is a type signature. -+ } else if (ExprKind == UETT_XNUTypeSignature) { -+ RetTy = Context.getStringLiteralArrayType(Context.CharTy, 0u); -+ - } else { - isInvalid = CheckUnaryExprOrTypeTraitOperand(E, UETT_SizeOf); - } -@@ -4676,7 +4704,7 @@ Sema::CreateUnaryExprOrTypeTraitExpr(Expr *E, SourceLocation OpLoc, - - // C99 6.5.3.4p4: the type (an unsigned integer type) is size_t. - return new (Context) UnaryExprOrTypeTraitExpr( -- ExprKind, E, Context.getSizeType(), OpLoc, E->getSourceRange().getEnd()); -+ ExprKind, E, RetTy, OpLoc, E->getSourceRange().getEnd()); - } - - /// ActOnUnaryExprOrTypeTraitExpr - Handle @c sizeof(type) and @c sizeof @c -diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp -index 6f9e02528..431dd4ac1 100644 ---- a/clang/lib/Sema/SemaExprCXX.cpp -+++ b/clang/lib/Sema/SemaExprCXX.cpp -@@ -5532,6 +5532,7 @@ static bool EvaluateBinaryTypeTrait(Sema &Self, TypeTrait BTT, QualType LhsT, - } - case BTT_IsSame: - return Self.Context.hasSameType(LhsT, RhsT); -+ case BTT_XNUTypeCompatible: // PASTA PATCH - case BTT_TypeCompatible: { - // GCC ignores cv-qualifiers on arrays for this builtin. - Qualifiers LhsQuals, RhsQuals; -diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h -index a8589191f..8047b0044 100644 ---- a/clang/lib/Sema/TreeTransform.h -+++ b/clang/lib/Sema/TreeTransform.h -@@ -5212,6 +5212,12 @@ TreeTransform::TransformConstantArrayType(TypeLocBuilder &TLB, - return QualType(); - } - -+ // PASTA PATCH: We might be dealing with an adjusted type due to Multiplier -+ // issue #130. -+ if (auto AT = dyn_cast(Result.getTypePtr())) { -+ Result = AT->getAdjustedType(); -+ } -+ - // We might have either a ConstantArrayType or a VariableArrayType now: - // a ConstantArrayType is allowed to have an element type which is a - // VariableArrayType if the type is dependent. Fortunately, all array -@@ -5244,6 +5250,12 @@ QualType TreeTransform::TransformIncompleteArrayType( - return QualType(); - } - -+ // PASTA PATCH: We might be dealing with an adjusted type due to Multiplier -+ // issue #130. -+ if (auto AT = dyn_cast(Result.getTypePtr())) { -+ Result = AT->getAdjustedType(); -+ } -+ - IncompleteArrayTypeLoc NewTL = TLB.push(Result); - NewTL.setLBracketLoc(TL.getLBracketLoc()); - NewTL.setRBracketLoc(TL.getRBracketLoc()); -@@ -5289,6 +5301,12 @@ TreeTransform::TransformVariableArrayType(TypeLocBuilder &TLB, - return QualType(); - } - -+ // PASTA PATCH: We might be dealing with an adjusted type due to Multiplier -+ // issue #130. -+ if (auto AT = dyn_cast(Result.getTypePtr())) { -+ Result = AT->getAdjustedType(); -+ } -+ - // We might have constant size array now, but fortunately it has the same - // location layout. - ArrayTypeLoc NewTL = TLB.push(Result); -@@ -5337,6 +5355,12 @@ TreeTransform::TransformDependentSizedArrayType(TypeLocBuilder &TLB, - return QualType(); - } - -+ // PASTA PATCH: We might be dealing with an adjusted type due to Multiplier -+ // issue #130. -+ if (auto AT = dyn_cast(Result.getTypePtr())) { -+ Result = AT->getAdjustedType(); -+ } -+ - // We might have any sort of array type now, but fortunately they - // all have the same location layout. - ArrayTypeLoc NewTL = TLB.push(Result); --- -2.39.0 - diff --git a/ports/llvm-15/0027-unknown-attrs-as-annotations.patch b/ports/llvm-15/0027-unknown-attrs-as-annotations.patch deleted file mode 100644 index b185db39..00000000 --- a/ports/llvm-15/0027-unknown-attrs-as-annotations.patch +++ /dev/null @@ -1,180 +0,0 @@ -diff --git a/clang/include/clang/Basic/LangOptions.def b/clang/include/clang/Basic/LangOptions.def -index ad366821f..86c99507d 100644 ---- a/clang/include/clang/Basic/LangOptions.def -+++ b/clang/include/clang/Basic/LangOptions.def -@@ -309,6 +309,8 @@ LANGOPT(FastRelaxedMath , 1, 0, "OpenCL fast relaxed math") - BENIGN_LANGOPT(CLNoSignedZero , 1, 0, "Permit Floating Point optimization without regard to signed zeros") - COMPATIBLE_LANGOPT(CLUnsafeMath , 1, 0, "Unsafe Floating Point Math") - COMPATIBLE_LANGOPT(CLFiniteMathOnly , 1, 0, "__FINITE_MATH_ONLY__ predefined macro") -+ -+LANGOPT(UnknownAttrAnnotate, 1, 0, "Unknown attributes are treated as annotation or annotation type attributes during semantic analysis") - /// FP_CONTRACT mode (on/off/fast). - BENIGN_ENUM_LANGOPT(DefaultFPContractMode, FPModeKind, 2, FPM_Off, "FP contraction type") - COMPATIBLE_LANGOPT(ExpStrictFP, 1, false, "Enable experimental strict floating point") -diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td -index 3cab37b21..71a6a6600 100644 ---- a/clang/include/clang/Driver/Options.td -+++ b/clang/include/clang/Driver/Options.td -@@ -4277,6 +4277,11 @@ def working_directory : JoinedOrSeparate<["-"], "working-directory">, Flags<[CC1 - def working_directory_EQ : Joined<["-"], "working-directory=">, Flags<[CC1Option]>, - Alias; - -+def funknown_attrs_as_annotate : Flag<["-"], "funknown-attrs-as-annotate">, -+ Flags<[CC1Option]>, -+ HelpText<"Treat unknown attributes as annotation or annotation type attributes in semantic analysis">, -+ MarshallingInfoFlag>; -+ - // Double dash options, which are usually an alias for one of the previous - // options. - -diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp -index 3704ed858..ae3935cc5 100644 ---- a/clang/lib/Driver/ToolChains/Clang.cpp -+++ b/clang/lib/Driver/ToolChains/Clang.cpp -@@ -4595,6 +4595,11 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA, - } - } - -+ if (const Arg *A = Args.getLastArg(options::OPT_funknown_attrs_as_annotate)) { -+ CmdArgs.push_back("-funknown-attrs-as-annotate"); -+ A->claim(); -+ } -+ - if (IsOpenMPDevice) { - // We have to pass the triple of the host if compiling for an OpenMP device. - std::string NormalizedTriple = -diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp -index bf73ddfd1..b2832d4fc 100644 ---- a/clang/lib/Parse/ParseDeclCXX.cpp -+++ b/clang/lib/Parse/ParseDeclCXX.cpp -@@ -4276,18 +4276,22 @@ bool Parser::ParseCXX11AttributeArgs( - Syntax = ParsedAttr::AS_Microsoft; - } - -+ - // If the attribute isn't known, we will not attempt to parse any -- // arguments. -- if (Syntax != ParsedAttr::AS_Microsoft && -- !hasAttribute(LO.CPlusPlus ? AttributeCommonInfo::Syntax::AS_CXX11 -- : AttributeCommonInfo::Syntax::AS_C2x, -- ScopeName, AttrName, getTargetInfo(), getLangOpts())) { -- if (getLangOpts().MicrosoftExt || getLangOpts().HLSL) { -+ // arguments. Unless we are treating unknown attributes as annotation -+ // attributes. -+ if (!getLangOpts().UnknownAttrAnnotate) { -+ if (Syntax != ParsedAttr::AS_Microsoft && -+ !hasAttribute(LO.CPlusPlus ? AttributeCommonInfo::Syntax::AS_CXX11 -+ : AttributeCommonInfo::Syntax::AS_C2x, -+ ScopeName, AttrName, getTargetInfo(), getLangOpts())) { -+ if (getLangOpts().MicrosoftExt || getLangOpts().HLSL) { -+ } -+ // Eat the left paren, then skip to the ending right paren. -+ ConsumeParen(); -+ SkipUntil(tok::r_paren); -+ return false; - } -- // Eat the left paren, then skip to the ending right paren. -- ConsumeParen(); -- SkipUntil(tok::r_paren); -- return false; - } - - if (ScopeName && (ScopeName->isStr("gnu") || ScopeName->isStr("__gnu__"))) { -diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp -index 838fd4835..e8efeba90 100644 ---- a/clang/lib/Sema/SemaDeclAttr.cpp -+++ b/clang/lib/Sema/SemaDeclAttr.cpp -@@ -4168,6 +4168,21 @@ static void handleAnnotateAttr(Sema &S, Decl *D, const ParsedAttr &AL) { - S.AddAnnotationAttr(D, AL, Str, Args); - } - -+static void -+handleUnknownAttrAsAnnotateAttr(Sema &S, Decl *D, const ParsedAttr &AL) { -+ // Get name of unknown attribute: -+ StringRef Str = AL.getAttrName()->getName(); -+ -+ llvm::SmallVector Args; -+ Args.reserve(AL.getNumArgs()); -+ for (unsigned Idx = 0; Idx < AL.getNumArgs(); Idx++) { -+ assert(!AL.isArgIdent(Idx)); -+ Args.push_back(AL.getArgAsExpr(Idx)); -+ } -+ -+ S.AddAnnotationAttr(D, AL, Str, Args); -+} -+ - static void handleAlignValueAttr(Sema &S, Decl *D, const ParsedAttr &AL) { - S.AddAlignValueAttr(D, AL, AL.getArgAsExpr(0)); - } -@@ -8355,11 +8370,15 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL, - // though they were unknown attributes. - if (AL.getKind() == ParsedAttr::UnknownAttribute || - !AL.existsInTarget(S.Context.getTargetInfo())) { -- S.Diag(AL.getLoc(), -- AL.isDeclspecAttribute() -+ if (S.getLangOpts().UnknownAttrAnnotate) { -+ handleUnknownAttrAsAnnotateAttr(S, D, AL); -+ } else { -+ S.Diag(AL.getLoc(), -+ AL.isDeclspecAttribute() - ? (unsigned)diag::warn_unhandled_ms_attribute_ignored - : (unsigned)diag::warn_unknown_attribute_ignored) -- << AL << AL.getRange(); -+ << AL << AL.getRange(); -+ } - return; - } - -diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp -index edcac4d2e..9ed87b45d 100644 ---- a/clang/lib/Sema/SemaType.cpp -+++ b/clang/lib/Sema/SemaType.cpp -@@ -8186,6 +8186,24 @@ static void HandleMatrixTypeAttr(QualType &CurType, const ParsedAttr &Attr, - CurType = T; - } - -+static void HandleUnkownTypeAttrAsAnnotateTypeAttr(TypeProcessingState &State, -+ QualType &CurType, const ParsedAttr &PA) { -+ Sema &S = State.getSema(); -+ StringRef Str = PA.getAttrName()->getName(); -+ -+ llvm::SmallVector Args; -+ Args.reserve(PA.getNumArgs()); -+ for (unsigned Idx = 0; Idx < PA.getNumArgs(); Idx++) { -+ assert(!PA.isArgIdent(Idx)); -+ Args.push_back(PA.getArgAsExpr(Idx)); -+ } -+ if (!S.ConstantFoldAttrArgs(PA, Args)) -+ return; -+ auto *AnnotateTypeAttr = -+ AnnotateTypeAttr::Create(S.Context, Str, Args.data(), Args.size(), PA); -+ CurType = State.getAttributedType(AnnotateTypeAttr, CurType, CurType); -+} -+ - static void HandleAnnotateTypeAttr(TypeProcessingState &State, - QualType &CurType, const ParsedAttr &PA) { - Sema &S = State.getSema(); -@@ -8289,12 +8307,17 @@ static void processTypeAttrs(TypeProcessingState &state, QualType &type, - - case ParsedAttr::UnknownAttribute: - if (attr.isStandardAttributeSyntax()) { -- state.getSema().Diag(attr.getLoc(), -- diag::warn_unknown_attribute_ignored) -- << attr << attr.getRange(); -- // Mark the attribute as invalid so we don't emit the same diagnostic -- // multiple times. -- attr.setInvalid(); -+ if (state.getSema().getLangOpts().UnknownAttrAnnotate) { -+ HandleUnkownTypeAttrAsAnnotateTypeAttr(state, type, attr); -+ attr.setUsedAsTypeAttr(); -+ } else { -+ state.getSema().Diag(attr.getLoc(), -+ diag::warn_unknown_attribute_ignored) -+ << attr << attr.getRange(); -+ // Mark the attribute as invalid so we don't emit the same diagnostic -+ // multiple times. -+ attr.setInvalid(); -+ } - } - break; - diff --git a/ports/llvm-16/0025-PASTA-patches.patch b/ports/llvm-16/0025-PASTA-patches.patch deleted file mode 100644 index 0c45a863..00000000 --- a/ports/llvm-16/0025-PASTA-patches.patch +++ /dev/null @@ -1,2663 +0,0 @@ -From 663bfa3a1a437c1538fc43d9c0e1fcaefb3a8d09 Mon Sep 17 00:00:00 2001 -From: Peter Goodman -Date: Mon, 8 May 2023 12:48:13 -0400 -Subject: [PATCH] PASTA patches for LLVM 16.0.3. - ---- - clang/include/clang/AST/PrettyPrinter.h | 10 +- - clang/include/clang/Basic/TokenKinds.def | 7 + - clang/include/clang/Lex/PPCallbacks.h | 14 + - .../include/clang/Lex/PPCallbacksEventKind.h | 168 +++++++++++ - clang/include/clang/Lex/Preprocessor.h | 46 ++- - clang/include/clang/Lex/TokenLexer.h | 7 +- - clang/lib/AST/ASTContext.cpp | 129 +++++++-- - clang/lib/AST/ExprConstant.cpp | 9 + - clang/lib/AST/ItaniumMangle.cpp | 8 + - clang/lib/AST/Type.cpp | 9 +- - clang/lib/Basic/SourceManager.cpp | 3 +- - clang/lib/Lex/Lexer.cpp | 9 + - clang/lib/Lex/PPDirectives.cpp | 148 +++++++--- - clang/lib/Lex/PPExpressions.cpp | 29 ++ - clang/lib/Lex/PPLexerChange.cpp | 39 +++ - clang/lib/Lex/PPMacroExpansion.cpp | 268 +++++++++++++++++- - clang/lib/Lex/Pragma.cpp | 76 ++++- - clang/lib/Lex/Preprocessor.cpp | 44 ++- - clang/lib/Lex/TokenLexer.cpp | 226 ++++++++++++++- - clang/lib/Parse/ParseExpr.cpp | 32 ++- - clang/lib/Parse/ParseTemplate.cpp | 8 + - clang/lib/Sema/SemaDecl.cpp | 12 + - clang/lib/Sema/SemaExpr.cpp | 32 ++- - clang/lib/Sema/SemaExprCXX.cpp | 1 + - clang/lib/Sema/TreeTransform.h | 24 ++ - 25 files changed, 1233 insertions(+), 125 deletions(-) - create mode 100644 clang/include/clang/Lex/PPCallbacksEventKind.h - -diff --git a/clang/include/clang/AST/PrettyPrinter.h b/clang/include/clang/AST/PrettyPrinter.h -index 5aeaca7be..643cd7e5d 100644 ---- a/clang/include/clang/AST/PrettyPrinter.h -+++ b/clang/include/clang/AST/PrettyPrinter.h -@@ -76,7 +76,8 @@ struct PrintingPolicy { - PrintCanonicalTypes(false), PrintInjectedClassNameWithArguments(true), - UsePreferredNames(true), AlwaysIncludeTypeForTemplateArgument(false), - CleanUglifiedParameters(false), EntireContentsOfLargeArray(true), -- UseEnumerators(true) {} -+ UseEnumerators(true), PrintAdjustedArrayType(true), -+ PrintSubstitutedTemplateParameters(false) {} - - /// Adjust this printing policy for cases where it's known that we're - /// printing C++ code (for instance, if AST dumping reaches a C++-only -@@ -299,6 +300,13 @@ struct PrintingPolicy { - /// enumerator name or via cast of an integer. - unsigned UseEnumerators : 1; - -+ /// PASTA PATCH: Should we print the adjusted array type, or the original -+ /// array type? -+ unsigned PrintAdjustedArrayType : 1; -+ -+ /// PASTA PATCH: Should we print substituted template parameters? -+ unsigned PrintSubstitutedTemplateParameters : 1; -+ - /// Callbacks to use to allow the behavior of printing to be customized. - const PrintingCallbacks *Callbacks = nullptr; - }; -diff --git a/clang/include/clang/Basic/TokenKinds.def b/clang/include/clang/Basic/TokenKinds.def -index 96feae991..0fa2f84d5 100644 ---- a/clang/include/clang/Basic/TokenKinds.def -+++ b/clang/include/clang/Basic/TokenKinds.def -@@ -565,6 +565,13 @@ ALIAS("__is_same_as", __is_same, KEYCXX) - KEYWORD(__private_extern__ , KEYALL) - KEYWORD(__module_private__ , KEYALL) - -+// PASTA PATCH: Apple-specific XNU extensions. -+UNARY_EXPR_OR_TYPE_TRAIT(__builtin_ptrauth_type_discriminator, PtrAuthTypeDiscriminator, KEYALL) -+UNARY_EXPR_OR_TYPE_TRAIT(__builtin_xnu_type_signature, XNUTypeSignature, KEYALL) -+UNARY_EXPR_OR_TYPE_TRAIT(__builtin_xnu_type_summary, XNUTypeSummary, KEYALL) -+UNARY_EXPR_OR_TYPE_TRAIT(__builtin_tmo_type_get_metadata, TMOTypeGetMetadata, KEYALL) -+TYPE_TRAIT_2(__builtin_xnu_types_compatible, XNUTypeCompatible, KEYALL) -+ - // Extension that will be enabled for Microsoft, Borland and PS4, but can be - // disabled via '-fno-declspec'. - KEYWORD(__declspec , 0) -diff --git a/clang/include/clang/Lex/PPCallbacks.h b/clang/include/clang/Lex/PPCallbacks.h -index 94f96cf9c..c7b5b7d4b 100644 ---- a/clang/include/clang/Lex/PPCallbacks.h -+++ b/clang/include/clang/Lex/PPCallbacks.h -@@ -40,6 +40,14 @@ public: - EnterFile, ExitFile, SystemHeaderPragma, RenameFile - }; - -+ // PASTA PATCH: Add in an event that lets us get better visibility into the -+ // behavior of the preprocessor, as things are happening. -+#define LLVM_CLANG_HAS_PASTA_EVENTS 20230322L -+ enum EventKind : int; -+ -+ // PASTA PATCH: -+ virtual void Event(const Token &Tok, EventKind Kind, uintptr_t Data) {} -+ - /// Callback invoked whenever a source file is entered or exited. - /// - /// \param Loc Indicates the new location. -@@ -441,6 +449,12 @@ public: - - ~PPChainedCallbacks() override; - -+ // PASTA PATCH -+ void Event(const Token &Tok, EventKind Kind, uintptr_t Data) override { -+ First->Event(Tok, Kind, Data); -+ Second->Event(Tok, Kind, Data); -+ } -+ - void FileChanged(SourceLocation Loc, FileChangeReason Reason, - SrcMgr::CharacteristicKind FileType, - FileID PrevFID) override { -diff --git a/clang/include/clang/Lex/PPCallbacksEventKind.h b/clang/include/clang/Lex/PPCallbacksEventKind.h -new file mode 100644 -index 000000000..4f399f9c9 ---- /dev/null -+++ b/clang/include/clang/Lex/PPCallbacksEventKind.h -@@ -0,0 +1,168 @@ -+// Copyright (c) 2023-present, Trail of Bits, Inc. -+// All rights reserved. -+// -+// This source code is licensed in accordance with the terms specified in -+// the LICENSE.TXT file found in the root directory of this source tree. -+ -+#pragma once -+ -+#include "PPCallbacks.h" -+ -+namespace clang { -+ -+enum PPCallbacks::EventKind : int { -+ // Tell us just after one of the lexers has lexed a token. -+ // -+ // `Tok` is the token generated from one of the underlying lexers. -+ // `Data` is zero or it is a raw source location for where the lexer was -+ // invoked. -+ TokenFromLexer, -+ TokenFromTokenLexer, -+ TokenFromCachingLexer, -+ TokenFromAfterModuleImportLexer, -+ -+ // Tell the listener that the parser has split a token. This happens in C++ -+ // code for templates, e.g. `constant>>0`, where the `>>>` is first -+ // treated as one token, but then where the parser realizes that it is -+ // really `constant> > 0`. -+ BeginSplitToken, -+ EndSplitToken, -+ -+ // Tell the listener that we've just lexed the hash token that should start -+ // off a directive. -+ // -+ // `Tok` is the `#`. -+ BeginDirective, -+ -+ // Ends with an `EndDirective`. -+ // -+ // `Tok` is the `#`. -+ BeginSkippedArea, -+ -+ // Tell the listener that we're in a named directive, e.g. `if` or `define`. -+ // -+ // `Tok` is the `#`. -+ // `Data` is a `Token *` of the token lexed after the `#`. -+ SetNamedDirective, -+ -+ // Tell the listener that we're in an unnamed directive, e.g. GNU line -+ // numbers, such as `# 1`. -+ // -+ // `Tok` is the `#`. -+ // `Data` is a `const Token *` of the token lexed after the `#`. -+ SetUnnamedDirective, -+ -+ // End a directive. -+ // -+ // `Tok` is the `tok::eod` token. -+ EndDirective, -+ -+ // We thought something was a directive, but it wasn't, e.g. due to us -+ // parsing a .S file. -+ EndNonDirective, -+ -+ // `Tok` is the name of the macro being expanded. -+ // `Data` is the `MacroInfo *`. For built-in macros, this may be `nullptr`. -+ BeginMacroExpansion, -+ SwitchToExpansion, -+ BeginPreArgumentExpansion, -+ EndPreArgumentExpansion, -+ PrepareToCancelExpansion, // E.g. `_Pragma` in macro argument pre-exansion. -+ CancelExpansion, // E.g. `_Pragma` in a macro parameter. -+ EndMacroExpansion, -+ -+ // `Tok` is the name of the macro being expanded. -+ // `Data` is a `MacroInfo *`. -+ BeginMacroCallArgumentList, -+ -+ // `Tok` is the token that terminated the argument list, i.e. a `)`. -+ // `Data` is a `MacroArgs *`. For built-in macros, this may be `nullptr`. -+ EndMacroCallArgumentList, -+ -+ // `Tok` is the token just before the first token of the argument, e.g. -+ // `(` or `,`. -+ // `Data` is a `Token *` of the macro name. -+ BeginMacroCallArgument, -+ -+ // `Tok` is the token just before the first token of the argument. -+ // `Data` is a `Token *` just after the last token of the argument, e.g. a -+ // `)` or a `,`. -+ EndMacroCallArgument, -+ -+ // `Tok` is the token just before the first token of the variadic arguments, -+ // e.g. a `(` or a `,`. -+ // `Data` is a `Token *` of the macro name. -+ BeginVariadicCallArgumentList, -+ -+ // `Tok` is the token just before the first token of the variadic arguments -+ // `Data` is a `Token *` just after the last token of the arguments, e.g. a -+ // `)` or a `,`. -+ EndVariadicCallArgumentList, -+ -+ // `Tok` is the token which begins the substitution. -+ // `Data` is `nullptr`. -+ BeginSubstitution, -+ -+ // `Tok` is the token (previously visible via another event) which we want -+ // to say begins the substitution. -+ // `Data` is `nullptr`. -+ BeginDelayedSubstitution, -+ -+ // `Tok` is the last token before the substituted tokens will begin being -+ // outputted. -+ // `Data` is a `Token *` of the first token of the substitution. -+ SwitchToSubstitution, -+ -+ // `Tok` is the last substituted token. -+ // `Data` is a `Token *` of the first token of the substitution. -+ EndSubstitution, -+ -+ // `Tok` is the token (possibly previously visible via another event) which -+ // is about to be pasted with something else. -+ // `Data` is a `Token *` of the `##` token. -+ BeginConcatenation, -+ EndConcatenation, -+ -+ // `Tok` is a `##` for concatenation. -+ // `Data` is a `Token *` of the token into which the right hand side of the -+ // `##` is being concatenated. -+ ConcatenationOperatorToken, -+ -+ // `Tok` is a right hand side for concatenation. -+ // `Data` is a `Token *` of the token into which `Tok` is being concatenated. -+ ConcatenationAccumulationToken, -+ -+ // `Tok` is the first output token in the intermediate results buffer. -+ // `Data` is the number of tokens in the buffer. -+ // -+ // NOTE(pag): Before/After concatenation don't actually show us the resulting -+ // pasted token. Instead, that is done in `TokenLexer::Lex`, as it -+ // is reading from the resulting tokens. To observe the -+ // concatenation, use `BeginConcatenation` and `EndConcatenation`. -+ BeforeParameterSubstitutions, -+ AfterParameterSubstitutions, -+ BeforeVAOpt, -+ AfterVAOpt, -+ BeforeStringify, -+ AfterStringify, -+ BeforeConcatenation, -+ AfterConcatenation, -+ BeforeMacroParameterUse, -+ AfterMacroParameterUse, -+ BeforeRemoveCommas, -+ AfterRemoveCommas, -+ -+ // `Tok` is the token inside of a `__VA_OPT__` that is being skipped if -+ // `__VA_ARGS__` is empty. `Data` is zero. -+ SkippedVAOptToken, -+ -+ // // `Tok` is a special token elided from a macro definition body, e.g. -+ // // `#` before a parameter name. -+ // // `Data` is a `MacroInfo *`. -+ // ElidedTokenInDefinitionBody, -+ -+ // // `Tok` is a token in the body. -+ // TokenInDefinitionBody, -+}; -+ -+} // namespace clang -diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h -index 322626802..72f23ebab 100644 ---- a/clang/include/clang/Lex/Preprocessor.h -+++ b/clang/include/clang/Lex/Preprocessor.h -@@ -790,6 +790,9 @@ private: - /// encountered (e.g. a file is \#included, etc). - std::unique_ptr Callbacks; - -+ // PASTA PATCH: Add post-lex action. -+ std::function PostLexAction; -+ - struct MacroExpandsInfo { - Token Tok; - MacroDefinition MD; -@@ -1624,8 +1627,8 @@ public: - - /// Process directives while skipping until the through header or - /// #pragma hdrstop is found. -- void HandleSkippedDirectiveWhileUsingPCH(Token &Result, -- SourceLocation HashLoc); -+ void HandleSkippedDirectiveWhileUsingPCH(Token &Result, // PASTA PATCH -+ const Token &SavedHash); // PASTA PATCH - - /// Enter the specified FileID as the main source file, - /// which implicitly adds the builtin defines etc. -@@ -2434,15 +2437,7 @@ private: - CurPPLexer = nullptr; - } - -- void PopIncludeMacroStack() { -- CurLexer = std::move(IncludeMacroStack.back().TheLexer); -- CurPPLexer = IncludeMacroStack.back().ThePPLexer; -- CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer); -- CurDirLookup = IncludeMacroStack.back().TheDirLookup; -- CurLexerSubmodule = IncludeMacroStack.back().TheSubmodule; -- CurLexerKind = IncludeMacroStack.back().CurLexerKind; -- IncludeMacroStack.pop_back(); -- } -+ void PopIncludeMacroStack(); // PASTA PATCH - - void PropagateLineStartLeadingSpaceInfo(Token &Result); - -@@ -2506,7 +2501,7 @@ private: - /// \p FoundElse is false, then \#else directives are ok, if not, then we have - /// already seen one so a \#else directive is a duplicate. When this returns, - /// the caller can lex the first valid token. -- void SkipExcludedConditionalBlock(SourceLocation HashTokenLoc, -+ void SkipExcludedConditionalBlock(const Token &HashToken, // PASTA PATCH - SourceLocation IfTokenLoc, - bool FoundNonSkipPortion, bool FoundElse, - SourceLocation ElseLoc = SourceLocation()); -@@ -2574,7 +2569,7 @@ private: - /// After reading "MACRO(", this method is invoked to read all of the formal - /// arguments specified for the macro invocation. Returns null on error. - MacroArgs *ReadMacroCallArgumentList(Token &MacroName, MacroInfo *MI, -- SourceLocation &MacroEnd); -+ Token &MacroEndTok); // PASTA PATCH - - /// If an identifier token is read that is to be expanded - /// as a builtin macro, handle it and return the next token as 'Tok'. -@@ -2641,12 +2636,12 @@ private: - /// Handle*Directive - implement the various preprocessor directives. These - /// should side-effect the current preprocessor object so that the next call - /// to Lex() will return the appropriate token next. -- void HandleLineDirective(); -- void HandleDigitDirective(Token &Tok); -- void HandleUserDiagnosticDirective(Token &Tok, bool isWarning); -- void HandleIdentSCCSDirective(Token &Tok); -- void HandleMacroPublicDirective(Token &Tok); -- void HandleMacroPrivateDirective(); -+ void HandleLineDirective(const Token &SavedHash); // PASTA PATCH -+ void HandleDigitDirective(const Token &SavedHash, Token &Tok); // PASTA PATCH -+ void HandleUserDiagnosticDirective(const Token &SavedHash, Token &Tok, bool isWarning); // PASTA PATCH -+ void HandleIdentSCCSDirective(const Token &SavedHash, Token &Tok); // PASTA PATCH -+ void HandleMacroPublicDirective(const Token &SavedHash, Token &Tok); // PASTA PATCH -+ void HandleMacroPrivateDirective(const Token &SavedHash); // PASTA PATCH - - /// An additional notification that can be produced by a header inclusion or - /// import to tell the parser what happened. -@@ -2678,7 +2673,7 @@ private: - ModuleMap::KnownHeader &SuggestedModule, bool isAngled); - - // File inclusion. -- void HandleIncludeDirective(SourceLocation HashLoc, Token &Tok, -+ void HandleIncludeDirective(const Token &SavedHash, Token &Tok, // PASTA PATCH - ConstSearchDirIterator LookupFrom = nullptr, - const FileEntry *LookupFromFile = nullptr); - ImportAction -@@ -2686,9 +2681,9 @@ private: - Token &FilenameTok, SourceLocation EndLoc, - ConstSearchDirIterator LookupFrom = nullptr, - const FileEntry *LookupFromFile = nullptr); -- void HandleIncludeNextDirective(SourceLocation HashLoc, Token &Tok); -- void HandleIncludeMacrosDirective(SourceLocation HashLoc, Token &Tok); -- void HandleImportDirective(SourceLocation HashLoc, Token &Tok); -+ void HandleIncludeNextDirective(const Token &SavedHash, Token &Tok); // PASTA PATCH -+ void HandleIncludeMacrosDirective(const Token &SavedHash, Token &Tok); // PASTA PATCH -+ void HandleImportDirective(const Token &SavedHash, Token &Tok); // PASTA PATCH - void HandleMicrosoftImportDirective(Token &Tok); - - public: -@@ -2755,8 +2750,9 @@ private: - void replayPreambleConditionalStack(); - - // Macro handling. -- void HandleDefineDirective(Token &Tok, bool ImmediatelyAfterHeaderGuard); -- void HandleUndefDirective(); -+ void HandleDefineDirective(const Token &SavedHash, Token &Tok, // PASTA PATCH -+ bool ImmediatelyAfterHeaderGuard); // PASTA PATCH -+ void HandleUndefDirective(const Token &SavedHash); // PASTA PATCH - - // Conditional Inclusion. - void HandleIfdefDirective(Token &Result, const Token &HashToken, -diff --git a/clang/include/clang/Lex/TokenLexer.h b/clang/include/clang/Lex/TokenLexer.h -index 4d229ae61..923107b84 100644 ---- a/clang/include/clang/Lex/TokenLexer.h -+++ b/clang/include/clang/Lex/TokenLexer.h -@@ -13,7 +13,7 @@ - #ifndef LLVM_CLANG_LEX_TOKENLEXER_H - #define LLVM_CLANG_LEX_TOKENLEXER_H - --#include "clang/Basic/SourceLocation.h" -+#include "clang/Lex/Token.h" // PASTA PATCH - #include "llvm/ADT/ArrayRef.h" - - namespace clang { -@@ -40,6 +40,9 @@ class TokenLexer { - /// The current preprocessor object we are expanding for. - Preprocessor &PP; - -+ // PASTA PATCH: The name token of the macro. -+ Token MacroNameTok; -+ - /// This is the pointer to an array of tokens that the macro is - /// defined to, with arguments expanded for function-like macros. If this is - /// a token stream, these are the tokens we are returning. This points into -@@ -98,7 +101,7 @@ class TokenLexer { - - /// When true, the produced tokens have Token::IsReinjected flag set. - /// See the flag documentation for details. -- bool IsReinject : 1; -+ bool IsReinject; // PASTA PATCH - - public: - /// Create a TokenLexer for the specified macro with the specified actual -diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp -index 2884fe660..3248f650d 100644 ---- a/clang/lib/AST/ASTContext.cpp -+++ b/clang/lib/AST/ASTContext.cpp -@@ -3545,49 +3545,98 @@ QualType ASTContext::getConstantArrayType(QualType EltTy, - EltTy->isIncompleteType() || EltTy->isConstantSizeType()) && - "Constant array of VLAs is illegal!"); - -- // We only need the size as part of the type if it's instantiation-dependent. -- if (SizeExpr && !SizeExpr->isInstantiationDependent()) -- SizeExpr = nullptr; -- - // Convert the array size into a canonical width matching the pointer size for - // the target. - llvm::APInt ArySize(ArySizeIn); - ArySize = ArySize.zextOrTrunc(Target->getMaxPointerWidth()); - -+ // PASTA PATCH: Make it always retain `SizeExpr` so that we can see token -+ // provenance for arrays whose types are the result of some -+ // expression. -+ const Expr *OrigSizeExpr = SizeExpr; -+ if (SizeExpr && !SizeExpr->isInstantiationDependent() && -+ !isa(SizeExpr) && !isa(SizeExpr)) { -+ llvm::APSInt ArySizeInt(ArySizeIn, !ArySizeIn.isNegative()); -+ OrigSizeExpr = ConstantExpr::Create( -+ *this, const_cast(SizeExpr), clang::APValue(ArySizeInt)); -+ } -+ -+ // We only need the size as part of the type if it's instantiation-dependent. -+ if (SizeExpr && !SizeExpr->isInstantiationDependent()) -+ SizeExpr = nullptr; -+ - llvm::FoldingSetNodeID ID; - ConstantArrayType::Profile(ID, *this, EltTy, ArySize, SizeExpr, ASM, - IndexTypeQuals); - - void *InsertPos = nullptr; -- if (ConstantArrayType *ATP = -- ConstantArrayTypes.FindNodeOrInsertPos(ID, InsertPos)) -+ ConstantArrayType *ATP = -+ ConstantArrayTypes.FindNodeOrInsertPos(ID, InsertPos); -+ -+ // PASTA PATCH: Profile the original one. In practice, we should probably -+ // never have a cache hit. -+ llvm::FoldingSetNodeID OrigID; -+ void *OrigInsertPos = nullptr; -+ ConstantArrayType *OrigATP = nullptr; -+ if (OrigSizeExpr) { -+ OrigID.AddPointer(OrigSizeExpr); // Want uniqueness. -+ OrigATP = ConstantArrayTypes.FindNodeOrInsertPos(OrigID, OrigInsertPos); -+ } -+ -+ if (ATP && OrigATP) -+ return getAdjustedType(QualType(OrigATP, 0), QualType(ATP, 0)); -+ -+ if (!OrigSizeExpr && ATP) - return QualType(ATP, 0); - -- // If the element type isn't canonical or has qualifiers, or the array bound -- // is instantiation-dependent, this won't be a canonical type either, so fill -- // in the canonical type field. -- QualType Canon; -- // FIXME: Check below should look for qualifiers behind sugar. -- if (!EltTy.isCanonical() || EltTy.hasLocalQualifiers() || SizeExpr) { -- SplitQualType canonSplit = getCanonicalType(EltTy).split(); -- Canon = getConstantArrayType(QualType(canonSplit.Ty, 0), ArySize, nullptr, -- ASM, IndexTypeQuals); -- Canon = getQualifiedType(Canon, canonSplit.Quals); -+ if (!ATP) { -+ // If the element type isn't canonical or has qualifiers, or the array bound -+ // is instantiation-dependent, this won't be a canonical type either, so fill -+ // in the canonical type field. -+ QualType Canon; -+ // FIXME: Check below should look for qualifiers behind sugar. -+ if (!EltTy.isCanonical() || EltTy.hasLocalQualifiers() || SizeExpr) { -+ SplitQualType canonSplit = getCanonicalType(EltTy).split(); -+ Canon = getConstantArrayType(QualType(canonSplit.Ty, 0), ArySize, nullptr, -+ ASM, IndexTypeQuals); -+ Canon = getQualifiedType(Canon, canonSplit.Quals); - -- // Get the new insert position for the node we care about. -- ConstantArrayType *NewIP = -- ConstantArrayTypes.FindNodeOrInsertPos(ID, InsertPos); -- assert(!NewIP && "Shouldn't be in the map!"); (void)NewIP; -+ // Get the new insert position for the node we care about. -+ ConstantArrayType *NewIP = -+ ConstantArrayTypes.FindNodeOrInsertPos(ID, InsertPos); -+ assert(!NewIP && "Shouldn't be in the map!"); (void)NewIP; -+ -+ // TODO(pag): Calculate a new insert position for orig expressions? -+ } -+ -+ void *Mem = Allocate( -+ ConstantArrayType::totalSizeToAlloc(SizeExpr ? 1 : 0), -+ TypeAlignment); -+ ATP = new (Mem) -+ ConstantArrayType(EltTy, Canon, ArySize, SizeExpr, ASM, IndexTypeQuals); -+ ConstantArrayTypes.InsertNode(ATP, InsertPos); -+ Types.push_back(ATP); - } - -- void *Mem = Allocate( -- ConstantArrayType::totalSizeToAlloc(SizeExpr ? 1 : 0), -- TypeAlignment); -- auto *New = new (Mem) -- ConstantArrayType(EltTy, Canon, ArySize, SizeExpr, ASM, IndexTypeQuals); -- ConstantArrayTypes.InsertNode(New, InsertPos); -- Types.push_back(New); -- return QualType(New, 0); -+ // PASTA PATCH: Go get the original type. We need to re-calculate the -+ // `OrigInsertPos` because the buckets in `ConstantArrayTypes` -+ // may have changed. -+ OrigATP = ConstantArrayTypes.FindNodeOrInsertPos(OrigID, OrigInsertPos); -+ if (OrigSizeExpr && !OrigATP) { -+ void *OrigMem = Allocate( -+ ConstantArrayType::totalSizeToAlloc(OrigSizeExpr ? 1 : 0), -+ TypeAlignment); -+ OrigATP = new (OrigMem) -+ ConstantArrayType(EltTy, ATP->getCanonicalTypeInternal(), -+ ArySize, OrigSizeExpr, ASM, IndexTypeQuals); -+ ConstantArrayTypes.InsertNode(OrigATP, OrigInsertPos); -+ Types.push_back(OrigATP); -+ } -+ -+ if (ATP && OrigATP) -+ return getAdjustedType(QualType(OrigATP, 0), QualType(ATP, 0)); -+ -+ return QualType(ATP, 0); - } - - /// getVariableArrayDecayedType - Turns the given type, which may be -@@ -3688,6 +3737,12 @@ QualType ASTContext::getVariableArrayDecayedType(QualType type) const { - cat->getSizeExpr(), - cat->getSizeModifier(), - cat->getIndexTypeCVRQualifiers()); -+ -+ // PASTA PATCH: If we returned an adjusted type above, then we want to -+ // desugar it to the internal type. -+ if (auto AT = dyn_cast(result.getTypePtr())) { -+ result = AT->getAdjustedType(); -+ } - break; - } - -@@ -6867,11 +6922,23 @@ const ArrayType *ASTContext::getAsArrayType(QualType T) const { - // qualifiers into the array element type and return a new array type. - QualType NewEltTy = getQualifiedType(ATy->getElementType(), qs); - -- if (const auto *CAT = dyn_cast(ATy)) -- return cast(getConstantArrayType(NewEltTy, CAT->getSize(), -+ if (const auto *CAT = dyn_cast(ATy)) { -+ -+ // PASTA PATCH: `getConstantArrayType` might return an adjusted type. -+ QualType result = getConstantArrayType(NewEltTy, CAT->getSize(), - CAT->getSizeExpr(), - CAT->getSizeModifier(), -- CAT->getIndexTypeCVRQualifiers())); -+ CAT->getIndexTypeCVRQualifiers()); -+ -+ // PASTA PATCH: If we returned an adjusted type above, then we want to -+ // desugar it to the internal type. -+ if (auto AT = dyn_cast(result.getTypePtr())) { -+ result = AT->getAdjustedType(); -+ } -+ -+ return cast(result); -+ } -+ - if (const auto *IAT = dyn_cast(ATy)) - return cast(getIncompleteArrayType(NewEltTy, - IAT->getSizeModifier(), -diff --git a/clang/lib/AST/ExprConstant.cpp b/clang/lib/AST/ExprConstant.cpp -index 9c56ab125..5c573d72a 100644 ---- a/clang/lib/AST/ExprConstant.cpp -+++ b/clang/lib/AST/ExprConstant.cpp -@@ -13407,6 +13407,15 @@ bool IntExprEvaluator::VisitUnaryExprOrTypeTraitExpr( - Info.Ctx.getOpenMPDefaultSimdAlign(E->getArgumentType())) - .getQuantity(), - E); -+ -+ // PASTA PATCH: Fake these XNU-specific type traits. -+ case UETT_PtrAuthTypeDiscriminator: -+ case UETT_XNUTypeSummary: -+ case UETT_TMOTypeGetMetadata: -+ return Success(0, E); -+ -+ case UETT_XNUTypeSignature: -+ return false; - } - - llvm_unreachable("unknown expr/type trait"); -diff --git a/clang/lib/AST/ItaniumMangle.cpp b/clang/lib/AST/ItaniumMangle.cpp -index b23bc5f8d..d8af66dc2 100644 ---- a/clang/lib/AST/ItaniumMangle.cpp -+++ b/clang/lib/AST/ItaniumMangle.cpp -@@ -4657,6 +4657,14 @@ recurse: - }; - - switch(SAE->getKind()) { -+ -+ // PASTA PATCH: Sort of support these. -+ case UETT_PtrAuthTypeDiscriminator: -+ case UETT_XNUTypeSummary: -+ case UETT_TMOTypeGetMetadata: -+ case UETT_XNUTypeSignature: -+ break; -+ - case UETT_SizeOf: - Out << 's'; - MangleAlignofSizeofArg(); -diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp -index 54e62a193..324468784 100644 ---- a/clang/lib/AST/Type.cpp -+++ b/clang/lib/AST/Type.cpp -@@ -4084,9 +4084,14 @@ bool Type::hasUnnamedOrLocalType() const { - LinkageInfo LinkageComputer::computeTypeLinkageInfo(const Type *T) { - switch (T->getTypeClass()) { - #define TYPE(Class,Base) --#define NON_CANONICAL_TYPE(Class,Base) case Type::Class: -+ -+// PASTA PATCH: For Multiplier Issue #130, we use AdjustedTypes for arrays -+// and so we go and handle non-canonical types via desugarding -+// where Clang normally does not. -+#define NON_CANONICAL_TYPE(Class,Base) \ -+ case Type::Class: \ -+ return computeTypeLinkageInfo(cast(T)->desugar()); - #include "clang/AST/TypeNodes.inc" -- llvm_unreachable("didn't expect a non-canonical type here"); - - #define TYPE(Class,Base) - #define DEPENDENT_TYPE(Class,Base) case Type::Class: -diff --git a/clang/lib/Basic/SourceManager.cpp b/clang/lib/Basic/SourceManager.cpp -index 3d7a53879..8942ff1f7 100644 ---- a/clang/lib/Basic/SourceManager.cpp -+++ b/clang/lib/Basic/SourceManager.cpp -@@ -865,7 +865,8 @@ FileID SourceManager::getFileIDLocal(SourceLocation::UIntTy SLocOffset) const { - /// local one. - FileID SourceManager::getFileIDLoaded(SourceLocation::UIntTy SLocOffset) const { - if (SLocOffset < CurrentLoadedOffset) { -- assert(0 && "Invalid SLocOffset or bad function choice"); -+ // PASTA PATCH: https://github.com/trailofbits/pasta/issues/57. -+ // assert(0 && "Invalid SLocOffset or bad function choice"); - return FileID(); - } - -diff --git a/clang/lib/Lex/Lexer.cpp b/clang/lib/Lex/Lexer.cpp -index d49d9e9e4..56a8dbe9e 100644 ---- a/clang/lib/Lex/Lexer.cpp -+++ b/clang/lib/Lex/Lexer.cpp -@@ -23,6 +23,7 @@ - #include "clang/Lex/LexDiagnostic.h" - #include "clang/Lex/LiteralSupport.h" - #include "clang/Lex/MultipleIncludeOpt.h" -+#include "clang/Lex/PPCallbacksEventKind.h" // PASTA PATCH - #include "clang/Lex/Preprocessor.h" - #include "clang/Lex/PreprocessorOptions.h" - #include "clang/Lex/Token.h" -@@ -2982,6 +2983,14 @@ void Lexer::ReadToEndOfLine(SmallVectorImpl *Result) { - } - assert(Tmp.is(tok::eod) && "Unexpected token!"); - -+ // PASTA PATCH: Visibility into all tokens. -+ if (PP) { -+ if (auto Callbacks = PP->getPPCallbacks()) { -+ Callbacks->Event(Tmp, PPCallbacks::TokenFromTokenLexer, 0u); -+ Callbacks->Event(Tmp, PPCallbacks::EndDirective, 0); -+ } -+ } -+ - // Finally, we're done; - return; - } -diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp -index 6ae513dea..1db695e18 100644 ---- a/clang/lib/Lex/PPDirectives.cpp -+++ b/clang/lib/Lex/PPDirectives.cpp -@@ -27,6 +27,7 @@ - #include "clang/Lex/ModuleLoader.h" - #include "clang/Lex/ModuleMap.h" - #include "clang/Lex/PPCallbacks.h" -+#include "clang/Lex/PPCallbacksEventKind.h" // PASTA PATCH - #include "clang/Lex/Pragma.h" - #include "clang/Lex/Preprocessor.h" - #include "clang/Lex/PreprocessorOptions.h" -@@ -91,6 +92,12 @@ SourceRange Preprocessor::DiscardUntilEndOfDirective() { - LexUnexpandedToken(Tmp); - } - Res.setEnd(Tmp.getLocation()); -+ -+// // PASTA PATCH: -+// if (Callbacks) { -+// Callbacks->Event(Tmp, PPCallbacks::EndDirective, 0); -+// } -+ - return Res; - } - -@@ -458,7 +465,7 @@ void Preprocessor::SuggestTypoedDirective(const Token &Tok, - /// If ElseOk is true, then \#else directives are ok, if not, then we have - /// already seen one so a \#else directive is a duplicate. When this returns, - /// the caller can lex the first valid token. --void Preprocessor::SkipExcludedConditionalBlock(SourceLocation HashTokenLoc, -+void Preprocessor::SkipExcludedConditionalBlock(const Token &HashToken, // PASTA PATCH - SourceLocation IfTokenLoc, - bool FoundNonSkipPortion, - bool FoundElse, -@@ -484,6 +491,12 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation HashTokenLoc, - CurPPLexer->pushConditionalLevel(IfTokenLoc, /*isSkipping*/ false, - FoundNonSkipPortion, FoundElse); - -+ // PASTA PATCH -+ SourceLocation HashTokenLoc = HashToken.getLocation(); -+ if (Callbacks) { -+ Callbacks->Event(HashToken, PPCallbacks::BeginSkippedArea, 0); -+ } -+ - // Enter raw mode to disable identifier lookup (and thus macro expansion), - // disabling warnings, etc. - CurPPLexer->LexingRawMode = true; -@@ -631,6 +644,12 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation HashTokenLoc, - if (Sub.empty() || // "if" - Sub == "def" || // "ifdef" - Sub == "ndef") { // "ifndef" -+ -+ // PASTA PATCH: -+ if (Callbacks) { -+ Callbacks->Event(Tok, PPCallbacks::BeginSkippedArea, 0); -+ } -+ - // We know the entire #if/#ifdef/#ifndef block will be skipped, don't - // bother parsing the condition. - DiscardUntilEndOfDirective(); -@@ -641,6 +660,12 @@ void Preprocessor::SkipExcludedConditionalBlock(SourceLocation HashTokenLoc, - SuggestTypoedDirective(Tok, Directive); - } - } else if (Directive[0] == 'e') { -+ -+ // PASTA PATCH: -+ if (Callbacks) { -+ Callbacks->Event(Tok, PPCallbacks::BeginSkippedArea, 0); -+ } -+ - StringRef Sub = Directive.substr(1); - if (Sub == "ndif") { // "endif" - PPConditionalInfo CondInfo; -@@ -1089,16 +1114,16 @@ private: - /// #define (to warn about macros that don't match the PCH) - /// #pragma (to check for pragma hdrstop). - /// All other directives are completely discarded. --void Preprocessor::HandleSkippedDirectiveWhileUsingPCH(Token &Result, -- SourceLocation HashLoc) { -+void Preprocessor::HandleSkippedDirectiveWhileUsingPCH( // PASTA PATCH -+ Token &Result, const Token &SavedHash) { // PASTA PATCH - if (const IdentifierInfo *II = Result.getIdentifierInfo()) { - if (II->getPPKeywordID() == tok::pp_define) { -- return HandleDefineDirective(Result, -- /*ImmediatelyAfterHeaderGuard=*/false); -+ return HandleDefineDirective(SavedHash, Result, // PASTA PATCH -+ /*ImmediatelyAfterHeaderGuard=*/false); // PASTA PATCH - } - if (SkippingUntilPCHThroughHeader && - II->getPPKeywordID() == tok::pp_include) { -- return HandleIncludeDirective(HashLoc, Result); -+ return HandleIncludeDirective(SavedHash, Result); // PASTA PATCH - } - if (SkippingUntilPragmaHdrStop && II->getPPKeywordID() == tok::pp_pragma) { - Lex(Result); -@@ -1137,10 +1162,36 @@ void Preprocessor::HandleDirective(Token &Result) { - // Save the '#' token in case we need to return it later. - Token SavedHash = Result; - -+ // PASTA PATCH: Tell us when we're about to start a directive. -+ if (Callbacks) { -+ Callbacks->Event(SavedHash, PPCallbacks::BeginDirective, 0); -+ } -+ - // Read the next token, the directive flavor. This isn't expanded due to - // C99 6.10.3p8. - LexUnexpandedToken(Result); - -+ // PASTA PATCH: Tell us when we're about to start a directive. -+ if (Callbacks) { -+ switch (Result.getKind()) { -+ case tok::identifier: -+ case tok::raw_identifier: -+ case tok::kw_if: -+ case tok::kw_else: -+ Callbacks->Event(SavedHash, PPCallbacks::SetNamedDirective, -+ reinterpret_cast(&Result)); -+ break; -+ case tok::eod: -+ case tok::code_completion: -+ case tok::numeric_constant: -+ case tok::string_literal: -+ default: -+ Callbacks->Event(SavedHash, PPCallbacks::SetUnnamedDirective, -+ reinterpret_cast(&Result)); -+ break; -+ } -+ } -+ - // C99 6.10.3p11: Is this preprocessor directive in macro invocation? e.g.: - // #define A(x) #x - // A(abc -@@ -1174,7 +1225,7 @@ void Preprocessor::HandleDirective(Token &Result) { - ResetMacroExpansionHelper helper(this); - - if (SkippingUntilPCHThroughHeader || SkippingUntilPragmaHdrStop) -- return HandleSkippedDirectiveWhileUsingPCH(Result, SavedHash.getLocation()); -+ return HandleSkippedDirectiveWhileUsingPCH(Result, SavedHash); // PASTA PATCH - - switch (Result.getKind()) { - case tok::eod: -@@ -1188,7 +1239,7 @@ void Preprocessor::HandleDirective(Token &Result) { - case tok::numeric_constant: // # 7 GNU line marker directive. - if (getLangOpts().AsmPreprocessor) - break; // # 4 is not a preprocessor directive in .S files. -- return HandleDigitDirective(Result); -+ return HandleDigitDirective(SavedHash, Result); // PASTA PATCH - default: - IdentifierInfo *II = Result.getIdentifierInfo(); - if (!II) break; // Not an identifier. -@@ -1218,24 +1269,25 @@ void Preprocessor::HandleDirective(Token &Result) { - // C99 6.10.2 - Source File Inclusion. - case tok::pp_include: - // Handle #include. -- return HandleIncludeDirective(SavedHash.getLocation(), Result); -+ return HandleIncludeDirective(SavedHash, Result); // PASTA PATCH - case tok::pp___include_macros: - // Handle -imacros. -- return HandleIncludeMacrosDirective(SavedHash.getLocation(), Result); -+ return HandleIncludeMacrosDirective(SavedHash, Result); // PASTA PATCH - - // C99 6.10.3 - Macro Replacement. - case tok::pp_define: -- return HandleDefineDirective(Result, ImmediatelyAfterTopLevelIfndef); -+ return HandleDefineDirective(SavedHash, Result, // PASTA PATCH -+ ImmediatelyAfterTopLevelIfndef); // PASTA PATCH - case tok::pp_undef: -- return HandleUndefDirective(); -+ return HandleUndefDirective(SavedHash); // PASTA PATCH - - // C99 6.10.4 - Line Control. - case tok::pp_line: -- return HandleLineDirective(); -+ return HandleLineDirective(SavedHash); // PASTA PATCH - - // C99 6.10.5 - Error Directive. - case tok::pp_error: -- return HandleUserDiagnosticDirective(Result, false); -+ return HandleUserDiagnosticDirective(SavedHash, Result, false); // PASTA PATCH - - // C99 6.10.6 - Pragma Directive. - case tok::pp_pragma: -@@ -1243,9 +1295,9 @@ void Preprocessor::HandleDirective(Token &Result) { - - // GNU Extensions. - case tok::pp_import: -- return HandleImportDirective(SavedHash.getLocation(), Result); -+ return HandleImportDirective(SavedHash, Result); // PASTA PATCH - case tok::pp_include_next: -- return HandleIncludeNextDirective(SavedHash.getLocation(), Result); -+ return HandleIncludeNextDirective(SavedHash, Result); // PASTA PATCH - - case tok::pp_warning: - if (LangOpts.CPlusPlus) -@@ -1258,11 +1310,11 @@ void Preprocessor::HandleDirective(Token &Result) { - : diag::ext_pp_warning_directive) - << /*C2x*/ 0; - -- return HandleUserDiagnosticDirective(Result, true); -+ return HandleUserDiagnosticDirective(SavedHash, Result, true); // PASTA PATCH - case tok::pp_ident: -- return HandleIdentSCCSDirective(Result); -+ return HandleIdentSCCSDirective(SavedHash, Result); // PASTA PATCH - case tok::pp_sccs: -- return HandleIdentSCCSDirective(Result); -+ return HandleIdentSCCSDirective(SavedHash, Result); // PASTA PATCH - case tok::pp_assert: - //isExtension = true; // FIXME: implement #assert - break; -@@ -1272,12 +1324,12 @@ void Preprocessor::HandleDirective(Token &Result) { - - case tok::pp___public_macro: - if (getLangOpts().Modules || getLangOpts().ModulesLocalVisibility) -- return HandleMacroPublicDirective(Result); -+ return HandleMacroPublicDirective(SavedHash, Result); // PASTA PATCH - break; - - case tok::pp___private_macro: - if (getLangOpts().Modules || getLangOpts().ModulesLocalVisibility) -- return HandleMacroPrivateDirective(); -+ return HandleMacroPrivateDirective(SavedHash); // PASTA PATCH - break; - } - break; -@@ -1288,6 +1340,11 @@ void Preprocessor::HandleDirective(Token &Result) { - // various pseudo-ops. Just return the # token and push back the following - // token to be lexed next time. - if (getLangOpts().AsmPreprocessor) { -+ // PASTA PATCH: Get visibility on end of macro directives. -+ if (Callbacks) { -+ Callbacks->Event(SavedHash, PPCallbacks::EndNonDirective, 0); -+ } -+ - auto Toks = std::make_unique(2); - // Return the # and the token after it. - Toks[0] = SavedHash; -@@ -1376,7 +1433,7 @@ static bool GetLineValue(Token &DigitTok, unsigned &Val, - /// # line digit-sequence - /// # line digit-sequence "s-char-sequence" - /// \endverbatim --void Preprocessor::HandleLineDirective() { -+void Preprocessor::HandleLineDirective(const Token &SavedHash) { // PASTA PATCH - // Read the line # and string argument. Per C99 6.10.4p5, these tokens are - // expanded. - Token DigitTok; -@@ -1538,7 +1595,8 @@ static bool ReadLineMarkerFlags(bool &IsFileEntry, bool &IsFileExit, - /// # 42 "file" ('1' | '2')? - /// # 42 "file" ('1' | '2')? '3' '4'? - /// --void Preprocessor::HandleDigitDirective(Token &DigitTok) { -+void Preprocessor::HandleDigitDirective( // PASTA PATCH -+ const Token &SavedHash, Token &DigitTok) { // PASTA PATCH - // Validate the number and convert it to an unsigned. GNU does not have a - // line # limit other than it fit in 32-bits. - unsigned LineNo; -@@ -1614,8 +1672,8 @@ void Preprocessor::HandleDigitDirective(Token &DigitTok) { - - /// HandleUserDiagnosticDirective - Handle a #warning or #error directive. - /// --void Preprocessor::HandleUserDiagnosticDirective(Token &Tok, -- bool isWarning) { -+void Preprocessor::HandleUserDiagnosticDirective(const Token &SavedHash, // PASTA PATCH -+ Token &Tok, bool isWarning) { // PASTA PATCH - // Read the rest of the line raw. We do this because we don't want macros - // to be expanded and we don't require that the tokens be valid preprocessing - // tokens. For example, this is allowed: "#warning ` 'foo". GCC does -@@ -1636,7 +1694,8 @@ void Preprocessor::HandleUserDiagnosticDirective(Token &Tok, - - /// HandleIdentSCCSDirective - Handle a #ident/#sccs directive. - /// --void Preprocessor::HandleIdentSCCSDirective(Token &Tok) { -+void Preprocessor::HandleIdentSCCSDirective(const Token &SavedHash, // PASTA PATCH -+ Token &Tok) { // PASTA PATCH - // Yes, this directive is an extension. - Diag(Tok, diag::ext_pp_ident_directive); - -@@ -1671,7 +1730,8 @@ void Preprocessor::HandleIdentSCCSDirective(Token &Tok) { - } - - /// Handle a #public directive. --void Preprocessor::HandleMacroPublicDirective(Token &Tok) { -+void Preprocessor::HandleMacroPublicDirective(const Token &SavedHash, // PASTA PATCH -+ Token &Tok) { // PASTA PATCH - Token MacroNameTok; - ReadMacroName(MacroNameTok, MU_Undef); - -@@ -1698,7 +1758,7 @@ void Preprocessor::HandleMacroPublicDirective(Token &Tok) { - } - - /// Handle a #private directive. --void Preprocessor::HandleMacroPrivateDirective() { -+void Preprocessor::HandleMacroPrivateDirective(const Token &SavedHash) { // PASTA PATCH - Token MacroNameTok; - ReadMacroName(MacroNameTok, MU_Undef); - -@@ -1935,10 +1995,11 @@ Preprocessor::getIncludeNextStart(const Token &IncludeNextTok) const { - /// routine with functionality shared between \#include, \#include_next and - /// \#import. LookupFrom is set when this is a \#include_next directive, it - /// specifies the file to start searching from. --void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc, -+void Preprocessor::HandleIncludeDirective(const Token &SavedHash, // PASTA PATCH - Token &IncludeTok, - ConstSearchDirIterator LookupFrom, - const FileEntry *LookupFromFile) { -+ SourceLocation HashLoc = SavedHash.getLocation(); // PASTA PATCH - Token FilenameTok; - if (LexHeaderName(FilenameTok)) - return; -@@ -2563,7 +2624,7 @@ Preprocessor::ImportAction Preprocessor::HandleHeaderIncludeOrImport( - - /// HandleIncludeNextDirective - Implements \#include_next. - /// --void Preprocessor::HandleIncludeNextDirective(SourceLocation HashLoc, -+void Preprocessor::HandleIncludeNextDirective(const Token &SavedHash, // PASTA PATCH - Token &IncludeNextTok) { - Diag(IncludeNextTok, diag::ext_pp_include_next_directive); - -@@ -2571,7 +2632,7 @@ void Preprocessor::HandleIncludeNextDirective(SourceLocation HashLoc, - const FileEntry *LookupFromFile; - std::tie(Lookup, LookupFromFile) = getIncludeNextStart(IncludeNextTok); - -- return HandleIncludeDirective(HashLoc, IncludeNextTok, Lookup, -+ return HandleIncludeDirective(SavedHash, IncludeNextTok, Lookup, - LookupFromFile); - } - -@@ -2591,21 +2652,21 @@ void Preprocessor::HandleMicrosoftImportDirective(Token &Tok) { - - /// HandleImportDirective - Implements \#import. - /// --void Preprocessor::HandleImportDirective(SourceLocation HashLoc, -- Token &ImportTok) { -+void Preprocessor::HandleImportDirective(const Token &SavedHash, // PASTA PATCH -+ Token &ImportTok) { // PASTA PATCH - if (!LangOpts.ObjC) { // #import is standard for ObjC. - if (LangOpts.MSVCCompat) - return HandleMicrosoftImportDirective(ImportTok); - Diag(ImportTok, diag::ext_pp_import_directive); - } -- return HandleIncludeDirective(HashLoc, ImportTok); -+ return HandleIncludeDirective(SavedHash, ImportTok); // PASTA PATCH - } - - /// HandleIncludeMacrosDirective - The -imacros command line option turns into a - /// pseudo directive in the predefines buffer. This handles it by sucking all - /// tokens through the preprocessor and discarding them (only keeping the side - /// effects on the preprocessor). --void Preprocessor::HandleIncludeMacrosDirective(SourceLocation HashLoc, -+void Preprocessor::HandleIncludeMacrosDirective(const Token &SavedHash, // PASTA PATCH - Token &IncludeMacrosTok) { - // This directive should only occur in the predefines buffer. If not, emit an - // error and reject it. -@@ -2619,7 +2680,7 @@ void Preprocessor::HandleIncludeMacrosDirective(SourceLocation HashLoc, - - // Treat this as a normal #include for checking purposes. If this is - // successful, it will push a new lexer onto the include stack. -- HandleIncludeDirective(HashLoc, IncludeMacrosTok); -+ HandleIncludeDirective(SavedHash, IncludeMacrosTok); // PASTA PATCH - - Token TmpTok; - do { -@@ -3008,7 +3069,8 @@ MacroInfo *Preprocessor::ReadOptionalMacroParameterListAndBody( - /// HandleDefineDirective - Implements \#define. This consumes the entire macro - /// line then lets the caller lex the next real token. - void Preprocessor::HandleDefineDirective( -- Token &DefineTok, const bool ImmediatelyAfterHeaderGuard) { -+ const Token &SavedHash, Token &DefineTok, // PASTA PATCH -+ const bool ImmediatelyAfterHeaderGuard) { // PASTA PATCH - ++NumDefined; - - Token MacroNameTok; -@@ -3162,7 +3224,7 @@ void Preprocessor::HandleDefineDirective( - - /// HandleUndefDirective - Implements \#undef. - /// --void Preprocessor::HandleUndefDirective() { -+void Preprocessor::HandleUndefDirective(const Token &SavedHash) { // PASTA PATCH - ++NumUndefined; - - Token MacroNameTok; -@@ -3226,7 +3288,7 @@ void Preprocessor::HandleIfdefDirective(Token &Result, - if (MacroNameTok.is(tok::eod)) { - // Skip code until we get to #endif. This helps with recovery by not - // emitting an error when the #endif is reached. -- SkipExcludedConditionalBlock(HashToken.getLocation(), -+ SkipExcludedConditionalBlock(HashToken, // PASTA PATCH - DirectiveTok.getLocation(), - /*Foundnonskip*/ false, /*FoundElse*/ false); - return; -@@ -3281,7 +3343,7 @@ void Preprocessor::HandleIfdefDirective(Token &Result, - /*foundelse*/false); - } else { - // No, skip the contents of this block. -- SkipExcludedConditionalBlock(HashToken.getLocation(), -+ SkipExcludedConditionalBlock(HashToken, // PASTA PATCH - DirectiveTok.getLocation(), - /*Foundnonskip*/ false, - /*FoundElse*/ false); -@@ -3334,7 +3396,7 @@ void Preprocessor::HandleIfDirective(Token &IfToken, - /*foundnonskip*/true, /*foundelse*/false); - } else { - // No, skip the contents of this block. -- SkipExcludedConditionalBlock(HashToken.getLocation(), IfToken.getLocation(), -+ SkipExcludedConditionalBlock(HashToken, IfToken.getLocation(), // PASTA PATCH - /*Foundnonskip*/ false, - /*FoundElse*/ false); - } -@@ -3402,7 +3464,7 @@ void Preprocessor::HandleElseDirective(Token &Result, const Token &HashToken) { - } - - // Finally, skip the rest of the contents of this block. -- SkipExcludedConditionalBlock(HashToken.getLocation(), CI.IfLoc, -+ SkipExcludedConditionalBlock(HashToken, CI.IfLoc, // PASTA PATCH - /*Foundnonskip*/ true, - /*FoundElse*/ true, Result.getLocation()); - } -@@ -3483,6 +3545,6 @@ void Preprocessor::HandleElifFamilyDirective(Token &ElifToken, - - // Finally, skip the rest of the contents of this block. - SkipExcludedConditionalBlock( -- HashToken.getLocation(), CI.IfLoc, /*Foundnonskip*/ true, -+ HashToken, CI.IfLoc, /*Foundnonskip*/ true, // PASTA PATCH - /*FoundElse*/ CI.FoundElse, ElifToken.getLocation()); - } -diff --git a/clang/lib/Lex/PPExpressions.cpp b/clang/lib/Lex/PPExpressions.cpp -index aa411cfc5..d268bc3c7 100644 ---- a/clang/lib/Lex/PPExpressions.cpp -+++ b/clang/lib/Lex/PPExpressions.cpp -@@ -25,6 +25,7 @@ - #include "clang/Lex/LiteralSupport.h" - #include "clang/Lex/MacroInfo.h" - #include "clang/Lex/PPCallbacks.h" -+#include "clang/Lex/PPCallbacksEventKind.h" // PASTA PATCH - #include "clang/Lex/Preprocessor.h" - #include "clang/Lex/Token.h" - #include "llvm/ADT/APSInt.h" -@@ -101,6 +102,32 @@ struct DefinedTracker { - /// EvaluateDefined - Process a 'defined(sym)' expression. - static bool EvaluateDefined(PPValue &Result, Token &PeekTok, DefinedTracker &DT, - bool ValueLive, Preprocessor &PP) { -+ // PASTA PATCH -+ PPCallbacks *Callbacks = PP.getPPCallbacks(); -+ Token SavedStart = PeekTok; -+ if (Callbacks) { -+ Callbacks->Event(SavedStart, PPCallbacks::BeginDelayedSubstitution, 0); -+ } -+ auto Expand = [&] (void) { -+ if (Callbacks) { -+ Token ResultTok; -+ ResultTok.startToken(); -+ ResultTok.setKind(tok::numeric_constant); -+ const char *ResultStr = Result.Val.getExtValue() ? "1" : "0"; -+ PP.CreateString(ResultStr, ResultTok, SavedStart.getLocation(), -+ PeekTok.getEndLoc()); -+ -+ Callbacks->Event(PeekTok, PPCallbacks::SwitchToSubstitution, -+ reinterpret_cast(&SavedStart)); -+ -+ Callbacks->Event(ResultTok, PPCallbacks::TokenFromTokenLexer, -+ SavedStart.getLocation().getRawEncoding()); -+ -+ Callbacks->Event(SavedStart, PPCallbacks::EndSubstitution, -+ reinterpret_cast(&SavedStart)); -+ } -+ }; -+ - SourceLocation beginLoc(PeekTok.getLocation()); - Result.setBegin(beginLoc); - -@@ -154,12 +181,14 @@ static bool EvaluateDefined(PPValue &Result, Token &PeekTok, DefinedTracker &DT, - PP.Diag(LParenLoc, diag::note_matching) << tok::l_paren; - return true; - } -+ Expand(); // PASTA PATCH - // Consume the ). - PP.LexNonComment(PeekTok); - Result.setEnd(PeekTok.getLocation()); - } else { - // Consume identifier. - Result.setEnd(PeekTok.getLocation()); -+ Expand(); // PASTA PATCH - PP.LexNonComment(PeekTok); - } - -diff --git a/clang/lib/Lex/PPLexerChange.cpp b/clang/lib/Lex/PPLexerChange.cpp -index 66168467e..95228691b 100644 ---- a/clang/lib/Lex/PPLexerChange.cpp -+++ b/clang/lib/Lex/PPLexerChange.cpp -@@ -16,6 +16,7 @@ - #include "clang/Lex/HeaderSearch.h" - #include "clang/Lex/LexDiagnostic.h" - #include "clang/Lex/MacroInfo.h" -+#include "clang/Lex/PPCallbacksEventKind.h" // PASTA PATCH - #include "clang/Lex/Preprocessor.h" - #include "clang/Lex/PreprocessorOptions.h" - #include "llvm/ADT/StringSwitch.h" -@@ -30,6 +31,36 @@ using namespace clang; - // Miscellaneous Methods. - //===----------------------------------------------------------------------===// - -+void Preprocessor::PopIncludeMacroStack() { -+ -+ // PASTA PATCH: Tell us about the end of macro expansions. -+ if (Callbacks && CurTokenLexer && CurTokenLexer->Macro) { -+ Callbacks->Event(CurTokenLexer->MacroNameTok, -+ PPCallbacks::EndMacroExpansion, -+ reinterpret_cast(CurTokenLexer->Macro)); -+ } -+ -+ // PASTA PATCH: Make us aware of the end of `_Pragma` handling. -+ if (Callbacks && CurPPLexer && CurLexer.get() == CurPPLexer && -+ CurLexer->isPragmaLexer()) { -+ Token PragmaTok; -+ PragmaTok.setKind(tok::raw_identifier); -+ PragmaTok.setLocation(SourceMgr.getExpansionLoc(CurLexer->getFileLoc())); -+ PragmaTok.setLength(7u); -+ PragmaTok.setRawIdentifierData( -+ SourceMgr.getCharacterData(PragmaTok.getLocation(), nullptr)); -+ Callbacks->Event(PragmaTok, PPCallbacks::EndMacroExpansion, 0); -+ } -+ -+ CurLexer = std::move(IncludeMacroStack.back().TheLexer); -+ CurPPLexer = IncludeMacroStack.back().ThePPLexer; -+ CurTokenLexer = std::move(IncludeMacroStack.back().TheTokenLexer); -+ CurDirLookup = IncludeMacroStack.back().TheDirLookup; -+ CurLexerSubmodule = IncludeMacroStack.back().TheSubmodule; -+ CurLexerKind = IncludeMacroStack.back().CurLexerKind; -+ IncludeMacroStack.pop_back(); -+} -+ - /// isInPrimaryFile - Return true if we're in the top-level file, not in a - /// \#include. This looks through macro expansions and active _Pragma lexers. - bool Preprocessor::isInPrimaryFile() const { -@@ -588,6 +619,14 @@ bool Preprocessor::HandleEndOfTokenLexer(Token &Result) { - assert(CurTokenLexer && !CurPPLexer && - "Ending a macro when currently in a #include file!"); - -+ // PASTA PATCH: Tell us about the end of macro expansions. -+ if (CurTokenLexer && CurTokenLexer->Macro) { -+ Callbacks->Event(CurTokenLexer->MacroNameTok, -+ PPCallbacks::EndMacroExpansion, -+ reinterpret_cast(CurTokenLexer->Macro)); -+ CurTokenLexer->Macro = nullptr; -+ } -+ - if (!MacroExpandingLexersStack.empty() && - MacroExpandingLexersStack.back().first == CurTokenLexer.get()) - removeCachedMacroExpandedTokensOfLastLexer(); -diff --git a/clang/lib/Lex/PPMacroExpansion.cpp b/clang/lib/Lex/PPMacroExpansion.cpp -index 76d0d53ed..b278cbbed 100644 ---- a/clang/lib/Lex/PPMacroExpansion.cpp -+++ b/clang/lib/Lex/PPMacroExpansion.cpp -@@ -29,6 +29,7 @@ - #include "clang/Lex/LiteralSupport.h" - #include "clang/Lex/MacroArgs.h" - #include "clang/Lex/MacroInfo.h" -+#include "clang/Lex/PPCallbacksEventKind.h" // PASTA PATCH - #include "clang/Lex/Preprocessor.h" - #include "clang/Lex/PreprocessorLexer.h" - #include "clang/Lex/PreprocessorOptions.h" -@@ -59,6 +60,58 @@ - - using namespace clang; - -+namespace { -+ -+// PASTA PATCH: Tell us about the end of macro expansions. We have to add -+// this as a post-lex action, so that we can observe what is actually lexed -+// into `Identifier`. -+struct DeferExpansionEnd { -+ std::function &PostLexAction; -+ PPCallbacks * const Callbacks; -+ Token Identifier; -+ MacroInfo * const MI; -+ -+ DeferExpansionEnd(std::function &PostLexAction_, -+ PPCallbacks *Callbacks_, Token Identifier_, MacroInfo *MI_) -+ : PostLexAction(PostLexAction_), -+ Callbacks(Callbacks_), -+ Identifier(std::move(Identifier_)), -+ MI(MI_) {} -+ -+ ~DeferExpansionEnd(void) { -+ if (Callbacks) { -+ PostLexAction = [CB = Callbacks, Ident = std::move(Identifier), Info = MI, -+ PrevPostLexAction = std::move(PostLexAction)] -+ (const Token &Tok) { -+ CB->Event(Ident, PPCallbacks::EndMacroExpansion, -+ reinterpret_cast(Info)); -+ PrevPostLexAction(Tok); -+ }; -+ } -+ } -+}; -+ -+struct DeferExpansionCancellation { -+ std::function Action; -+ bool DoCancel{true}; -+ -+ template -+ DeferExpansionCancellation(CB Action_) -+ : Action(std::move(Action_)) {} -+ -+ ~DeferExpansionCancellation(void) { -+ if (DoCancel) { -+ Action(); -+ } -+ } -+ -+ void DisableCancellation(void) { -+ DoCancel = false; -+ } -+}; -+ -+} // namespace -+ - MacroDirective * - Preprocessor::getLocalMacroDirectiveHistory(const IdentifierInfo *II) const { - if (!II->hadMacroDefinition()) -@@ -480,6 +533,19 @@ bool Preprocessor::HandleMacroExpandedIdentifier(Token &Identifier, - - MacroInfo *MI = M.getMacroInfo(); - -+ // PASTA PATCH: Visibility into macro expansion. -+ Token SavedIdentifier = Identifier; -+ if (Callbacks) { -+ Callbacks->Event(SavedIdentifier, PPCallbacks::BeginMacroExpansion, -+ reinterpret_cast(MI)); -+ } -+ DeferExpansionCancellation CancelExpansion([&, this] (void) { -+ if (Callbacks) { -+ Callbacks->Event(SavedIdentifier, PPCallbacks::CancelExpansion, -+ reinterpret_cast(MI)); -+ } -+ }); -+ - // If this is a macro expansion in the "#if !defined(x)" line for the file, - // then the macro could expand to different things in other contexts, we need - // to disable the optimization in this case. -@@ -490,6 +556,8 @@ bool Preprocessor::HandleMacroExpandedIdentifier(Token &Identifier, - if (Callbacks) - Callbacks->MacroExpands(Identifier, M, Identifier.getLocation(), - /*Args=*/nullptr); -+ -+ CancelExpansion.DisableCancellation(); // PASTA PATCH - ExpandBuiltinMacro(Identifier); - return true; - } -@@ -511,7 +579,22 @@ bool Preprocessor::HandleMacroExpandedIdentifier(Token &Identifier, - InMacroArgs = true; - ArgMacro = &Identifier; - -- Args = ReadMacroCallArgumentList(Identifier, MI, ExpansionEnd); -+ // PASTA PATCH: Visibility into macro expansion. -+ if (Callbacks) { -+ Callbacks->Event(SavedIdentifier, PPCallbacks::BeginMacroCallArgumentList, -+ reinterpret_cast(MI)); -+ } -+ -+ // PASTA PATCH: Visibility to last token in argument list. -+ Token ExpansionEndTok; -+ Args = ReadMacroCallArgumentList(Identifier, MI, ExpansionEndTok); -+ ExpansionEnd = ExpansionEndTok.getLocation(); -+ -+ // PASTA PATCH: Visibility into macro expansion. -+ if (Callbacks) { -+ Callbacks->Event(ExpansionEndTok, PPCallbacks::EndMacroCallArgumentList, -+ reinterpret_cast(Args)); -+ } - - // Finished parsing args. - InMacroArgs = false; -@@ -570,7 +653,7 @@ bool Preprocessor::HandleMacroExpandedIdentifier(Token &Identifier, - - // If this macro expands to no tokens, don't bother to push it onto the - // expansion stack, only to take it right back off. -- if (MI->getNumTokens() == 0) { -+ if (false && MI->getNumTokens() == 0) { // PASTA PATCH - // No need for arg info. - if (Args) Args->destroy(*this); - -@@ -580,7 +663,7 @@ bool Preprocessor::HandleMacroExpandedIdentifier(Token &Identifier, - PropagateLineStartLeadingSpaceInfo(Identifier); - ++NumFastMacroExpanded; - return false; -- } else if (MI->getNumTokens() == 1 && -+ } else if (false && MI->getNumTokens() == 1 && // PASTA PATCH - isTrivialSingleTokenExpansion(MI, Identifier.getIdentifierInfo(), - *this)) { - // Otherwise, if this macro expands into a single trivially-expanded -@@ -628,6 +711,13 @@ bool Preprocessor::HandleMacroExpandedIdentifier(Token &Identifier, - return true; - } - -+ // PASTA PATCH: Switch state to now start expanding. -+ CancelExpansion.DisableCancellation(); -+ if (Callbacks) { -+ Callbacks->Event(SavedIdentifier, PPCallbacks::SwitchToExpansion, -+ reinterpret_cast(MI)); -+ } -+ - // Start expanding the macro. - EnterMacro(Identifier, ExpansionEnd, MI, Args); - return false; -@@ -767,13 +857,13 @@ static bool GenerateNewArgTokens(Preprocessor &PP, - /// error. - MacroArgs *Preprocessor::ReadMacroCallArgumentList(Token &MacroName, - MacroInfo *MI, -- SourceLocation &MacroEnd) { -+ Token &Tok) { // PASTA PATCH - // The number of fixed arguments to parse. - unsigned NumFixedArgsLeft = MI->getNumParams(); - bool isVariadic = MI->isVariadic(); - - // Outer loop, while there are more arguments, keep reading them. -- Token Tok; -+ // Token Tok; // PASTA PATCH: Taken from argument. - - // Read arguments as unexpanded tokens. This avoids issues, e.g., where - // an argument value in a macro could expand to ',' or '(' or ')'. -@@ -786,6 +876,9 @@ MacroArgs *Preprocessor::ReadMacroCallArgumentList(Token &MacroName, - SmallVector ArgTokens; - bool ContainsCodeCompletionTok = false; - bool FoundElidedComma = false; -+ bool InVariadicSection = false; // PASTA PATCH -+ bool InArgument = false; // PASTA PATCH -+ Token ArgStartTok; // PASTA PATCH - - SourceLocation TooManyArgsLoc; - -@@ -800,6 +893,21 @@ MacroArgs *Preprocessor::ReadMacroCallArgumentList(Token &MacroName, - size_t ArgTokenStart = ArgTokens.size(); - SourceLocation ArgStartLoc = Tok.getLocation(); - -+ // PASTA PATCH: Visibility into macro expansion. -+ ArgStartTok = Tok; -+ if (Callbacks) { -+ if (isVariadic && !NumFixedArgsLeft) { -+ Callbacks->Event(ArgStartTok, PPCallbacks::BeginVariadicCallArgumentList, -+ reinterpret_cast(&MacroName)); -+ InVariadicSection = true; -+ } -+ if (NumFixedArgsLeft) { -+ Callbacks->Event(ArgStartTok, PPCallbacks::BeginMacroCallArgument, -+ reinterpret_cast(&MacroName)); -+ InArgument = true; -+ } -+ } -+ - // C99 6.10.3p11: Keep track of the number of l_parens we have seen. Note - // that we already consumed the first one. - unsigned NumParens = 0; -@@ -826,7 +934,7 @@ MacroArgs *Preprocessor::ReadMacroCallArgumentList(Token &MacroName, - } else if (Tok.is(tok::r_paren)) { - // If we found the ) token, the macro arg list is done. - if (NumParens-- == 0) { -- MacroEnd = Tok.getLocation(); -+ // MacroEnd = Tok.getLocation(); // PASTA PATCH - if (!ArgTokens.empty() && - ArgTokens.back().commaAfterElided()) { - FoundElidedComma = true; -@@ -854,6 +962,27 @@ MacroArgs *Preprocessor::ReadMacroCallArgumentList(Token &MacroName, - break; - if (NumFixedArgsLeft > 1) - break; -+ -+ // PASTA PATCH: Visibility into variadic macro expansion. -+ if (Callbacks) { -+ if (InArgument) { -+ Callbacks->Event(ArgStartTok, PPCallbacks::EndMacroCallArgument, -+ reinterpret_cast(&Tok)); -+ InArgument = false; -+ } -+ ArgStartTok = Tok; -+ -+ if (!InVariadicSection) { -+ Callbacks->Event( -+ ArgStartTok, PPCallbacks::BeginVariadicCallArgumentList, -+ reinterpret_cast(&MacroName)); -+ InVariadicSection = true; -+ } -+ -+ Callbacks->Event(ArgStartTok, PPCallbacks::BeginMacroCallArgument, -+ reinterpret_cast(&MacroName)); -+ InArgument = true; -+ } - } - } else if (Tok.is(tok::comment) && !KeepMacroComments) { - // If this is a comment token in the argument list and we're just in -@@ -903,6 +1032,15 @@ MacroArgs *Preprocessor::ReadMacroCallArgumentList(Token &MacroName, - ? diag::warn_cxx98_compat_empty_fnmacro_arg - : diag::ext_empty_fnmacro_arg); - -+ // PASTA PATCH: -+ if (Callbacks) { -+ if (InArgument) { -+ InArgument = false; -+ Callbacks->Event(ArgStartTok, PPCallbacks::EndMacroCallArgument, -+ reinterpret_cast(&Tok)); -+ } -+ } -+ - // Add a marker EOF token to the end of the token list for this argument. - Token EOFTok; - EOFTok.startToken(); -@@ -915,6 +1053,18 @@ MacroArgs *Preprocessor::ReadMacroCallArgumentList(Token &MacroName, - --NumFixedArgsLeft; - } - -+ if (Callbacks) { -+ if (InArgument) { -+ InArgument = false; -+ Callbacks->Event(ArgStartTok, PPCallbacks::EndMacroCallArgument, -+ reinterpret_cast(&Tok)); -+ } -+ if (InVariadicSection) { -+ Callbacks->Event(ArgStartTok, PPCallbacks::EndVariadicCallArgumentList, -+ reinterpret_cast(&Tok)); -+ } -+ } -+ - // Okay, we either found the r_paren. Check to see if we parsed too few - // arguments. - unsigned MinArgsExpected = MI->getNumParams(); -@@ -1182,6 +1332,17 @@ static bool EvaluateHasIncludeCommon(Token &Tok, IdentifierInfo *II, - Preprocessor &PP, - ConstSearchDirIterator LookupFrom, - const FileEntry *LookupFromFile) { -+ // PASTA PATCH: Visibility into macro expansion. -+ PPCallbacks *Callbacks = PP.getPPCallbacks(); -+ Token SavedIdentifier = Tok; -+ SavedIdentifier.setIdentifierInfo(II); -+ MacroDefinition MD = PP.getMacroDefinition(II); -+ MacroInfo *MI = MD.getMacroInfo(); -+ if (Callbacks) { -+ Callbacks->Event(SavedIdentifier, PPCallbacks::BeginMacroCallArgumentList, -+ reinterpret_cast(MI)); -+ } -+ - // Save the location of the current token. If a '(' is later found, use - // that location. If not, use the end of this location instead. - SourceLocation LParenLoc = Tok.getLocation(); -@@ -1201,6 +1362,8 @@ static bool EvaluateHasIncludeCommon(Token &Tok, IdentifierInfo *II, - return false; - } while (Tok.getKind() == tok::comment); - -+ Token LParenTok = Tok; // PASTA PATCH -+ - // Ensure we have a '('. - if (Tok.isNot(tok::l_paren)) { - // No '(', use end of last token. -@@ -1211,6 +1374,10 @@ static bool EvaluateHasIncludeCommon(Token &Tok, IdentifierInfo *II, - if (Tok.isNot(tok::header_name)) - return false; - } else { -+ if (Callbacks) { // PASTA PATCH -+ Callbacks->Event(LParenTok, PPCallbacks::BeginMacroCallArgument, -+ reinterpret_cast(&SavedIdentifier)); -+ } - // Save '(' location for possible missing ')' message. - LParenLoc = Tok.getLocation(); - if (PP.LexHeaderName(Tok)) -@@ -1242,6 +1409,16 @@ static bool EvaluateHasIncludeCommon(Token &Tok, IdentifierInfo *II, - return false; - } - -+ // PASTA PATCH -+ if (Callbacks) { -+ Callbacks->Event(LParenTok, PPCallbacks::EndMacroCallArgument, -+ reinterpret_cast(&Tok)); -+ Callbacks->Event(LParenTok, PPCallbacks::EndMacroCallArgumentList, -+ 0); -+ Callbacks->Event(SavedIdentifier, PPCallbacks::SwitchToExpansion, -+ reinterpret_cast(MI)); -+ } -+ - bool isAngled = PP.GetIncludeFilenameSpelling(Tok.getLocation(), Filename); - // If GetIncludeFilenameSpelling set the start ptr to null, there was an - // error. -@@ -1285,6 +1462,13 @@ static void EvaluateFeatureLikeBuiltinMacro(llvm::raw_svector_ostream& OS, - llvm::function_ref< - int(Token &Tok, - bool &HasLexedNextTok)> Op) { -+ // PASTA PATCH: Visibility into macro expansion. -+ PPCallbacks *Callbacks = PP.getPPCallbacks(); -+ Token SavedIdentifier = Tok; -+ SavedIdentifier.setIdentifierInfo(II); -+ MacroDefinition MD = PP.getMacroDefinition(II); -+ MacroInfo *MI = MD.getMacroInfo(); -+ - // Parse the initial '('. - PP.LexUnexpandedToken(Tok); - if (Tok.isNot(tok::l_paren)) { -@@ -1303,6 +1487,15 @@ static void EvaluateFeatureLikeBuiltinMacro(llvm::raw_svector_ostream& OS, - SourceLocation LParenLoc = Tok.getLocation(); - std::optional Result; - -+ // PASTA PATCH -+ clang::Token ArgSepTok = Tok; -+ if (Callbacks) { -+ Callbacks->Event(SavedIdentifier, PPCallbacks::BeginMacroCallArgumentList, -+ reinterpret_cast(MI)); -+ Callbacks->Event(ArgSepTok, PPCallbacks::BeginMacroCallArgument, -+ reinterpret_cast(&SavedIdentifier)); -+ } -+ - Token ResultTok; - bool SuppressDiagnostic = false; - while (true) { -@@ -1322,6 +1515,15 @@ already_lexed: - return; - - case tok::comma: -+ // PASTA PATCH -+ if (ParenDepth == 1 && Callbacks) { -+ Callbacks->Event(ArgSepTok, PPCallbacks::EndMacroCallArgument, -+ reinterpret_cast(&Tok)); -+ ArgSepTok = Tok; -+ Callbacks->Event(ArgSepTok, PPCallbacks::BeginMacroCallArgument, -+ reinterpret_cast(&SavedIdentifier)); -+ } -+ - if (!SuppressDiagnostic) { - PP.Diag(Tok.getLocation(), diag::err_too_many_args_in_macro_invoc); - SuppressDiagnostic = true; -@@ -1342,6 +1544,15 @@ already_lexed: - if (--ParenDepth > 0) - continue; - -+ // PASTA PATCH -+ if (Callbacks) { -+ Callbacks->Event(ArgSepTok, PPCallbacks::EndMacroCallArgument, -+ reinterpret_cast(&Tok)); -+ Callbacks->Event(ArgSepTok, PPCallbacks::EndMacroCallArgumentList, 0); -+ Callbacks->Event(SavedIdentifier, PPCallbacks::SwitchToExpansion, -+ reinterpret_cast(MI)); -+ } -+ - // The last ')' has been reached; return the value if one found or - // a diagnostic and a dummy value. - if (Result) { -@@ -1513,7 +1724,22 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) { - bool IsAtStartOfLine = Tok.isAtStartOfLine(); - bool HasLeadingSpace = Tok.hasLeadingSpace(); - -+ // PASTA PATCH: -+ Token SavedIdentifier = Tok; -+ SavedIdentifier.setIdentifierInfo(II); -+ MacroDefinition MD = getMacroDefinition(II); -+ MacroInfo *MI = MD.getMacroInfo(); -+ DeferExpansionEnd NotifyMacroEnd(PostLexAction, Callbacks.get(), -+ SavedIdentifier, MI); -+ auto SwitchToExpansion = [&] (void) { -+ if (Callbacks) { -+ Callbacks->Event(SavedIdentifier, PPCallbacks::SwitchToExpansion, -+ reinterpret_cast(MI)); -+ } -+ }; -+ - if (II == Ident__LINE__) { -+ SwitchToExpansion(); // PASTA PATCH - // C99 6.10.8: "__LINE__: The presumed line number (within the current - // source file) of the current source line (an integer constant)". This can - // be affected by #line. -@@ -1536,6 +1762,7 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) { - Tok.setKind(tok::numeric_constant); - } else if (II == Ident__FILE__ || II == Ident__BASE_FILE__ || - II == Ident__FILE_NAME__) { -+ SwitchToExpansion(); // PASTA PATCH - // C99 6.10.8: "__FILE__: The presumed name of the current source file (a - // character string literal)". This can be affected by #line. - PresumedLoc PLoc = SourceMgr.getPresumedLoc(Tok.getLocation()); -@@ -1575,6 +1802,7 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) { - } - Tok.setKind(tok::string_literal); - } else if (II == Ident__DATE__) { -+ SwitchToExpansion(); // PASTA PATCH - Diag(Tok.getLocation(), diag::warn_pp_date_time); - if (!DATELoc.isValid()) - ComputeDATE_TIME(DATELoc, TIMELoc, *this); -@@ -1585,6 +1813,7 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) { - Tok.getLength())); - return; - } else if (II == Ident__TIME__) { -+ SwitchToExpansion(); // PASTA PATCH - Diag(Tok.getLocation(), diag::warn_pp_date_time); - if (!TIMELoc.isValid()) - ComputeDATE_TIME(DATELoc, TIMELoc, *this); -@@ -1595,6 +1824,7 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) { - Tok.getLength())); - return; - } else if (II == Ident__INCLUDE_LEVEL__) { -+ SwitchToExpansion(); // PASTA PATCH - // Compute the presumed include depth of this token. This can be affected - // by GNU line markers. - unsigned Depth = 0; -@@ -1610,6 +1840,7 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) { - OS << Depth; - Tok.setKind(tok::numeric_constant); - } else if (II == Ident__TIMESTAMP__) { -+ SwitchToExpansion(); // PASTA PATCH - Diag(Tok.getLocation(), diag::warn_pp_date_time); - // MSVC, ICC, GCC, VisualAge C++ extension. The generated string should be - // of the form "Ddd Mmm dd hh::mm::ss yyyy", which is returned by asctime. -@@ -1636,6 +1867,7 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) { - OS << '"' << StringRef(Result).drop_back() << '"'; - Tok.setKind(tok::string_literal); - } else if (II == Ident__FLT_EVAL_METHOD__) { -+ SwitchToExpansion(); // PASTA PATCH - // __FLT_EVAL_METHOD__ is set to the default value. - OS << getTUFPEvalMethod(); - // __FLT_EVAL_METHOD__ expands to a simple numeric value. -@@ -1647,6 +1879,7 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) { - Diag(getLastFPEvalPragmaLocation(), diag::note_pragma_entered_here); - } - } else if (II == Ident__COUNTER__) { -+ SwitchToExpansion(); // PASTA PATCH - // __COUNTER__ expands to a simple numeric value. - OS << CounterValue++; - Tok.setKind(tok::numeric_constant); -@@ -1848,6 +2081,7 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) { - (II->getName() == getLangOpts().CurrentModule); - }); - } else if (II == Ident__MODULE__) { -+ SwitchToExpansion(); // PASTA PATCH - // The current module as an identifier. - OS << getLangOpts().CurrentModule; - IdentifierInfo *ModuleII = getIdentifierInfo(getLangOpts().CurrentModule); -@@ -1856,6 +2090,12 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) { - } else if (II == Ident__identifier) { - SourceLocation Loc = Tok.getLocation(); - -+ // PASTA PATCH -+ if (Callbacks) { -+ Callbacks->Event(SavedIdentifier, PPCallbacks::BeginMacroCallArgumentList, -+ 0); -+ } -+ - // We're expecting '__identifier' '(' identifier ')'. Try to recover - // if the parens are missing. - LexNonComment(Tok); -@@ -1869,6 +2109,13 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) { - return; - } - -+ // PASTA PATCH -+ Token LParenTok = Tok; -+ if (Callbacks) { -+ Callbacks->Event(LParenTok, PPCallbacks::BeginMacroCallArgument, -+ reinterpret_cast(&SavedIdentifier)); -+ } -+ - SourceLocation LParenLoc = Tok.getLocation(); - LexNonComment(Tok); - -@@ -1897,6 +2144,15 @@ void Preprocessor::ExpandBuiltinMacro(Token &Tok) { - << Tok.getKind() << tok::r_paren; - Diag(LParenLoc, diag::note_matching) << tok::l_paren; - } -+ -+ // PASTA PATCH -+ if (Callbacks) { -+ Callbacks->Event(LParenTok, PPCallbacks::EndMacroCallArgument, -+ reinterpret_cast(&Tok)); -+ Callbacks->Event(Tok, PPCallbacks::EndMacroCallArgumentList, 0); -+ } -+ SwitchToExpansion(); // PASTA PATCH -+ - return; - } else if (II == Ident__is_target_arch) { - EvaluateFeatureLikeBuiltinMacro( -diff --git a/clang/lib/Lex/Pragma.cpp b/clang/lib/Lex/Pragma.cpp -index 4da9d1603..5a18f5220 100644 ---- a/clang/lib/Lex/Pragma.cpp -+++ b/clang/lib/Lex/Pragma.cpp -@@ -29,6 +29,7 @@ - #include "clang/Lex/MacroInfo.h" - #include "clang/Lex/ModuleLoader.h" - #include "clang/Lex/PPCallbacks.h" -+#include "clang/Lex/PPCallbacksEventKind.h" // PASTA PATCH - #include "clang/Lex/Preprocessor.h" - #include "clang/Lex/PreprocessorLexer.h" - #include "clang/Lex/PreprocessorOptions.h" -@@ -207,10 +208,27 @@ void Preprocessor::Handle_Pragma(Token &Tok) { - // In Case #2, we check the syntax now, but then put the tokens back into the - // token stream for later consumption. - -+ // PASTA PATCH -+ // If we're expanding a macro argument, let the checking and subsequent lexing -+ // happen, but tell the listener that we're going to be cancelling this. -+ Token SavedIdentifier = Tok; -+ MacroDefinition MD = getMacroDefinition(Tok.getIdentifierInfo()); -+ MacroInfo *MI = MD.getMacroInfo(); -+ if (Callbacks) { -+ if (InMacroArgPreExpansion) { -+ Callbacks->Event(SavedIdentifier, PPCallbacks::PrepareToCancelExpansion, -+ reinterpret_cast(MI)); -+ } else { -+ Callbacks->Event(SavedIdentifier, PPCallbacks::BeginMacroCallArgumentList, -+ reinterpret_cast(MI)); -+ } -+ } -+ - TokenCollector Toks = {*this, InMacroArgPreExpansion, {}, Tok}; - - // Remember the pragma token location. - SourceLocation PragmaLoc = Tok.getLocation(); -+ SourceLocation PragmaEndLoc = Tok.getEndLoc(); // PASTA PATCH - - // Read the '('. - Toks.lex(); -@@ -219,6 +237,13 @@ void Preprocessor::Handle_Pragma(Token &Tok) { - return; - } - -+ // PASTA PATCH -+ Token LParenTok = Tok; -+ if (Callbacks && !InMacroArgPreExpansion) { -+ Callbacks->Event(LParenTok, PPCallbacks::BeginMacroCallArgument, -+ reinterpret_cast(&SavedIdentifier)); -+ } -+ - // Read the '"..."'. - Toks.lex(); - if (!tok::isStringLiteral(Tok.getKind())) { -@@ -254,8 +279,43 @@ void Preprocessor::Handle_Pragma(Token &Tok) { - return; - } - -+ // PASTA PATCH: Go and simulate the directive events that would have happened -+ // if this had been an actual `#pragma` directive. -+ Token HashTok; -+ Token PragmaDirectiveTok; -+ if (Callbacks && !InMacroArgPreExpansion) { -+ Callbacks->Event(LParenTok, PPCallbacks::EndMacroCallArgument, -+ reinterpret_cast(&Tok)); -+ Callbacks->Event(LParenTok, PPCallbacks::EndMacroCallArgumentList, 0); -+ Callbacks->Event(SavedIdentifier, PPCallbacks::SwitchToExpansion, -+ reinterpret_cast(MI)); -+ -+ HashTok.startToken(); -+ HashTok.setKind(tok::hash); -+ CreateString("#", HashTok, PragmaLoc, -+ PragmaLoc.getLocWithOffset(1)); -+ HashTok.setLength(1u); -+ Callbacks->Event(HashTok, PPCallbacks::BeginDirective, 0); -+ -+ PragmaDirectiveTok.startToken(); -+ PragmaDirectiveTok.setKind(tok::raw_identifier); -+ CreateString("pragma", PragmaDirectiveTok, PragmaLoc.getLocWithOffset(1), -+ PragmaEndLoc); -+ Callbacks->Event(PragmaDirectiveTok, PPCallbacks::TokenFromLexer, 0); -+ Callbacks->Event(HashTok, PPCallbacks::SetNamedDirective, -+ reinterpret_cast(&PragmaDirectiveTok)); -+ } -+ - // If we're expanding a macro argument, put the tokens back. - if (InMacroArgPreExpansion) { -+ if (Callbacks) { // PASTA PATCH -+// Token EOD; -+// EOD.startToken(); -+// EOD.setKind(tok::eod); -+// Callbacks->Event(EOD, PPCallbacks::EndNonDirective, 0); -+ Callbacks->Event(SavedIdentifier, PPCallbacks::CancelExpansion, -+ reinterpret_cast(MI)); -+ } - Toks.revert(); - return; - } -@@ -1984,10 +2044,13 @@ static IdentifierInfo *HandleMacroAnnotationPragma(Preprocessor &PP, Token &Tok, - } - IdentifierInfo *II = Tok.getIdentifierInfo(); - -- if (!II->hasMacroDefinition()) { -- PP.Diag(Tok, diag::err_pp_visibility_non_macro) << II; -- return nullptr; -- } -+ // PASTA PATCH: PASTA's two-pass pre-processing means that macros won't end -+ // up being explicitly represented in the second pass, and so -+ // this error will always trigger. -+ // if (!II->hasMacroDefinition()) { -+ // PP.Diag(Tok, diag::err_pp_visibility_non_macro) << II; -+ // return nullptr; -+ // } - - PP.Lex(Tok); - if (Tok.is(tok::comma)) { -@@ -2001,6 +2064,11 @@ static IdentifierInfo *HandleMacroAnnotationPragma(Preprocessor &PP, Token &Tok, - PP.Diag(Tok, diag::err_expected) << ")"; - return nullptr; - } -+ -+ if (!II->hasMacroDefinition()) { -+ return nullptr; // PASTA PATCH (see above) -+ } -+ - return II; - } - -diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp -index 0d411abf8..03e08ac24 100644 ---- a/clang/lib/Lex/Preprocessor.cpp -+++ b/clang/lib/Lex/Preprocessor.cpp -@@ -44,6 +44,7 @@ - #include "clang/Lex/MacroArgs.h" - #include "clang/Lex/MacroInfo.h" - #include "clang/Lex/ModuleLoader.h" -+#include "clang/Lex/PPCallbacksEventKind.h" // PASTA PATCH - #include "clang/Lex/Pragma.h" - #include "clang/Lex/PreprocessingRecord.h" - #include "clang/Lex/PreprocessorLexer.h" -@@ -92,6 +93,7 @@ Preprocessor::Preprocessor(std::shared_ptr PPOpts, - Identifiers(IILookup), PragmaHandlers(new PragmaNamespace(StringRef())), - TUKind(TUKind), SkipMainFilePreamble(0, true), - CurSubmoduleState(&NullSubmoduleState) { -+ PostLexAction = [] (const Token &) {}; // PASTA PATCH - OwnsHeaderSearch = OwnsHeaders; - - // Default to discarding comments. -@@ -682,9 +684,17 @@ void Preprocessor::replayPreambleConditionalStack() { - "CurPPLexer is null when calling replayPreambleConditionalStack."); - CurPPLexer->setConditionalLevels(PreambleConditionalStack.getStack()); - PreambleConditionalStack.doneReplaying(); -+ -+ // PASTA PATCH -+ Token HashToken; -+ HashToken.startToken(); -+ HashToken.setKind(tok::hash); -+ HashToken.setLocation(PreambleConditionalStack.SkipInfo->HashTokenLoc); -+ HashToken.setLength(1u); -+ - if (PreambleConditionalStack.reachedEOFWhileSkipping()) - SkipExcludedConditionalBlock( -- PreambleConditionalStack.SkipInfo->HashTokenLoc, -+ HashToken, // PASTA PATCH - PreambleConditionalStack.SkipInfo->IfTokenLoc, - PreambleConditionalStack.SkipInfo->FoundNonSkipPortion, - PreambleConditionalStack.SkipInfo->FoundElse, -@@ -878,27 +888,59 @@ bool Preprocessor::HandleIdentifier(Token &Identifier) { - void Preprocessor::Lex(Token &Result) { - ++LexLevel; - -+ auto InputRawLoc = Result.getLocation().getRawEncoding(); // PASTA PATCH -+ - // We loop here until a lex function returns a token; this avoids recursion. - bool ReturnedToken; - do { - switch (CurLexerKind) { - case CLK_Lexer: - ReturnedToken = CurLexer->Lex(Result); -+ -+ // PASTA PATCH: Visibility into all tokens. -+ if (ReturnedToken && Callbacks) { -+ Callbacks->Event(Result, PPCallbacks::TokenFromLexer, InputRawLoc); -+ } - break; - case CLK_TokenLexer: - ReturnedToken = CurTokenLexer->Lex(Result); -+ -+ // PASTA PATCH: Visibility into all tokens. -+ if (ReturnedToken && Callbacks) { -+ Callbacks->Event(Result, PPCallbacks::TokenFromTokenLexer, InputRawLoc); -+ } - break; - case CLK_CachingLexer: - CachingLex(Result); - ReturnedToken = true; -+ -+ // PASTA PATCH: Visibility into all tokens. -+ if (ReturnedToken && Callbacks) { -+ Callbacks->Event(Result, PPCallbacks::TokenFromCachingLexer, -+ InputRawLoc); -+ } - break; - case CLK_DependencyDirectivesLexer: - ReturnedToken = CurLexer->LexDependencyDirectiveToken(Result); - break; - case CLK_LexAfterModuleImport: - ReturnedToken = LexAfterModuleImport(Result); -+ -+ // PASTA PATCH: Visibility into all tokens. -+ if (ReturnedToken && Callbacks) { -+ Callbacks->Event(Result, PPCallbacks::TokenFromAfterModuleImportLexer, -+ InputRawLoc); -+ } - break; - } -+ -+ // PASTA PATCH -+ if (ReturnedToken && Callbacks && Result.is(tok::eod)) { -+ Callbacks->Event(Result, PPCallbacks::EndDirective, 0); -+ } -+ PostLexAction(Result); -+ PostLexAction = [] (const Token &) {}; -+ - } while (!ReturnedToken); - - if (Result.is(tok::unknown) && TheModuleLoader.HadFatalFailure) -diff --git a/clang/lib/Lex/TokenLexer.cpp b/clang/lib/Lex/TokenLexer.cpp -index ebe7dd66c..6b6805e29 100644 ---- a/clang/lib/Lex/TokenLexer.cpp -+++ b/clang/lib/Lex/TokenLexer.cpp -@@ -21,6 +21,7 @@ - #include "clang/Lex/Lexer.h" - #include "clang/Lex/MacroArgs.h" - #include "clang/Lex/MacroInfo.h" -+#include "clang/Lex/PPCallbacksEventKind.h" // PASTA PATCH - #include "clang/Lex/Preprocessor.h" - #include "clang/Lex/Token.h" - #include "clang/Lex/VariadicMacroSupport.h" -@@ -35,6 +36,16 @@ - - using namespace clang; - -+// PASTA PATCH -+static const Token &TokenReference(const Token *Ptr, size_t NumToks) { -+ if (Ptr && NumToks) { -+ return *Ptr; -+ } else { -+ static const Token kInvalidToken{}; -+ return kInvalidToken; -+ } -+} -+ - /// Create a TokenLexer for the specified macro with the specified actual - /// arguments. Note that this ctor takes ownership of the ActualArgs pointer. - void TokenLexer::Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI, -@@ -43,6 +54,8 @@ void TokenLexer::Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI, - // associated with it. - destroy(); - -+ MacroNameTok = Tok; // PASTA PATCH -+ - Macro = MI; - ActualArgs = Actuals; - CurTokenIdx = 0; -@@ -82,8 +95,42 @@ void TokenLexer::Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI, - - // If this is a function-like macro, expand the arguments and change - // Tokens to point to the expanded tokens. -- if (Macro->isFunctionLike() && Macro->getNumParams()) -+ if (Macro->isFunctionLike() && Macro->getNumParams()) { // PASTA PATCH -+ auto Callbacks = PP.getPPCallbacks(); -+ if (ActualArgs && Callbacks) { -+ unsigned NumArgs = ActualArgs->getNumMacroArguments(); -+ if (NumArgs && Macro->isVariadic() && ActualArgs->isVarargsElidedUse()) { -+ NumArgs -= 1u; // Last argument is `VariadicArgIndex`. -+ } -+ -+ // In theory we only need to pre-expand what needs pre-expansion. In -+ // practice, Clang goes and sometimes requests pre-expansion for the -+ // sake of figuring out __VA_OPT__ stuff, via the -+ // `MacroArgs::invokedWithVariadicArgument` function. -+ Callbacks->Event(MacroNameTok, PPCallbacks::BeginPreArgumentExpansion, -+ reinterpret_cast(Macro)); -+ -+ // Pre expand each argument, which internally caches the results. -+ for (unsigned ArgNo = 0; ArgNo < NumArgs; ++ArgNo) { -+ Callbacks->Event(MacroNameTok, PPCallbacks::BeginMacroCallArgument, 0); -+ (void) ActualArgs->getPreExpArgument(ArgNo, PP); -+ Callbacks->Event(MacroNameTok, PPCallbacks::EndMacroCallArgument, 0); -+ } -+ -+ Callbacks->Event(MacroNameTok, PPCallbacks::EndPreArgumentExpansion, -+ reinterpret_cast(Macro)); -+ } -+ if (Callbacks) { // PASTA PATCH -+ Callbacks->Event(MacroNameTok, PPCallbacks::BeforeParameterSubstitutions, -+ NumTokens); -+ } - ExpandFunctionArguments(); -+ if (Callbacks) { // PASTA PATCH -+ Callbacks->Event(TokenReference(Tokens, NumTokens), -+ PPCallbacks::AfterParameterSubstitutions, -+ NumTokens); -+ } -+ } // PASTA PATCH - - // Mark the macro as currently disabled, so that it is not recursively - // expanded. The macro must be disabled only after argument pre-expansion of -@@ -166,6 +213,14 @@ bool TokenLexer::MaybeRemoveCommaBeforeVaArgs( - if (HasPasteOperator) - PP.Diag(ResultToks.back().getLocation(), diag::ext_paste_comma); - -+ auto Callbacks = PP.getPPCallbacks(); // PASTA PATCH -+ if (Callbacks) { -+ Callbacks->Event( -+ TokenReference(ResultToks.data(), ResultToks.size()), -+ PPCallbacks::BeforeRemoveCommas, -+ reinterpret_cast(ResultToks.size())); -+ } -+ - // Remove the comma. - ResultToks.pop_back(); - -@@ -181,6 +236,13 @@ bool TokenLexer::MaybeRemoveCommaBeforeVaArgs( - ResultToks.back().setFlag(Token::CommaAfterElided); - } - -+ if (Callbacks) { // PASTA PATCH -+ Callbacks->Event( -+ TokenReference(ResultToks.data(), ResultToks.size()), -+ PPCallbacks::AfterRemoveCommas, -+ reinterpret_cast(ResultToks.size())); -+ } -+ - // Never add a space, even if the comma, ##, or arg had a space. - NextTokGetsSpace = false; - return true; -@@ -253,6 +315,8 @@ void TokenLexer::ExpandFunctionArguments() { - - VAOptExpansionContext VCtx(PP); - -+ auto Callbacks = PP.getPPCallbacks(); // PASTA PATCH -+ - for (unsigned I = 0, E = NumTokens; I != E; ++I) { - const Token &CurTok = Tokens[I]; - // We don't want a space for the next token after a paste -@@ -269,6 +333,17 @@ void TokenLexer::ExpandFunctionArguments() { - assert(Tokens[I + 1].is(tok::l_paren) && - "__VA_OPT__ must be followed by '('"); - -+ if (Callbacks) { // PASTA PATCH -+ ResultToks.push_back(CurTok); -+ ResultToks.push_back(Tokens[I + 1]); -+ Callbacks->Event( -+ TokenReference(ResultToks.data(), ResultToks.size()), -+ PPCallbacks::BeforeVAOpt, -+ reinterpret_cast(ResultToks.size())); -+ ResultToks.pop_back(); -+ ResultToks.pop_back(); -+ } -+ - ++I; // Skip the l_paren - VCtx.sawVAOptFollowedByOpeningParens(CurTok.getLocation(), - ResultToks.size()); -@@ -302,12 +377,24 @@ void TokenLexer::ExpandFunctionArguments() { - ActualArgs->invokedWithVariadicArgument(Macro, PP); - } - if (!*CalledWithVariadicArguments) { -+ if (Callbacks) { // PASTA PATCH -+ Callbacks->Event(CurTok, PPCallbacks::SkippedVAOptToken, 0u); -+ } -+ - // Skip this token. - continue; - } - // ... else the macro was called with variadic arguments, and we do not - // have a closing rparen - so process this token normally. - } else { -+ if (Callbacks) { // PASTA PATCH -+ ResultToks.push_back(CurTok); -+ Callbacks->Event( -+ TokenReference(ResultToks.data(), ResultToks.size()), -+ PPCallbacks::AfterVAOpt, -+ reinterpret_cast(ResultToks.size())); -+ ResultToks.pop_back(); -+ } - // Current token is the closing r_paren which marks the end of the - // __VA_OPT__ invocation, so handle any place-marker pasting (if - // empty) by removing hashhash either before (if exists) or after. And -@@ -323,6 +410,12 @@ void TokenLexer::ExpandFunctionArguments() { - // is a token that represents an empty string. - stringifyVAOPTContents(ResultToks, VCtx, - /*ClosingParenLoc*/ Tokens[I].getLocation()); -+ if (Callbacks) { // PASTA PATCH -+ Callbacks->Event( -+ TokenReference(ResultToks.data(), ResultToks.size()), -+ PPCallbacks::AfterStringify, -+ reinterpret_cast(ResultToks.size())); -+ } - - } else if (/*No tokens within VAOPT*/ - ResultToks.size() == VCtx.getNumberOfTokensPriorToVAOpt()) { -@@ -369,6 +462,15 @@ void TokenLexer::ExpandFunctionArguments() { - // parameter or __VA_OPT__ when the #define was lexed. - - if (CurTok.isOneOf(tok::hash, tok::hashat)) { -+ if (Callbacks) { // PASTA PATCH -+ ResultToks.push_back(CurTok); -+ Callbacks->Event( -+ TokenReference(ResultToks.data(), ResultToks.size()), -+ PPCallbacks::BeforeStringify, -+ reinterpret_cast(ResultToks.size())); -+ ResultToks.pop_back(); -+ } -+ - int ArgNo = Macro->getParameterNum(Tokens[I+1].getIdentifierInfo()); - assert((ArgNo != -1 || VCtx.isVAOptToken(Tokens[I + 1])) && - "Token following # is not an argument or __VA_OPT__!"); -@@ -396,7 +498,36 @@ void TokenLexer::ExpandFunctionArguments() { - if (NextTokGetsSpace) - Res.setFlag(Token::LeadingSpace); - -+ if (Callbacks) { // PASTA PATCH -+ ResultToks.push_back(Tokens[I+1]); // Parameter name. -+ Callbacks->Event( -+ TokenReference(ResultToks.data(), ResultToks.size()), -+ PPCallbacks::BeforeMacroParameterUse, -+ reinterpret_cast(ResultToks.size())); -+ ResultToks.pop_back(); -+ auto OldSize = ResultToks.size(); -+ -+ // Render the argument tokens. -+ for (auto ParamArgToks = UnexpArg; ParamArgToks->isNot(tok::eof); -+ ++ParamArgToks) { -+ ResultToks.push_back(*ParamArgToks); // Argument token. -+ } -+ Callbacks->Event( -+ TokenReference(ResultToks.data(), ResultToks.size()), -+ PPCallbacks::AfterMacroParameterUse, -+ reinterpret_cast(ResultToks.size())); -+ ResultToks.resize(OldSize); -+ } -+ - ResultToks.push_back(Res); -+ -+ if (Callbacks) { // PASTA PATCH -+ Callbacks->Event( -+ TokenReference(ResultToks.data(), ResultToks.size()), -+ PPCallbacks::AfterStringify, -+ reinterpret_cast(ResultToks.size())); -+ } -+ - MadeChange = true; - ++I; // Skip arg name. - NextTokGetsSpace = false; -@@ -430,6 +561,13 @@ void TokenLexer::ExpandFunctionArguments() { - continue; - } - -+ if (PasteAfter && Callbacks) { // PASTA PATCH -+ Callbacks->Event( -+ TokenReference(ResultToks.data(), ResultToks.size()), -+ PPCallbacks::BeforeConcatenation, -+ reinterpret_cast(ResultToks.size())); -+ } -+ - // An argument is expanded somehow, the result is different than the - // input. - MadeChange = true; -@@ -444,6 +582,16 @@ void TokenLexer::ExpandFunctionArguments() { - Macro, ArgNo, PP)) - continue; - -+ -+ if (Callbacks) { // PASTA PATCH -+ ResultToks.push_back(CurTok); -+ Callbacks->Event( -+ TokenReference(ResultToks.data(), ResultToks.size()), -+ PPCallbacks::BeforeMacroParameterUse, -+ reinterpret_cast(ResultToks.size())); -+ ResultToks.pop_back(); -+ } -+ - // If it is not the LHS/RHS of a ## operator, we must pre-expand the - // argument and substitute the expanded tokens into the result. This is - // C99 6.10.3.1p1. -@@ -453,7 +601,7 @@ void TokenLexer::ExpandFunctionArguments() { - // Only preexpand the argument if it could possibly need it. This - // avoids some work in common cases. - const Token *ArgTok = ActualArgs->getUnexpArgument(ArgNo); -- if (ActualArgs->ArgNeedsPreexpansion(ArgTok, PP)) -+ if (Callbacks || ActualArgs->ArgNeedsPreexpansion(ArgTok, PP)) // PASTA PATCH - ResultArgToks = &ActualArgs->getPreExpArgument(ArgNo, PP)[0]; - else - ResultArgToks = ArgTok; // Use non-preexpanded tokens. -@@ -504,6 +652,14 @@ void TokenLexer::ExpandFunctionArguments() { - if (RParenAfter) - VCtx.hasPlaceholderBeforeRParen(); - } -+ -+ if (Callbacks) { // PASTA PATCH -+ Callbacks->Event( -+ TokenReference(ResultToks.data(), ResultToks.size()), -+ PPCallbacks::AfterMacroParameterUse, -+ reinterpret_cast(ResultToks.size())); -+ } -+ - continue; - } - -@@ -552,10 +708,38 @@ void TokenLexer::ExpandFunctionArguments() { - Token::LeadingSpace, NextTokGetsSpace); - } - -+ if (Callbacks) { // PASTA PATCH -+ Callbacks->Event( -+ TokenReference(ResultToks.data(), ResultToks.size()), -+ PPCallbacks::AfterMacroParameterUse, -+ reinterpret_cast(ResultToks.size())); -+ -+ if (PasteBefore) { -+ Callbacks->Event( -+ TokenReference(ResultToks.data(), ResultToks.size()), -+ PPCallbacks::AfterConcatenation, -+ reinterpret_cast(ResultToks.size())); -+ } -+ } -+ - NextTokGetsSpace = false; - continue; - } - -+ if (Callbacks) { // PASTA PATCH -+ Callbacks->Event( -+ TokenReference(ResultToks.data(), ResultToks.size()), -+ PPCallbacks::AfterMacroParameterUse, -+ reinterpret_cast(ResultToks.size())); -+ -+ if (PasteBefore) { -+ Callbacks->Event( -+ TokenReference(ResultToks.data(), ResultToks.size()), -+ PPCallbacks::AfterConcatenation, -+ reinterpret_cast(ResultToks.size())); -+ } -+ } -+ - // If an empty argument is on the LHS or RHS of a paste, the standard (C99 - // 6.10.3.3p2,3) calls for a bunch of placemarker stuff to occur. We - // implement this by eating ## operators when a LHS or RHS expands to -@@ -581,9 +765,9 @@ void TokenLexer::ExpandFunctionArguments() { - // of removing the paste operator if __VA_OPT__ reduces to the notional - // placemarker above when we encounter the closing paren of VA_OPT. - if (!VCtx.isInVAOpt() || -- ResultToks.size() > VCtx.getNumberOfTokensPriorToVAOpt()) -+ ResultToks.size() > VCtx.getNumberOfTokensPriorToVAOpt()) { - ResultToks.pop_back(); -- else -+ } else - VCtx.hasPlaceholderAfterHashhashAtStart(); - } - -@@ -627,6 +811,12 @@ bool TokenLexer::Lex(Token &Tok) { - // that it is no longer being expanded. - if (Macro) Macro->EnableMacro(); - -+// // PASTA PATCH -+// if (Macro && Callbacks) { -+// Callbacks->Event(MacroNameTok, PPCallbacks::EndMacroExpansion, -+// reinterpret_cast(Macro)); -+// } -+ - Tok.startToken(); - Tok.setFlagValue(Token::StartOfLine , AtStartOfLine); - Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace || NextTokGetsSpace); -@@ -657,9 +847,20 @@ bool TokenLexer::Lex(Token &Tok) { - // 'L#macro_arg' construct in a function-like macro. - (PP.getLangOpts().MSVCCompat && - isWideStringLiteralFromMacro(Tok, Tokens[CurTokenIdx])))) { -+ -+ auto Callbacks = PP.getPPCallbacks(); // PASTA PATCH -+ if (Callbacks) { -+ Callbacks->Event(Tok, PPCallbacks::BeginConcatenation, -+ reinterpret_cast(&(Tokens[CurTokenIdx]))); -+ } -+ - // When handling the microsoft /##/ extension, the final token is - // returned by pasteTokens, not the pasted token. -- if (pasteTokens(Tok)) -+ auto PasteIsSimple = pasteTokens(Tok); // PASTA PATCH -+ if (Callbacks) { -+ Callbacks->Event(Tok, PPCallbacks::EndConcatenation, 0); -+ } -+ if (PasteIsSimple) - return true; - - TokenIsFromPaste = true; -@@ -755,6 +956,8 @@ bool TokenLexer::pasteTokens(Token &LHSTok, ArrayRef TokenStream, - return TokenStream.size() == CurIdx; - }; - -+ auto Callbacks = PP.getPPCallbacks(); // PASTA PATCH -+ - do { - // Consume the ## operator if any. - PasteOpLoc = TokenStream[CurIdx].getLocation(); -@@ -765,6 +968,19 @@ bool TokenLexer::pasteTokens(Token &LHSTok, ArrayRef TokenStream, - // Get the RHS token. - const Token &RHS = TokenStream[CurIdx]; - -+ if (Callbacks) { // PASTA PATCH -+ if (CurIdx && TokenStream[CurIdx - 1u].is(tok::hashhash)) { -+ Callbacks->Event( -+ TokenStream[CurIdx - 1u], -+ PPCallbacks::ConcatenationOperatorToken, -+ reinterpret_cast(&LHSTok)); -+ } -+ Callbacks->Event( -+ RHS, -+ PPCallbacks::ConcatenationAccumulationToken, -+ reinterpret_cast(&LHSTok)); -+ } -+ - // Allocate space for the result token. This is guaranteed to be enough for - // the two tokens. - Buffer.resize(LHSTok.getLength() + RHS.getLength()); -diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp -index 66d937ac5..99eeabfea 100644 ---- a/clang/lib/Parse/ParseExpr.cpp -+++ b/clang/lib/Parse/ParseExpr.cpp -@@ -1436,6 +1436,13 @@ ExprResult Parser::ParseCastExpression(CastParseKind ParseKind, - case tok::kw_sizeof: // unary-expression: 'sizeof' unary-expression - // unary-expression: 'sizeof' '(' type-name ')' - case tok::kw_vec_step: // unary-expression: OpenCL 'vec_step' expression -+ -+ // PASTA PATCH: XNU-specific UETTs. -+ case tok::kw___builtin_ptrauth_type_discriminator: -+ case tok::kw___builtin_xnu_type_signature: -+ case tok::kw___builtin_xnu_type_summary: -+ case tok::kw___builtin_tmo_type_get_metadata: -+ - // unary-expression: '__builtin_omp_required_simd_align' '(' type-name ')' - case tok::kw___builtin_omp_required_simd_align: - if (NotPrimaryExpression) -@@ -2311,7 +2318,12 @@ Parser::ParseExprAfterUnaryExprOrTypeTrait(const Token &OpTok, - assert(OpTok.isOneOf(tok::kw_typeof, tok::kw_typeof_unqual, tok::kw_sizeof, - tok::kw___alignof, tok::kw_alignof, tok::kw__Alignof, - tok::kw_vec_step, -- tok::kw___builtin_omp_required_simd_align) && -+ tok::kw___builtin_omp_required_simd_align, -+ // PASTA PATCH: XNU-specific UETTs. -+ tok::kw___builtin_ptrauth_type_discriminator, -+ tok::kw___builtin_xnu_type_signature, -+ tok::kw___builtin_xnu_type_summary, -+ tok::kw___builtin_tmo_type_get_metadata) && - "Not a typeof/sizeof/alignof/vec_step expression!"); - - ExprResult Operand; -@@ -2432,7 +2444,13 @@ ExprResult Parser::ParseSYCLUniqueStableNameExpression() { - ExprResult Parser::ParseUnaryExprOrTypeTraitExpression() { - assert(Tok.isOneOf(tok::kw_sizeof, tok::kw___alignof, tok::kw_alignof, - tok::kw__Alignof, tok::kw_vec_step, -- tok::kw___builtin_omp_required_simd_align) && -+ tok::kw___builtin_omp_required_simd_align, -+ -+ // PASTA PATCH: XNU-specific UETTs. -+ tok::kw___builtin_ptrauth_type_discriminator, -+ tok::kw___builtin_xnu_type_signature, -+ tok::kw___builtin_xnu_type_summary, -+ tok::kw___builtin_tmo_type_get_metadata) && - "Not a sizeof/alignof/vec_step expression!"); - Token OpTok = Tok; - ConsumeToken(); -@@ -2509,6 +2527,16 @@ ExprResult Parser::ParseUnaryExprOrTypeTraitExpression() { - else if (OpTok.is(tok::kw___builtin_omp_required_simd_align)) - ExprKind = UETT_OpenMPRequiredSimdAlign; - -+ // PASTA PATCH: XNU-specific things. -+ else if (OpTok.is(tok::kw___builtin_ptrauth_type_discriminator)) -+ ExprKind = UETT_PtrAuthTypeDiscriminator; -+ else if (OpTok.is(tok::kw___builtin_xnu_type_signature)) -+ ExprKind = UETT_XNUTypeSignature; -+ else if (OpTok.is(tok::kw___builtin_xnu_type_summary)) -+ ExprKind = UETT_XNUTypeSummary; -+ else if (OpTok.is(tok::kw___builtin_tmo_type_get_metadata)) -+ ExprKind = UETT_TMOTypeGetMetadata; -+ - if (isCastExpr) - return Actions.ActOnUnaryExprOrTypeTraitExpr(OpTok.getLocation(), - ExprKind, -diff --git a/clang/lib/Parse/ParseTemplate.cpp b/clang/lib/Parse/ParseTemplate.cpp -index 6fc67b696..6adea68b2 100644 ---- a/clang/lib/Parse/ParseTemplate.cpp -+++ b/clang/lib/Parse/ParseTemplate.cpp -@@ -13,6 +13,7 @@ - #include "clang/AST/ASTContext.h" - #include "clang/AST/DeclTemplate.h" - #include "clang/AST/ExprCXX.h" -+#include "clang/Lex/PPCallbacksEventKind.h" // PASTA PATCH - #include "clang/Parse/ParseDiagnostic.h" - #include "clang/Parse/Parser.h" - #include "clang/Parse/RAIIObjectsForParser.h" -@@ -1225,6 +1226,13 @@ bool Parser::ParseGreaterThanInTemplateList(SourceLocation LAngleLoc, - AfterGreaterLoc = PP.SplitToken(AfterGreaterLoc, Tok.getLength()); - Tok.setLocation(AfterGreaterLoc); - -+ // PASTA PATCH: We want to observe when tokens are split up so that we can -+ // have this reflected in PASTA's token lists. -+ if (PPCallbacks *Callbacks = PP.getPPCallbacks()) { -+ Callbacks->Event(Greater, PPCallbacks::BeginSplitToken, 0); -+ Callbacks->Event(Tok, PPCallbacks::EndSplitToken, 0); -+ } -+ - // Update the token cache to match what we just did if necessary. - if (CachingTokens) { - // If the previous cached token is being merged, delete it. -diff --git a/clang/lib/Sema/SemaDecl.cpp b/clang/lib/Sema/SemaDecl.cpp -index 051fad042..85b6cf20e 100644 ---- a/clang/lib/Sema/SemaDecl.cpp -+++ b/clang/lib/Sema/SemaDecl.cpp -@@ -6340,6 +6340,11 @@ NamedDecl *Sema::HandleDeclarator(Scope *S, Declarator &D, - Previous.setRedeclarationKind(ForExternalRedeclaration); - } - -+ // PASTA PATCH: Disable auto-creation of builtins from things like `atoi`; -+ // it ends up leading to a lot of source location information -+ // being dropped. -+ CreateBuiltins = CreateBuiltins ? false : false; -+ - LookupName(Previous, S, CreateBuiltins); - } else { // Something like "int foo::x;" - LookupQualifiedName(Previous, DC); -@@ -6534,6 +6539,13 @@ FixInvalidVariablyModifiedTypeLoc(TypeLoc SrcTL, TypeLoc DstTL) { - DstPTL.setRParenLoc(SrcPTL.getRParenLoc()); - return; - } -+ -+ // PASTA PATCH: Might have an AdjustedTypeLoc for DstTL due to multiplier -+ // patches for issue #130. -+ if (AdjustedTypeLoc AdjTL = DstTL.getAs()) { -+ DstTL = AdjTL.getOriginalLoc(); -+ } -+ - ArrayTypeLoc SrcATL = SrcTL.castAs(); - ArrayTypeLoc DstATL = DstTL.castAs(); - TypeLoc SrcElemTL = SrcATL.getElementLoc(); -diff --git a/clang/lib/Sema/SemaExpr.cpp b/clang/lib/Sema/SemaExpr.cpp -index 2842add2c..1d5bc314d 100644 ---- a/clang/lib/Sema/SemaExpr.cpp -+++ b/clang/lib/Sema/SemaExpr.cpp -@@ -4267,6 +4267,15 @@ bool Sema::CheckUnaryExprOrTypeTraitOperand(Expr *E, - bool IsUnevaluatedOperand = - (ExprKind == UETT_SizeOf || ExprKind == UETT_AlignOf || - ExprKind == UETT_PreferredAlignOf || ExprKind == UETT_VecStep); -+ -+ // PASTA PATCH: Add in XNU-specific unary expr/type traits. -+ IsUnevaluatedOperand = -+ IsUnevaluatedOperand || -+ ExprKind == UETT_PtrAuthTypeDiscriminator || -+ ExprKind == UETT_XNUTypeSignature || -+ ExprKind == UETT_XNUTypeSummary || -+ ExprKind == UETT_TMOTypeGetMetadata; -+ - if (IsUnevaluatedOperand) { - ExprResult Result = CheckUnevaluatedOperand(E); - if (Result.isInvalid()) -@@ -4637,8 +4646,14 @@ Sema::CreateUnaryExprOrTypeTraitExpr(TypeSourceInfo *TInfo, - TInfo->getType()->isVariablyModifiedType()) - TInfo = TransformToPotentiallyEvaluated(TInfo); - -+ // PASTA PATCH: This returns a string literal that is a type signature. -+ QualType RetTy = Context.getSizeType(); -+ if (ExprKind == UETT_XNUTypeSignature) { -+ RetTy = Context.getStringLiteralArrayType(Context.CharTy, 0u); -+ } -+ - return new (Context) UnaryExprOrTypeTraitExpr( -- ExprKind, TInfo, Context.getSizeType(), OpLoc, R.getEnd()); -+ ExprKind, TInfo, RetTy, OpLoc, R.getEnd()); - } - - /// Build a sizeof or alignof expression given an expression -@@ -4652,6 +4667,9 @@ Sema::CreateUnaryExprOrTypeTraitExpr(Expr *E, SourceLocation OpLoc, - - E = PE.get(); - -+ // PASTA PATCH: We want a string literal array return type for the signature. -+ QualType RetTy = Context.getSizeType(); -+ - // Verify that the operand is valid. - bool isInvalid = false; - if (E->isTypeDependent()) { -@@ -4666,6 +4684,16 @@ Sema::CreateUnaryExprOrTypeTraitExpr(Expr *E, SourceLocation OpLoc, - } else if (E->refersToBitField()) { // C99 6.5.3.4p1. - Diag(E->getExprLoc(), diag::err_sizeof_alignof_typeof_bitfield) << 0; - isInvalid = true; -+ -+ // PASTA PATCH: Let these through. -+ } else if (ExprKind == UETT_PtrAuthTypeDiscriminator || -+ ExprKind == UETT_XNUTypeSummary || -+ ExprKind == UETT_TMOTypeGetMetadata) { -+ -+ // PASTA PATCH: This returns a string literal that is a type signature. -+ } else if (ExprKind == UETT_XNUTypeSignature) { -+ RetTy = Context.getStringLiteralArrayType(Context.CharTy, 0u); -+ - } else { - isInvalid = CheckUnaryExprOrTypeTraitOperand(E, UETT_SizeOf); - } -@@ -4681,7 +4709,7 @@ Sema::CreateUnaryExprOrTypeTraitExpr(Expr *E, SourceLocation OpLoc, - - // C99 6.5.3.4p4: the type (an unsigned integer type) is size_t. - return new (Context) UnaryExprOrTypeTraitExpr( -- ExprKind, E, Context.getSizeType(), OpLoc, E->getSourceRange().getEnd()); -+ ExprKind, E, RetTy, OpLoc, E->getSourceRange().getEnd()); - } - - /// ActOnUnaryExprOrTypeTraitExpr - Handle @c sizeof(type) and @c sizeof @c -diff --git a/clang/lib/Sema/SemaExprCXX.cpp b/clang/lib/Sema/SemaExprCXX.cpp -index abf5a72e7..f6eb0ca11 100644 ---- a/clang/lib/Sema/SemaExprCXX.cpp -+++ b/clang/lib/Sema/SemaExprCXX.cpp -@@ -5654,6 +5654,7 @@ static bool EvaluateBinaryTypeTrait(Sema &Self, TypeTrait BTT, QualType LhsT, - } - case BTT_IsSame: - return Self.Context.hasSameType(LhsT, RhsT); -+ case BTT_XNUTypeCompatible: // PASTA PATCH - case BTT_TypeCompatible: { - // GCC ignores cv-qualifiers on arrays for this builtin. - Qualifiers LhsQuals, RhsQuals; -diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h -index 4244bbc1e..83046c48c 100644 ---- a/clang/lib/Sema/TreeTransform.h -+++ b/clang/lib/Sema/TreeTransform.h -@@ -5306,6 +5306,12 @@ TreeTransform::TransformConstantArrayType(TypeLocBuilder &TLB, - return QualType(); - } - -+ // PASTA PATCH: We might be dealing with an adjusted type due to Multiplier -+ // issue #130. -+ if (auto AT = dyn_cast(Result.getTypePtr())) { -+ Result = AT->getAdjustedType(); -+ } -+ - // We might have either a ConstantArrayType or a VariableArrayType now: - // a ConstantArrayType is allowed to have an element type which is a - // VariableArrayType if the type is dependent. Fortunately, all array -@@ -5338,6 +5344,12 @@ QualType TreeTransform::TransformIncompleteArrayType( - return QualType(); - } - -+ // PASTA PATCH: We might be dealing with an adjusted type due to Multiplier -+ // issue #130. -+ if (auto AT = dyn_cast(Result.getTypePtr())) { -+ Result = AT->getAdjustedType(); -+ } -+ - IncompleteArrayTypeLoc NewTL = TLB.push(Result); - NewTL.setLBracketLoc(TL.getLBracketLoc()); - NewTL.setRBracketLoc(TL.getRBracketLoc()); -@@ -5383,6 +5395,12 @@ TreeTransform::TransformVariableArrayType(TypeLocBuilder &TLB, - return QualType(); - } - -+ // PASTA PATCH: We might be dealing with an adjusted type due to Multiplier -+ // issue #130. -+ if (auto AT = dyn_cast(Result.getTypePtr())) { -+ Result = AT->getAdjustedType(); -+ } -+ - // We might have constant size array now, but fortunately it has the same - // location layout. - ArrayTypeLoc NewTL = TLB.push(Result); -@@ -5431,6 +5449,12 @@ TreeTransform::TransformDependentSizedArrayType(TypeLocBuilder &TLB, - return QualType(); - } - -+ // PASTA PATCH: We might be dealing with an adjusted type due to Multiplier -+ // issue #130. -+ if (auto AT = dyn_cast(Result.getTypePtr())) { -+ Result = AT->getAdjustedType(); -+ } -+ - // We might have any sort of array type now, but fortunately they - // all have the same location layout. - ArrayTypeLoc NewTL = TLB.push(Result); --- -2.40.0 - diff --git a/ports/llvm-16/0028-Fixes-to-clang-s-tablegen-of-attributes.patch b/ports/llvm-16/0028-Fixes-to-clang-s-tablegen-of-attributes.patch deleted file mode 100644 index f78a5235..00000000 --- a/ports/llvm-16/0028-Fixes-to-clang-s-tablegen-of-attributes.patch +++ /dev/null @@ -1,34 +0,0 @@ -From e65c2af81d78a78474cf2a3cb3bc862f7105940a Mon Sep 17 00:00:00 2001 -From: Peter Goodman -Date: Thu, 20 Apr 2023 00:30:19 -0400 -Subject: [PATCH] Patches a null pointer deref in Clang. - ---- - clang/utils/TableGen/ClangAttrEmitter.cpp | 12 ++++++++++++ - 1 file changed, 12 insertions(+) - -diff --git a/clang/utils/TableGen/ClangAttrEmitter.cpp b/clang/utils/TableGen/ClangAttrEmitter.cpp -index de608c780..1d447703c 100644 ---- a/clang/utils/TableGen/ClangAttrEmitter.cpp -+++ b/clang/utils/TableGen/ClangAttrEmitter.cpp -@@ -1233,6 +1233,18 @@ namespace { - } - - void writeHasChildren(raw_ostream &OS) const override { OS << "true"; } -+ -+ // PASTA PATCH: Check the optionality of an attribute. -+ std::string getIsOmitted() const override { -+ if (!isOptional()) { -+ return this->SimpleArgument::getIsOmitted(); -+ } -+ -+ std::string output; -+ llvm::raw_string_ostream OS(output); -+ OS << "(get" << getUpperName() << "() == nullptr)"; -+ return output; -+ } - }; - - class VariadicExprArgument : public VariadicArgument { --- -2.40.0 diff --git a/ports/llvm-16/0030-UnknownAttrsAsAnnotate-and-AttributedType-Attrs.patch b/ports/llvm-16/0030-UnknownAttrsAsAnnotate-and-AttributedType-Attrs.patch deleted file mode 100644 index d594059f..00000000 --- a/ports/llvm-16/0030-UnknownAttrsAsAnnotate-and-AttributedType-Attrs.patch +++ /dev/null @@ -1,389 +0,0 @@ -From 32fbac48f39e9fc24263495de5ca0b7df2d4c366 Mon Sep 17 00:00:00 2001 -From: Brent Pappas -Date: Fri, 14 Jul 2023 14:34:09 -0400 -Subject: [PATCH] UnknownAttrsAsAnnotate and AttributedType Attrs - -- Adds the flag `funknown-attrs-as-annotate` to Clang to treat unknown - attributes as annotate attributes by default. -- Adds a an `const Attr *` field to `AttributedType` to store the actual - `Attr` that the type is attributed with, along with methods to access - it. - -Co-authored-by: Laura Bauman ---- - clang/include/clang/AST/ASTContext.h | 3 +- - clang/include/clang/AST/Type.h | 17 ++++++++-- - clang/include/clang/Basic/LangOptions.def | 2 ++ - clang/include/clang/Driver/Options.td | 5 +++ - clang/lib/AST/ASTContext.cpp | 22 +++++++------ - clang/lib/AST/ASTImporter.cpp | 4 ++- - clang/lib/AST/Type.cpp | 5 +-- - clang/lib/Driver/ToolChains/Clang.cpp | 5 +++ - clang/lib/Parse/ParseDeclCXX.cpp | 24 ++++++++------ - clang/lib/Sema/SemaDeclAttr.cpp | 25 +++++++++++++-- - clang/lib/Sema/SemaType.cpp | 38 ++++++++++++++++++----- - clang/lib/Sema/TreeTransform.h | 5 ++- - 12 files changed, 116 insertions(+), 39 deletions(-) - -diff --git a/clang/include/clang/AST/ASTContext.h b/clang/include/clang/AST/ASTContext.h -index 023837192..6cd459359 100644 ---- a/clang/include/clang/AST/ASTContext.h -+++ b/clang/include/clang/AST/ASTContext.h -@@ -1582,7 +1582,8 @@ public: - QualType getInjectedClassNameType(CXXRecordDecl *Decl, QualType TST) const; - - QualType getAttributedType(attr::Kind attrKind, QualType modifiedType, -- QualType equivalentType) const; -+ QualType equivalentType, -+ const Attr *typeAttr = nullptr) const; - - QualType getBTFTagAttributedType(const BTFTypeTagAttr *BTFAttr, - QualType Wrapped); -diff --git a/clang/include/clang/AST/Type.h b/clang/include/clang/AST/Type.h -index 180251d7f..670e6f20d 100644 ---- a/clang/include/clang/AST/Type.h -+++ b/clang/include/clang/AST/Type.h -@@ -56,6 +56,7 @@ - - namespace clang { - -+class Attr; - class BTFTypeTagAttr; - class ExtQuals; - class QualType; -@@ -4880,13 +4881,14 @@ public: - private: - friend class ASTContext; // ASTContext creates these - -+ const Attr *TypeAttr; - QualType ModifiedType; - QualType EquivalentType; - -- AttributedType(QualType canon, attr::Kind attrKind, QualType modified, -- QualType equivalent) -+ AttributedType(QualType canon, Kind attrKind, QualType modified, -+ QualType equivalent, const Attr *typeAttr = nullptr) - : Type(Attributed, canon, equivalent->getDependence()), -- ModifiedType(modified), EquivalentType(equivalent) { -+ TypeAttr(typeAttr), ModifiedType(modified), EquivalentType(equivalent) { - AttributedTypeBits.AttrKind = attrKind; - } - -@@ -4895,6 +4897,8 @@ public: - return static_cast(AttributedTypeBits.AttrKind); - } - -+ bool hasAttr() const { return TypeAttr != nullptr; } -+ const Attr *getAttr() const { return TypeAttr; } - QualType getModifiedType() const { return ModifiedType; } - QualType getEquivalentType() const { return EquivalentType; } - -@@ -4958,6 +4962,13 @@ public: - Profile(ID, getAttrKind(), ModifiedType, EquivalentType); - } - -+ static void Profile(llvm::FoldingSetNodeID &ID, const Attr *typeAttr, -+ QualType modified, QualType equivalent) { -+ ID.AddPointer(typeAttr); -+ ID.AddPointer(modified.getAsOpaquePtr()); -+ ID.AddPointer(equivalent.getAsOpaquePtr()); -+ } -+ - static void Profile(llvm::FoldingSetNodeID &ID, Kind attrKind, - QualType modified, QualType equivalent) { - ID.AddInteger(attrKind); -diff --git a/clang/include/clang/Basic/LangOptions.def b/clang/include/clang/Basic/LangOptions.def -index d1cbe4306..4b2240d57 100644 ---- a/clang/include/clang/Basic/LangOptions.def -+++ b/clang/include/clang/Basic/LangOptions.def -@@ -311,6 +311,8 @@ LANGOPT(FastRelaxedMath , 1, 0, "OpenCL fast relaxed math") - BENIGN_LANGOPT(CLNoSignedZero , 1, 0, "Permit Floating Point optimization without regard to signed zeros") - COMPATIBLE_LANGOPT(CLUnsafeMath , 1, 0, "Unsafe Floating Point Math") - COMPATIBLE_LANGOPT(CLFiniteMathOnly , 1, 0, "__FINITE_MATH_ONLY__ predefined macro") -+ -+LANGOPT(UnknownAttrAnnotate, 1, 0, "Unknown attributes are treated as annotation or annotation type attributes during semantic analysis") - /// FP_CONTRACT mode (on/off/fast). - BENIGN_ENUM_LANGOPT(DefaultFPContractMode, FPModeKind, 2, FPM_Off, "FP contraction type") - COMPATIBLE_LANGOPT(ExpStrictFP, 1, false, "Enable experimental strict floating point") -diff --git a/clang/include/clang/Driver/Options.td b/clang/include/clang/Driver/Options.td -index 652c15afc..64cb09a9a 100644 ---- a/clang/include/clang/Driver/Options.td -+++ b/clang/include/clang/Driver/Options.td -@@ -4377,6 +4377,11 @@ def working_directory : Separate<["-"], "working-directory">, Flags<[CC1Option]> - def working_directory_EQ : Joined<["-"], "working-directory=">, Flags<[CC1Option]>, - Alias; - -+def funknown_attrs_as_annotate : Flag<["-"], "funknown-attrs-as-annotate">, -+ Flags<[CC1Option]>, -+ HelpText<"Treat unknown attributes as annotation or annotation type attributes in semantic analysis">, -+ MarshallingInfoFlag>; -+ - // Double dash options, which are usually an alias for one of the previous - // options. - -diff --git a/clang/lib/AST/ASTContext.cpp b/clang/lib/AST/ASTContext.cpp -index 8054eb2e1..6fe6badff 100644 ---- a/clang/lib/AST/ASTContext.cpp -+++ b/clang/lib/AST/ASTContext.cpp -@@ -3203,10 +3203,11 @@ QualType ASTContext::getFunctionTypeWithExceptionSpec( - - // Might have a calling-convention attribute. - if (const auto *AT = dyn_cast(Orig)) -- return getAttributedType( -- AT->getAttrKind(), -- getFunctionTypeWithExceptionSpec(AT->getModifiedType(), ESI), -- getFunctionTypeWithExceptionSpec(AT->getEquivalentType(), ESI)); -+ return getAttributedType( -+ AT->getAttrKind(), -+ getFunctionTypeWithExceptionSpec(AT->getModifiedType(), ESI), -+ getFunctionTypeWithExceptionSpec(AT->getEquivalentType(), ESI), -+ AT->getAttr()); - - // Anything else must be a function type. Rebuild it with the new exception - // specification. -@@ -4765,11 +4766,14 @@ QualType ASTContext::getUnresolvedUsingType( - return QualType(newType, 0); - } - --QualType ASTContext::getAttributedType(attr::Kind attrKind, -- QualType modifiedType, -- QualType equivalentType) const { -+QualType ASTContext::getAttributedType( -+ attr::Kind attrKind, QualType modifiedType, -+ QualType equivalentType, const Attr *typeAttr/* = nullptr */)const { - llvm::FoldingSetNodeID id; -- AttributedType::Profile(id, attrKind, modifiedType, equivalentType); -+ if (typeAttr) -+ AttributedType::Profile(id, typeAttr, modifiedType, equivalentType); -+ else -+ AttributedType::Profile(id, attrKind, modifiedType, equivalentType); - - void *insertPos = nullptr; - AttributedType *type = AttributedTypes.FindNodeOrInsertPos(id, insertPos); -@@ -4777,7 +4781,7 @@ QualType ASTContext::getAttributedType(attr::Kind attrKind, - - QualType canon = getCanonicalType(equivalentType); - type = new (*this, TypeAlignment) -- AttributedType(canon, attrKind, modifiedType, equivalentType); -+ AttributedType(canon, attrKind, modifiedType, equivalentType, typeAttr); - - Types.push_back(type); - AttributedTypes.InsertNode(type, insertPos); -diff --git a/clang/lib/AST/ASTImporter.cpp b/clang/lib/AST/ASTImporter.cpp -index 6f367ef05..f6bbaef71 100644 ---- a/clang/lib/AST/ASTImporter.cpp -+++ b/clang/lib/AST/ASTImporter.cpp -@@ -1501,7 +1501,9 @@ ExpectedType ASTNodeImporter::VisitAttributedType(const AttributedType *T) { - return ToEquivalentTypeOrErr.takeError(); - - return Importer.getToContext().getAttributedType(T->getAttrKind(), -- *ToModifiedTypeOrErr, *ToEquivalentTypeOrErr); -+ *ToModifiedTypeOrErr, -+ *ToEquivalentTypeOrErr, -+ T->getAttr()); - } - - ExpectedType ASTNodeImporter::VisitTemplateTypeParmType( -diff --git a/clang/lib/AST/Type.cpp b/clang/lib/AST/Type.cpp -index 54e62a193..290fe9d5f 100644 ---- a/clang/lib/AST/Type.cpp -+++ b/clang/lib/AST/Type.cpp -@@ -1158,7 +1158,7 @@ public: - return QualType(T, 0); - - return Ctx.getAttributedType(T->getAttrKind(), modifiedType, -- equivalentType); -+ equivalentType, T->getAttr()); - } - - QualType VisitSubstTemplateTypeParmType(const SubstTemplateTypeParmType *T) { -@@ -1461,7 +1461,8 @@ struct SubstObjCTypeArgsVisitor - - // Rebuild the attributed type. - return Ctx.getAttributedType(newAttrType->getAttrKind(), -- newAttrType->getModifiedType(), newEquivType); -+ newAttrType->getModifiedType(), newEquivType, -+ newAttrType->getAttr()); - } - }; - -diff --git a/clang/lib/Driver/ToolChains/Clang.cpp b/clang/lib/Driver/ToolChains/Clang.cpp -index 77554aa2c..e5b59cbe3 100644 ---- a/clang/lib/Driver/ToolChains/Clang.cpp -+++ b/clang/lib/Driver/ToolChains/Clang.cpp -@@ -4615,6 +4615,11 @@ void Clang::ConstructJob(Compilation &C, const JobAction &JA, - } - } - -+ if (const Arg *A = Args.getLastArg(options::OPT_funknown_attrs_as_annotate)) { -+ CmdArgs.push_back("-funknown-attrs-as-annotate"); -+ A->claim(); -+ } -+ - if (IsOpenMPDevice) { - // We have to pass the triple of the host if compiling for an OpenMP device. - std::string NormalizedTriple = -diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp -index 227c1df2b..4d13a9b6a 100644 ---- a/clang/lib/Parse/ParseDeclCXX.cpp -+++ b/clang/lib/Parse/ParseDeclCXX.cpp -@@ -4368,18 +4368,22 @@ bool Parser::ParseCXX11AttributeArgs( - Syntax = ParsedAttr::AS_Microsoft; - } - -+ - // If the attribute isn't known, we will not attempt to parse any -- // arguments. -- if (Syntax != ParsedAttr::AS_Microsoft && -- !hasAttribute(LO.CPlusPlus ? AttributeCommonInfo::Syntax::AS_CXX11 -- : AttributeCommonInfo::Syntax::AS_C2x, -- ScopeName, AttrName, getTargetInfo(), getLangOpts())) { -- if (getLangOpts().MicrosoftExt || getLangOpts().HLSL) { -+ // arguments. Unless we are treating unknown attributes as annotation -+ // attributes. -+ if (!getLangOpts().UnknownAttrAnnotate) { -+ if (Syntax != ParsedAttr::AS_Microsoft && -+ !hasAttribute(LO.CPlusPlus ? AttributeCommonInfo::Syntax::AS_CXX11 -+ : AttributeCommonInfo::Syntax::AS_C2x, -+ ScopeName, AttrName, getTargetInfo(), getLangOpts())) { -+ if (getLangOpts().MicrosoftExt || getLangOpts().HLSL) { -+ } -+ // Eat the left paren, then skip to the ending right paren. -+ ConsumeParen(); -+ SkipUntil(tok::r_paren); -+ return false; - } -- // Eat the left paren, then skip to the ending right paren. -- ConsumeParen(); -- SkipUntil(tok::r_paren); -- return false; - } - - if (ScopeName && (ScopeName->isStr("gnu") || ScopeName->isStr("__gnu__"))) { -diff --git a/clang/lib/Sema/SemaDeclAttr.cpp b/clang/lib/Sema/SemaDeclAttr.cpp -index a303c7f57..228b54a61 100644 ---- a/clang/lib/Sema/SemaDeclAttr.cpp -+++ b/clang/lib/Sema/SemaDeclAttr.cpp -@@ -4285,6 +4285,21 @@ static void handleAnnotateAttr(Sema &S, Decl *D, const ParsedAttr &AL) { - S.AddAnnotationAttr(D, AL, Str, Args); - } - -+static void -+handleUnknownAttrAsAnnotateAttr(Sema &S, Decl *D, const ParsedAttr &AL) { -+ // Get name of unknown attribute: -+ StringRef Str = AL.getAttrName()->getName(); -+ -+ llvm::SmallVector Args; -+ Args.reserve(AL.getNumArgs()); -+ for (unsigned Idx = 0; Idx < AL.getNumArgs(); Idx++) { -+ assert(!AL.isArgIdent(Idx)); -+ Args.push_back(AL.getArgAsExpr(Idx)); -+ } -+ -+ S.AddAnnotationAttr(D, AL, Str, Args); -+} -+ - static void handleAlignValueAttr(Sema &S, Decl *D, const ParsedAttr &AL) { - S.AddAlignValueAttr(D, AL, AL.getArgAsExpr(0)); - } -@@ -8594,11 +8609,15 @@ ProcessDeclAttribute(Sema &S, Scope *scope, Decl *D, const ParsedAttr &AL, - // though they were unknown attributes. - if (AL.getKind() == ParsedAttr::UnknownAttribute || - !AL.existsInTarget(S.Context.getTargetInfo())) { -- S.Diag(AL.getLoc(), -- AL.isDeclspecAttribute() -+ if (S.getLangOpts().UnknownAttrAnnotate) { -+ handleUnknownAttrAsAnnotateAttr(S, D, AL); -+ } else { -+ S.Diag(AL.getLoc(), -+ AL.isDeclspecAttribute() - ? (unsigned)diag::warn_unhandled_ms_attribute_ignored - : (unsigned)diag::warn_unknown_attribute_ignored) -- << AL << AL.getRange(); -+ << AL << AL.getRange(); -+ } - return; - } - -diff --git a/clang/lib/Sema/SemaType.cpp b/clang/lib/Sema/SemaType.cpp -index 8cb1ed28f..7e60f4bce 100644 ---- a/clang/lib/Sema/SemaType.cpp -+++ b/clang/lib/Sema/SemaType.cpp -@@ -257,7 +257,8 @@ namespace { - QualType getAttributedType(Attr *A, QualType ModifiedType, - QualType EquivType) { - QualType T = -- sema.Context.getAttributedType(A->getKind(), ModifiedType, EquivType); -+ sema.Context.getAttributedType(A->getKind(), ModifiedType, EquivType, -+ A); - AttrsForTypes.push_back({cast(T.getTypePtr()), A}); - AttrsForTypesSorted = false; - return T; -@@ -8287,6 +8288,24 @@ static void HandleMatrixTypeAttr(QualType &CurType, const ParsedAttr &Attr, - CurType = T; - } - -+static void HandleUnkownTypeAttrAsAnnotateTypeAttr(TypeProcessingState &State, -+ QualType &CurType, const ParsedAttr &PA) { -+ Sema &S = State.getSema(); -+ StringRef Str = PA.getAttrName()->getName(); -+ -+ llvm::SmallVector Args; -+ Args.reserve(PA.getNumArgs()); -+ for (unsigned Idx = 0; Idx < PA.getNumArgs(); Idx++) { -+ assert(!PA.isArgIdent(Idx)); -+ Args.push_back(PA.getArgAsExpr(Idx)); -+ } -+ if (!S.ConstantFoldAttrArgs(PA, Args)) -+ return; -+ auto *AnnotateTypeAttr = -+ AnnotateTypeAttr::Create(S.Context, Str, Args.data(), Args.size(), PA); -+ CurType = State.getAttributedType(AnnotateTypeAttr, CurType, CurType); -+} -+ - static void HandleAnnotateTypeAttr(TypeProcessingState &State, - QualType &CurType, const ParsedAttr &PA) { - Sema &S = State.getSema(); -@@ -8390,12 +8409,17 @@ static void processTypeAttrs(TypeProcessingState &state, QualType &type, - - case ParsedAttr::UnknownAttribute: - if (attr.isStandardAttributeSyntax()) { -- state.getSema().Diag(attr.getLoc(), -- diag::warn_unknown_attribute_ignored) -- << attr << attr.getRange(); -- // Mark the attribute as invalid so we don't emit the same diagnostic -- // multiple times. -- attr.setInvalid(); -+ if (state.getSema().getLangOpts().UnknownAttrAnnotate) { -+ HandleUnkownTypeAttrAsAnnotateTypeAttr(state, type, attr); -+ attr.setUsedAsTypeAttr(); -+ } else { -+ state.getSema().Diag(attr.getLoc(), -+ diag::warn_unknown_attribute_ignored) -+ << attr << attr.getRange(); -+ // Mark the attribute as invalid so we don't emit the same diagnostic -+ // multiple times. -+ attr.setInvalid(); -+ } - } - break; - -diff --git a/clang/lib/Sema/TreeTransform.h b/clang/lib/Sema/TreeTransform.h -index 4244bbc1e..fbfff5404 100644 ---- a/clang/lib/Sema/TreeTransform.h -+++ b/clang/lib/Sema/TreeTransform.h -@@ -6995,9 +6995,8 @@ QualType TreeTransform::TransformAttributedType( - } - } - -- result = SemaRef.Context.getAttributedType(TL.getAttrKind(), -- modifiedType, -- equivalentType); -+ result = SemaRef.Context.getAttributedType(TL.getAttrKind(), modifiedType, -+ equivalentType, TL.getAttr()); - } - - AttributedTypeLoc newTL = TLB.push(result); --- -2.34.1 - diff --git a/ports/llvm-16/portfile.cmake b/ports/llvm-16/portfile.cmake index 7b5cf4f3..fdcb5c85 100644 --- a/ports/llvm-16/portfile.cmake +++ b/ports/llvm-16/portfile.cmake @@ -18,16 +18,6 @@ vcpkg_from_github( 0029-Do-not-attempt-macro-expansion-on-invalid-sourceloc.patch ) -if("pasta" IN_LIST FEATURES) - z_vcpkg_apply_patches( - SOURCE_PATH "${SOURCE_PATH}" - PATCHES - 0025-PASTA-patches.patch - 0028-Fixes-to-clang-s-tablegen-of-attributes.patch - 0030-UnknownAttrsAsAnnotate-and-AttributedType-Attrs.patch - ) -endif() - string(REPLACE "." ";" VERSION_LIST ${LLVM_VERSION}) list(GET VERSION_LIST 0 LLVM_VERSION_MAJOR) list(GET VERSION_LIST 1 LLVM_VERSION_MINOR) diff --git a/ports/llvm-16/vcpkg.json b/ports/llvm-16/vcpkg.json index ae0c36a4..6ccfeb25 100644 --- a/ports/llvm-16/vcpkg.json +++ b/ports/llvm-16/vcpkg.json @@ -295,9 +295,6 @@ } ] }, - "pasta": { - "description": "Include patches for PASTA tool." - }, "polly": { "description": "Include Polly (Polyhedral optimizations for LLVM) project.", "dependencies": [ diff --git a/ports/llvm-15/0001-Fix-install-paths.patch b/ports/llvm-17/0001-Fix-install-paths.patch similarity index 100% rename from ports/llvm-15/0001-Fix-install-paths.patch rename to ports/llvm-17/0001-Fix-install-paths.patch diff --git a/ports/llvm-15/0006-Fix-libffi.patch b/ports/llvm-17/0006-Fix-libffi.patch similarity index 100% rename from ports/llvm-15/0006-Fix-libffi.patch rename to ports/llvm-17/0006-Fix-libffi.patch diff --git a/ports/llvm-15/0020-fix-FindZ3.cmake.patch b/ports/llvm-17/0020-fix-FindZ3.cmake.patch similarity index 100% rename from ports/llvm-15/0020-fix-FindZ3.cmake.patch rename to ports/llvm-17/0020-fix-FindZ3.cmake.patch diff --git a/ports/llvm-15/0021-fix-find_dependency.patch b/ports/llvm-17/0021-fix-find_dependency.patch similarity index 67% rename from ports/llvm-15/0021-fix-find_dependency.patch rename to ports/llvm-17/0021-fix-find_dependency.patch index 860132db..47960cfe 100644 --- a/ports/llvm-15/0021-fix-find_dependency.patch +++ b/ports/llvm-17/0021-fix-find_dependency.patch @@ -1,8 +1,11 @@ + llvm/cmake/modules/LLVMConfig.cmake.in | 19 ++++++++++++------- + 1 file changed, 12 insertions(+), 7 deletions(-) + diff --git a/llvm/cmake/modules/LLVMConfig.cmake.in b/llvm/cmake/modules/LLVMConfig.cmake.in -index f1362fa032c2..a9a5b83852b5 100644 +index 42dfd607f7e6..af2aa990f0e9 100644 --- a/llvm/cmake/modules/LLVMConfig.cmake.in +++ b/llvm/cmake/modules/LLVMConfig.cmake.in -@@ -51,16 +51,18 @@ set(LLVM_ENABLE_ASSERTIONS @LLVM_ENABLE_ASSERTIONS@) +@@ -51,21 +51,23 @@ set(LLVM_ENABLE_ASSERTIONS @LLVM_ENABLE_ASSERTIONS@) set(LLVM_ENABLE_EH @LLVM_ENABLE_EH@) @@ -16,6 +19,12 @@ index f1362fa032c2..a9a5b83852b5 100644 set(LLVM_ENABLE_RTTI @LLVM_ENABLE_RTTI@) + set(LLVM_ENABLE_LIBEDIT @HAVE_LIBEDIT@) + if(LLVM_ENABLE_LIBEDIT) +- find_package(LibEdit) ++ find_dependency(LibEdit) + endif() + set(LLVM_ENABLE_TERMINFO @LLVM_ENABLE_TERMINFO@) if(LLVM_ENABLE_TERMINFO) - find_package(Terminfo) @@ -23,11 +32,10 @@ index f1362fa032c2..a9a5b83852b5 100644 endif() set(LLVM_ENABLE_THREADS @LLVM_ENABLE_THREADS@) -@@ -69,21 +71,23 @@ set(LLVM_ENABLE_UNWIND_TABLES @LLVM_ENABLE_UNWIND_TABLES@) - +@@ -75,25 +77,28 @@ set(LLVM_ENABLE_UNWIND_TABLES @LLVM_ENABLE_UNWIND_TABLES@) set(LLVM_ENABLE_ZLIB @LLVM_ENABLE_ZLIB@) if(LLVM_ENABLE_ZLIB) -- set(ZLIB_ROOT @ZLIB_ROOT@) + set(ZLIB_ROOT @ZLIB_ROOT@) - find_package(ZLIB) + find_dependency(ZLIB) endif() @@ -44,6 +52,12 @@ index f1362fa032c2..a9a5b83852b5 100644 + find_dependency(LibXml2) endif() + set(LLVM_ENABLE_CURL @LLVM_ENABLE_CURL@) + if(LLVM_ENABLE_CURL) +- find_package(CURL) ++ find_dependency(CURL) + endif() + set(LLVM_WITH_Z3 @LLVM_WITH_Z3@) +if(LLVM_WITH_Z3) + find_dependency(Z3 4.7.1) diff --git a/ports/llvm-15/0026-fix-prefix-path-calc.patch b/ports/llvm-17/0026-fix-prefix-path-calc.patch similarity index 100% rename from ports/llvm-15/0026-fix-prefix-path-calc.patch rename to ports/llvm-17/0026-fix-prefix-path-calc.patch diff --git a/ports/llvm-17/0029-Do-not-attempt-macro-expansion-on-invalid-sourceloc.patch b/ports/llvm-17/0029-Do-not-attempt-macro-expansion-on-invalid-sourceloc.patch new file mode 100644 index 00000000..2bdbd3b3 --- /dev/null +++ b/ports/llvm-17/0029-Do-not-attempt-macro-expansion-on-invalid-sourceloc.patch @@ -0,0 +1,28 @@ +From efcdeb698fe6a475d1eb0a8f4770ef974cefb0e1 Mon Sep 17 00:00:00 2001 +From: 2over12 +Date: Wed, 31 May 2023 08:57:45 -0400 +Subject: [PATCH] Do not attempt to find macro expansions if there is invalid + source info for a decl + +--- + clang/lib/AST/Expr.cpp | 4 ++++ + 1 file changed, 4 insertions(+) + +diff --git a/clang/lib/AST/Expr.cpp b/clang/lib/AST/Expr.cpp +index e45ae68cd5fe..59a7d50d8ad3 100644 +--- a/clang/lib/AST/Expr.cpp ++++ b/clang/lib/AST/Expr.cpp +@@ -263,6 +263,10 @@ bool Expr::isFlexibleArrayMemberLike( + TypeSourceInfo *TInfo = FD->getTypeSourceInfo(); + while (TInfo) { + TypeLoc TL = TInfo->getTypeLoc(); ++ if (TL.getSourceRange().isInvalid()) { ++ break; ++ } ++ + // Look through typedefs. + if (TypedefTypeLoc TTL = TL.getAsAdjusted()) { + const TypedefNameDecl *TDL = TTL.getTypedefNameDecl(); +-- +2.39.2 (Apple Git-143) + diff --git a/ports/llvm-15/clang_usage b/ports/llvm-17/clang_usage similarity index 100% rename from ports/llvm-15/clang_usage rename to ports/llvm-17/clang_usage diff --git a/ports/llvm-15/flang_usage b/ports/llvm-17/flang_usage similarity index 100% rename from ports/llvm-15/flang_usage rename to ports/llvm-17/flang_usage diff --git a/ports/llvm-15/lld_usage b/ports/llvm-17/lld_usage similarity index 100% rename from ports/llvm-15/lld_usage rename to ports/llvm-17/lld_usage diff --git a/ports/llvm-15/llvm_usage b/ports/llvm-17/llvm_usage similarity index 100% rename from ports/llvm-15/llvm_usage rename to ports/llvm-17/llvm_usage diff --git a/ports/llvm-15/mlir_usage b/ports/llvm-17/mlir_usage similarity index 100% rename from ports/llvm-15/mlir_usage rename to ports/llvm-17/mlir_usage diff --git a/ports/llvm-15/portfile.cmake b/ports/llvm-17/portfile.cmake similarity index 87% rename from ports/llvm-15/portfile.cmake rename to ports/llvm-17/portfile.cmake index 1aac4ab0..4c04ee4d 100644 --- a/ports/llvm-15/portfile.cmake +++ b/ports/llvm-17/portfile.cmake @@ -1,37 +1,21 @@ -set(LLVM_VERSION "15.0.7") - vcpkg_check_linkage(ONLY_STATIC_LIBRARY) vcpkg_from_github( OUT_SOURCE_PATH SOURCE_PATH REPO llvm/llvm-project - REF llvmorg-${LLVM_VERSION} - SHA512 99beff9ee6f8c26f16ea53f03ba6209a119099cbe361701b0d5f4df9d5cc5f2f0da7c994c899a4cec876da8428564dc7a8e798226a9ba8b5c18a3ef8b181d39e - HEAD_REF release/15.x + REF llvmorg-${VERSION} + SHA512 ad75af49394b8cf2e6152ded55754901f48dc8f4efacfdc6d0651d3b5532893630773571c4fda5e853b9784a8a8a9d286a6a89511a8319726d774b801166ac33 + HEAD_REF release/17.x PATCHES - 0001-Fix-install-paths.patch # This patch fixes paths in ClangConfig.cmake, LLVMConfig.cmake, LLDConfig.cmake etc. - 0002-Fix-DR-1734.patch - 0003-Fix-tools-path.patch - 0004-Fix-compiler-rt-install-path.patch - 0005-Fix-tools-install-path.patch + 0001-Fix-install-paths.patch 0006-Fix-libffi.patch - 0007-Fix-install-bolt.patch 0020-fix-FindZ3.cmake.patch 0021-fix-find_dependency.patch - 0023-clang-sys-include-dir-path.patch 0026-fix-prefix-path-calc.patch + 0029-Do-not-attempt-macro-expansion-on-invalid-sourceloc.patch ) -if("pasta" IN_LIST FEATURES) - z_vcpkg_apply_patches( - SOURCE_PATH "${SOURCE_PATH}" - PATCHES - 0025-PASTA-patches.patch - 0027-unknown-attrs-as-annotations.patch - ) -endif() - -string(REPLACE "." ";" VERSION_LIST ${LLVM_VERSION}) +string(REPLACE "." ";" VERSION_LIST ${VERSION}) list(GET VERSION_LIST 0 LLVM_VERSION_MAJOR) list(GET VERSION_LIST 1 LLVM_VERSION_MINOR) list(GET VERSION_LIST 2 LLVM_VERSION_PATCH) @@ -168,24 +152,10 @@ if("clang" IN_LIST FEATURES OR "clang-tools-extra" IN_LIST FEATURES) -DCLANG_ENABLE_STATIC_ANALYZER=OFF ) endif() - if(VCPKG_TARGET_IS_OSX) - list(APPEND FEATURE_OPTIONS - -DDEFAULT_SYSROOT:FILEPATH=/Library/Developer/CommandLineTools/SDKs/MacOSX.sdk - -DLLVM_CREATE_XCODE_TOOLCHAIN=ON - ) - endif() - # 1) LLVM/Clang tools are relocated from ./bin/ to ./tools/llvm/ (LLVM_TOOLS_INSTALL_DIR=tools/llvm) - # 2) Clang resource files are relocated from ./lib/clang/ to ./tools/llvm/lib/clang/ (see patch 0007-fix-compiler-rt-install-path.patch) - # So, the relative path should be changed from ../lib/clang/ to ./lib/clang/ - # This needs to not include version suffixes like '-rc3' - list(APPEND FEATURE_OPTIONS -DCLANG_RESOURCE_DIR=lib/clang/${LLVM_VERSION_MAJOR}.${LLVM_VERSION_MINOR}.${LLVM_VERSION_PATCH}) endif() if("clang-tools-extra" IN_LIST FEATURES) list(APPEND LLVM_ENABLE_PROJECTS "clang-tools-extra") endif() -if("compiler-rt" IN_LIST FEATURES) - list(APPEND LLVM_ENABLE_PROJECTS "compiler-rt") -endif() if("flang" IN_LIST FEATURES) if(VCPKG_DETECTED_CMAKE_CXX_COMPILER_ID STREQUAL "MSVC" AND VCPKG_TARGET_ARCHITECTURE STREQUAL "x86") message(FATAL_ERROR "Building Flang with MSVC is not supported on x86. Disable it until issues are fixed.") @@ -243,6 +213,9 @@ if("libcxxabi" IN_LIST FEATURES) endif() list(APPEND LLVM_ENABLE_RUNTIMES "libcxxabi") endif() +if("compiler-rt" IN_LIST FEATURES) + list(APPEND LLVM_ENABLE_RUNTIMES "compiler-rt") +endif() if("libunwind" IN_LIST FEATURES) list(APPEND LLVM_ENABLE_RUNTIMES "libunwind") endif() @@ -256,6 +229,7 @@ set(known_llvm_targets BPF Hexagon Lanai + LoongArch Mips MSP430 NVPTX @@ -279,7 +253,12 @@ endforeach() # this is for experimental targets set(known_llvm_experimental_targets - SPRIV + ARC + CSKY + DirectX + M68k + SPIRV + Xtensa ) set(LLVM_EXPERIMENTAL_TARGETS_TO_BUILD "") @@ -349,13 +328,11 @@ vcpkg_cmake_configure( "-DLLVM_ENABLE_RUNTIMES=${LLVM_ENABLE_RUNTIMES}" "-DLLVM_TARGETS_TO_BUILD=${LLVM_TARGETS_TO_BUILD}" "-DLLVM_EXPERIMENTAL_TARGETS_TO_BUILD=${LLVM_EXPERIMENTAL_TARGETS_TO_BUILD}" - -DPACKAGE_VERSION=${LLVM_VERSION} + -DPACKAGE_VERSION=${VERSION} # Limit the maximum number of concurrent link jobs to 1. This should fix low amount of memory issue for link. "-DLLVM_PARALLEL_LINK_JOBS=${LLVM_LINK_JOBS}" -DCMAKE_INSTALL_PACKAGEDIR:STRING=share - -DLLVM_TOOLS_INSTALL_DIR:STRING=tools/llvm - -DCLANG_TOOLS_INSTALL_DIR:STRING=tools/llvm - -DMLIR_TOOLS_INSTALL_DIR:STRING=tools/llvm + "-DRUNTIMES_CMAKE_ARGS=-DCMAKE_PREFIX_PATH=${CURRENT_INSTALLED_DIR}" ) vcpkg_cmake_install(ADD_BIN_TO_PATH) @@ -373,7 +350,6 @@ function(llvm_cmake_package_config_fixup package_name) set(args) # Maintains case even if package_name name is case-sensitive list(APPEND args PACKAGE_NAME "${lower_package}") - list(APPEND args TOOLS_PATH "tools/llvm") if(arg_DO_NOT_DELETE_PARENT_CONFIG_PATH) list(APPEND args "DO_NOT_DELETE_PARENT_CONFIG_PATH") endif() @@ -431,16 +407,29 @@ if(empty_dirs) endforeach() endif() -vcpkg_copy_tool_dependencies(${CURRENT_PACKAGES_DIR}/tools/llvm) - if(NOT DEFINED VCPKG_BUILD_TYPE OR VCPKG_BUILD_TYPE STREQUAL "debug") - file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/tools" + file(REMOVE_RECURSE "${CURRENT_PACKAGES_DIR}/debug/bin" "${CURRENT_PACKAGES_DIR}/debug/include" "${CURRENT_PACKAGES_DIR}/debug/share" "${CURRENT_PACKAGES_DIR}/debug/lib/clang" ) endif() +# Use 'bin' instead of 'tools/llvm' +file(GLOB_RECURSE release_targets + "${CURRENT_PACKAGES_DIR}/share/*/*Targets-*.cmake" + "${CURRENT_PACKAGES_DIR}/share/*/*Exports-*.cmake" +) +foreach(release_target IN LISTS release_targets) + file(READ "${release_target}" contents) + string(REPLACE "${CURRENT_INSTALLED_DIR}" "\${_IMPORT_PREFIX}" contents "${contents}") + string(REGEX REPLACE + "\\\${_IMPORT_PREFIX}/tools/llvm-17/([^ \"]+${EXECUTABLE_SUFFIX})" + "\${_IMPORT_PREFIX}/bin/\\1" + contents "${contents}") + file(WRITE "${release_target}" "${contents}") +endforeach() + if("mlir" IN_LIST FEATURES) vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/share/mlir/MLIRConfig.cmake" "set(MLIR_MAIN_SRC_DIR \"${SOURCE_PATH}/mlir\")" "") vcpkg_replace_string("${CURRENT_PACKAGES_DIR}/share/mlir/MLIRConfig.cmake" "${CURRENT_BUILDTREES_DIR}" "\${MLIR_INCLUDE_DIRS}") diff --git a/ports/llvm-15/vcpkg.json b/ports/llvm-17/vcpkg.json similarity index 86% rename from ports/llvm-15/vcpkg.json rename to ports/llvm-17/vcpkg.json index b89a7b13..f15a5bae 100644 --- a/ports/llvm-15/vcpkg.json +++ b/ports/llvm-17/vcpkg.json @@ -1,7 +1,6 @@ { - "name": "llvm-15", - "version": "15.0.7", - "port-version": 3, + "name": "llvm-17", + "version": "17.0.1", "description": "The LLVM Compiler Infrastructure.", "homepage": "https://llvm.org", "license": "Apache-2.0", @@ -35,7 +34,7 @@ "description": "BOLT is a post-link optimizer developed to speed up large applications.", "dependencies": [ { - "name": "llvm-15", + "name": "llvm-17", "default-features": false, "features": [ "tools" @@ -47,7 +46,7 @@ "description": "Include C Language Family Front-end.", "dependencies": [ { - "name": "llvm-15", + "name": "llvm-17", "default-features": false, "features": [ "compiler-rt", @@ -66,7 +65,7 @@ "description": "Build with cxx-common target set", "dependencies": [ { - "name": "llvm-15", + "name": "llvm-17", "default-features": false, "features": [ "target-aarch64", @@ -83,7 +82,7 @@ "description": "Build with default options.", "dependencies": [ { - "name": "llvm-15", + "name": "llvm-17", "default-features": false, "features": [ "enable-assertions", @@ -101,7 +100,7 @@ "description": "Build with platform-specific default targets.", "dependencies": [ { - "name": "llvm-15", + "name": "llvm-17", "default-features": false, "features": [ "target-aarch64" @@ -109,7 +108,7 @@ "platform": "arm64" }, { - "name": "llvm-15", + "name": "llvm-17", "default-features": false, "features": [ "target-x86" @@ -117,7 +116,7 @@ "platform": "x86 | x64" }, { - "name": "llvm-15", + "name": "llvm-17", "default-features": false, "features": [ "target-arm" @@ -125,7 +124,7 @@ "platform": "arm & !arm64" }, { - "name": "llvm-15", + "name": "llvm-17", "default-features": false, "features": [ "target-all" @@ -153,7 +152,7 @@ "description": "Build LLVM with exception handler.", "dependencies": [ { - "name": "llvm-15", + "name": "llvm-17", "default-features": false, "features": [ "enable-rtti" @@ -192,7 +191,7 @@ "description": "Compile with Z3 SMT solver support for Clang static analyzer.", "dependencies": [ { - "name": "llvm-15", + "name": "llvm-17", "default-features": false, "features": [ "clang" @@ -217,7 +216,7 @@ "description": "Include Fortran front end.", "dependencies": [ { - "name": "llvm-15", + "name": "llvm-17", "default-features": false, "features": [ "clang", @@ -234,7 +233,7 @@ "description": "Include libcxx library.", "dependencies": [ { - "name": "llvm-15", + "name": "llvm-17", "default-features": false, "features": [ "libcxxabi" @@ -252,7 +251,7 @@ "description": "Include LLVM linker.", "dependencies": [ { - "name": "llvm-15", + "name": "llvm-17", "default-features": false, "features": [ "tools" @@ -264,7 +263,7 @@ "description": "Include LLVM debugger.", "dependencies": [ { - "name": "llvm-15", + "name": "llvm-17", "default-features": false, "features": [ "tools" @@ -276,7 +275,7 @@ "description": "Include MLIR (Multi-Level IR Compiler Framework) project.", "dependencies": [ { - "name": "llvm-15", + "name": "llvm-17", "default-features": false, "features": [ "tools" @@ -288,7 +287,7 @@ "description": "Include LLVM OpenMP libraries.", "dependencies": [ { - "name": "llvm-15", + "name": "llvm-17", "default-features": false, "features": [ "utils" @@ -296,14 +295,11 @@ } ] }, - "pasta": { - "description": "Include patches for PASTA tool." - }, "polly": { "description": "Include Polly (Polyhedral optimizations for LLVM) project.", "dependencies": [ { - "name": "llvm-15", + "name": "llvm-17", "default-features": false, "features": [ "tools", @@ -323,7 +319,7 @@ "description": "Build with all backends.", "dependencies": [ { - "name": "llvm-15", + "name": "llvm-17", "default-features": false, "features": [ "target-aarch64", @@ -333,18 +329,24 @@ "target-bpf", "target-hexagon", "target-lanai", + "target-loongarch", "target-mips", "target-msp430", "target-nvptx", "target-powerpc", "target-riscv", "target-sparc", - "target-spirv", "target-systemz", "target-ve", "target-webassembly", "target-x86", - "target-xcore" + "target-xcore", + "target-arc", + "target-csky", + "target-directx", + "target-m68k", + "target-spirv", + "target-xtensa" ] } ] @@ -367,6 +369,9 @@ "target-lanai": { "description": "Build with Lanai backend." }, + "target-loongarch": { + "description": "Build with LoongArch backend." + }, "target-mips": { "description": "Build with Mips backend." }, @@ -385,9 +390,6 @@ "target-sparc": { "description": "Build with Sparc backend." }, - "target-spirv": { - "description": "Build with Spriv backend." - }, "target-systemz": { "description": "Build with SystemZ backend." }, @@ -403,11 +405,29 @@ "target-xcore": { "description": "Build with XCore backend." }, + "target-arc": { + "description": "Build with ARC backend (experimental)." + }, + "target-csky": { + "description": "Build with CSKY backend (experimental)." + }, + "target-directx": { + "description": "Build with DirectX backend (experimental)." + }, + "target-m68k": { + "description": "Build with M68k backend (experimental)." + }, + "target-spirv": { + "description": "Build with SPIRV backend (experimental)." + }, + "target-xtensa": { + "description": "Build with Xtensa backend (experimental)." + }, "tools": { "description": "Build LLVM tools.", "dependencies": [ { - "name": "llvm-15", + "name": "llvm-17", "default-features": false, "features": [ "enable-threads" diff --git a/vcpkg_info.txt b/vcpkg_info.txt index 5d68a591..75ec1169 100644 --- a/vcpkg_info.txt +++ b/vcpkg_info.txt @@ -1,2 +1,2 @@ https://github.com/trail-of-forks/vcpkg.git -e32948cca1926244ab0d9ccc31bfd20a048cd3d7 +ccd7884f024d73a45937f71b1bc757d2cd82c2f4