thesamesam created this revision.
Herald added subscribers: kadircet, arphaman.
Herald added a project: All.
thesamesam requested review of this revision.
Herald added subscribers: cfe-commits, MaskRay, ilya-biryukov.
Herald added a project: clang-tools-extra.
Previously build system generated single huge CompletionModel.cpp with size
about 6.5 MiB.
Compilation + assembling on PPC32 with GCC + GNU assembler resulted in 22010
errors:
{standard input}: Assembler messages:
{standard input}:181155: Error: operand out of range (0x0000000000008164 is
not between 0xffffffffffff8000 and 0x0000000000007fff)
{standard input}:181192: Error: operand out of range (0x000000000000816c is
not between 0xffffffffffff8000 and 0x0000000000007fff)
{standard input}:181211: Error: operand out of range (0x0000000000008080 is
not between 0xffffffffffff8000 and 0x0000000000007fff)
{standard input}:181220: Error: operand out of range (0x00000000000081b8 is
not between 0xffffffffffff8000 and 0x0000000000007fff)
{standard input}:181225: Error: operand out of range (0x0000000000008168 is
not between 0xffffffffffff8000 and 0x0000000000007fff)
[...]
Separate compilation + assembling of given portion of code avoids exceeding
capabilities of compiler / assembler.
Bug: https://bugs.gentoo.org/829602
Thanks-to: Arfrever Frehtes Taifersar Arahesis <arfre...@apache.org>
Tested-by: erhar...@mailbox.org <erhar...@mailbox.org>
Repository:
rG LLVM Github Monorepo
https://reviews.llvm.org/D136283
Files:
clang-tools-extra/clangd/CMakeLists.txt
clang-tools-extra/clangd/quality/CompletionModel.cmake
clang-tools-extra/clangd/quality/CompletionModelCodegen.py
clang-tools-extra/clangd/unittests/CMakeLists.txt
Index: clang-tools-extra/clangd/unittests/CMakeLists.txt
===================================================================
--- clang-tools-extra/clangd/unittests/CMakeLists.txt
+++ clang-tools-extra/clangd/unittests/CMakeLists.txt
@@ -21,6 +21,7 @@
include(${CMAKE_CURRENT_SOURCE_DIR}/../quality/CompletionModel.cmake)
gen_decision_forest(${CMAKE_CURRENT_SOURCE_DIR}/decision_forest_model DecisionForestRuntimeTest ::ns1::ns2::test::Example)
+file(GLOB DecisionForestRuntimeTest_files "${CMAKE_CURRENT_BINARY_DIR}/DecisionForestRuntimeTest/DecisionForestRuntimeTest*.cpp")
add_custom_target(ClangdUnitTests)
add_unittest(ClangdUnitTests ClangdTests
@@ -95,7 +96,7 @@
TypeHierarchyTests.cpp
URITests.cpp
XRefsTests.cpp
- ${CMAKE_CURRENT_BINARY_DIR}/DecisionForestRuntimeTest.cpp
+ ${DecisionForestRuntimeTest_files}
support/CancellationTests.cpp
support/ContextTests.cpp
@@ -134,9 +135,9 @@
$<TARGET_OBJECTS:obj.clangDaemonTweaks>
)
-# Include generated ComletionModel headers.
+# Include generated Completion Model header.
target_include_directories(ClangdTests PUBLIC
- $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+ $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/DecisionForestRuntimeTest>
)
clang_target_link_libraries(ClangdTests
Index: clang-tools-extra/clangd/quality/CompletionModelCodegen.py
===================================================================
--- clang-tools-extra/clangd/quality/CompletionModelCodegen.py
+++ clang-tools-extra/clangd/quality/CompletionModelCodegen.py
@@ -1,13 +1,15 @@
"""Code generator for Code Completion Model Inference.
Tool runs on the Decision Forest model defined in {model} directory.
-It generates two files: {output_dir}/{filename}.h and {output_dir}/{filename}.cpp
-The generated files defines the Example class named {cpp_class} having all the features as class members.
+It generates files: {output_dir}/{filename}.h, {output_dir}/{filename}.cpp,
+and {output_dir}/{filename}{number}.cpp for each Decision Tree.
+The generated files define the Example class named {cpp_class} having all the features as class members.
The generated runtime provides an `Evaluate` function which can be used to score a code completion candidate.
"""
import argparse
import json
+import os
import struct
@@ -194,34 +196,36 @@
`float Evaluate(const {Example}&)` function. This function can be
used to score an Example."""
- code = ""
+ functions_codes = {}
# Generate evaluation function of each tree.
- code += "namespace {\n"
tree_num = 0
for tree_json in forest_json:
- code += "LLVM_ATTRIBUTE_NOINLINE float EvaluateTree%d(const %s& E) {\n" % (tree_num, cpp_class.name)
- code += " " + \
+ functions_codes[f"{tree_num}"] = "LLVM_ATTRIBUTE_NOINLINE float EvaluateTree%d(const %s& E) {\n" % (tree_num, cpp_class.name)
+ functions_codes[f"{tree_num}"] += " " + \
"\n ".join(
tree(tree_json, tree_num=tree_num, node_num=0)[0]) + "\n"
- code += "}\n\n"
+ functions_codes[f"{tree_num}"] += "}\n"
tree_num += 1
- code += "} // namespace\n\n"
# Combine the scores of all trees in the final function.
# MSAN will timeout if these functions are inlined.
- code += "float Evaluate(const %s& E) {\n" % cpp_class.name
- code += " float Score = 0;\n"
+ final_function_code = ""
for tree_num in range(len(forest_json)):
- code += " Score += EvaluateTree%d(E);\n" % tree_num
- code += " return Score;\n"
- code += "}\n"
+ final_function_code += "float EvaluateTree%d(const %s& E);\n" % (tree_num, cpp_class.name)
+ final_function_code += "\n"
+ final_function_code += "float Evaluate(const %s& E) {\n" % cpp_class.name
+ final_function_code += " float Score = 0;\n"
+ for tree_num in range(len(forest_json)):
+ final_function_code += " Score += EvaluateTree%d(E);\n" % tree_num
+ final_function_code += " return Score;\n"
+ final_function_code += "}\n"
- return code
+ return functions_codes, final_function_code
def gen_cpp_code(forest_json, features_json, filename, cpp_class):
- """Generates code for the .cpp file."""
+ """Generates code for the .cpp files."""
# Headers
# Required by OrderEncode(float F).
angled_include = [
@@ -242,11 +246,11 @@
for feature in features_json
if feature["kind"] == "ENUM")
nl = "\n"
- return """%s
-%s
+ functions_codes, final_function_code = evaluate_func(forest_json, cpp_class)
-#define BIT(X) (1LL << X)
+ cpp_code = {
+ "": """%s
%s
@@ -268,8 +272,23 @@
%s
%s
""" % (nl.join(angled_include), nl.join(quoted_include), cpp_class.ns_begin(),
- using_decls, cpp_class.name, evaluate_func(forest_json, cpp_class),
- cpp_class.ns_end())
+ cpp_class.name, final_function_code, cpp_class.ns_end())}
+
+ for function_code_num, function_code in functions_codes.items():
+ cpp_code[function_code_num] = """%s
+
+#define BIT(X) (1LL << X)
+
+%s
+
+%s
+
+%s
+%s
+""" % (nl.join(quoted_include), cpp_class.ns_begin(), using_decls,
+ function_code, cpp_class.ns_end())
+
+ return cpp_code
def main():
@@ -286,7 +305,6 @@
output_dir = ns.output_dir
filename = ns.filename
header_file = "%s/%s.h" % (output_dir, filename)
- cpp_file = "%s/%s.cpp" % (output_dir, filename)
cpp_class = CppClass(cpp_class=ns.cpp_class)
model_file = "%s/forest.json" % ns.model
@@ -298,12 +316,15 @@
with open(model_file) as m:
forest_json = json.load(m)
- with open(cpp_file, 'w+t') as output_cc:
- output_cc.write(
- gen_cpp_code(forest_json=forest_json,
- features_json=features_json,
- filename=filename,
- cpp_class=cpp_class))
+ os.makedirs(output_dir, exist_ok=True)
+
+ for cpp_code_num, cpp_code in gen_cpp_code(forest_json=forest_json,
+ features_json=features_json,
+ filename=filename,
+ cpp_class=cpp_class).items():
+ cpp_file = "%s/%s%s.cpp" % (output_dir, filename, cpp_code_num)
+ with open(cpp_file, 'w+t') as output_cc:
+ output_cc.write(cpp_code)
with open(header_file, 'w+t') as output_h:
output_h.write(gen_header_code(
Index: clang-tools-extra/clangd/quality/CompletionModel.cmake
===================================================================
--- clang-tools-extra/clangd/quality/CompletionModel.cmake
+++ clang-tools-extra/clangd/quality/CompletionModel.cmake
@@ -1,18 +1,30 @@
-# Run the Completion Model Codegenerator on the model present in the
+# Run the Completion Model Codegenerator on the model present in the
# ${model} directory.
-# Produces a pair of files called ${filename}.h and ${filename}.cpp in the
-# ${CMAKE_CURRENT_BINARY_DIR}. The generated header
+# Produces files called ${filename}.h and ${filename}*.cpp in the
+# ${CMAKE_CURRENT_BINARY_DIR}/${filename} directory. The generated header
# will define a C++ class called ${cpp_class} - which may be a
# namespace-qualified class name.
set(CLANGD_COMPLETION_MODEL_COMPILER ${CMAKE_CURRENT_LIST_DIR}/CompletionModelCodegen.py)
function(gen_decision_forest model filename cpp_class)
set(model_compiler ${CLANGD_COMPLETION_MODEL_COMPILER})
- set(output_dir ${CMAKE_CURRENT_BINARY_DIR})
+ set(output_dir ${CMAKE_CURRENT_BINARY_DIR}/${filename})
set(header_file ${output_dir}/${filename}.h)
- set(cpp_file ${output_dir}/${filename}.cpp)
- add_custom_command(OUTPUT ${header_file} ${cpp_file}
+ # Command for initial (CMake-invocation-time) generation of files.
+ message(STATUS "Generating code completion model runtime")
+ execute_process(COMMAND "${Python3_EXECUTABLE}" ${model_compiler}
+ --model ${model}
+ --output_dir ${output_dir}
+ --filename ${filename}
+ --cpp_class ${cpp_class})
+
+ file(GLOB cpp_files "${output_dir}/${filename}*.cpp")
+
+ # Command for potential regeneration of files after changes in model compiler,
+ # forest.json or features.json.
+ # This works reliably only if list of generated files is unchanged.
+ add_custom_command(OUTPUT ${header_file} ${cpp_files}
COMMAND "${Python3_EXECUTABLE}" ${model_compiler}
--model ${model}
--output_dir ${output_dir}
@@ -20,19 +32,19 @@
--cpp_class ${cpp_class}
COMMENT "Generating code completion model runtime..."
DEPENDS ${model_compiler} ${model}/forest.json ${model}/features.json
- VERBATIM )
+ VERBATIM)
set_source_files_properties(${header_file} PROPERTIES
GENERATED 1)
- set_source_files_properties(${cpp_file} PROPERTIES
+ set_source_files_properties(${cpp_files} PROPERTIES
GENERATED 1)
# Disable unused label warning for generated files.
if (CMAKE_CXX_COMPILER_ID STREQUAL "MSVC")
- set_source_files_properties(${cpp_file} PROPERTIES
+ set_source_files_properties(${cpp_files} PROPERTIES
COMPILE_FLAGS /wd4102)
else()
- set_source_files_properties(${cpp_file} PROPERTIES
+ set_source_files_properties(${cpp_files} PROPERTIES
COMPILE_FLAGS -Wno-unused)
endif()
endfunction()
Index: clang-tools-extra/clangd/CMakeLists.txt
===================================================================
--- clang-tools-extra/clangd/CMakeLists.txt
+++ clang-tools-extra/clangd/CMakeLists.txt
@@ -45,6 +45,7 @@
include(${CMAKE_CURRENT_SOURCE_DIR}/quality/CompletionModel.cmake)
gen_decision_forest(${CMAKE_CURRENT_SOURCE_DIR}/quality/model CompletionModel clang::clangd::Example)
+file(GLOB CompletionModel_files "${CMAKE_CURRENT_BINARY_DIR}/CompletionModel/CompletionModel*.cpp")
if(MSVC AND NOT CLANG_CL)
set_source_files_properties(CompileCommands.cpp PROPERTIES COMPILE_FLAGS -wd4130) # disables C4130: logical operation on address of string constant
@@ -102,7 +103,7 @@
TUScheduler.cpp
URI.cpp
XRefs.cpp
- ${CMAKE_CURRENT_BINARY_DIR}/CompletionModel.cpp
+ ${CompletionModel_files}
index/Background.cpp
index/BackgroundIndexLoader.cpp
@@ -140,9 +141,9 @@
omp_gen
)
-# Include generated CompletionModel headers.
+# Include generated Completion Model header.
target_include_directories(clangDaemon PUBLIC
- $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>
+ $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}/CompletionModel>
)
clang_target_link_libraries(clangDaemon
_______________________________________________
cfe-commits mailing list
cfe-commits@lists.llvm.org
https://lists.llvm.org/cgi-bin/mailman/listinfo/cfe-commits
