rust-lang/rust
1
0
Fork 0
Code Issues Pull Requests Actions 1 Packages Projects Releases Wiki Activity

Merge commit '98ed962c7d3eebe12c97588e61245273d265e72f' into master

Browse Source
This commit is contained in:
Guillaume Gomez
2024-07-10 12:44:23 +02:00
parent 649feb9c1a 98ed962c7d
commit 7cbe50e209
68 changed files with 2682 additions and 1135 deletions
Download Patch File Download Diff File Expand all files Collapse all files

404
compiler/rustc_codegen_gcc/src/back/lto.rs
View File

@@ -16,13 +16,14 @@
// /usr/bin/ld: warning: type of symbol `_RNvNvNvNtCs5JWOrf9uCus_5rayon11thread_pool19WORKER_THREAD_STATE7___getit5___KEY' changed from 1 to 6 in /tmp/ccKeUSiR.ltrans0.ltrans.o
// /usr/bin/ld: warning: type of symbol `_RNvNvNvNvNtNtNtCsAj5i4SGTR7_3std4sync4mpmc5waker17current_thread_id5DUMMY7___getit5___KEY' changed from 1 to 6 in /tmp/ccKeUSiR.ltrans0.ltrans.o
// /usr/bin/ld: warning: incremental linking of LTO and non-LTO objects; using -flinker-output=nolto-rel which will bypass whole program optimization
use std::ffi::CString;
use std::ffi::{CStr, CString};
use std::fs::{self, File};
use std::path::{Path, PathBuf};
use std::sync::Arc;
use gccjit::OutputKind;
use gccjit::{Context, OutputKind};
use object::read::archive::ArchiveFile;
use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule};
use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule, ThinShared};
use rustc_codegen_ssa::back::symbol_export;
use rustc_codegen_ssa::back::write::{CodegenContext, FatLtoInput};
use rustc_codegen_ssa::traits::*;
@@ -30,6 +31,7 @@ use rustc_codegen_ssa::{looks_like_rust_object_file, ModuleCodegen, ModuleKind};
use rustc_data_structures::memmap::Mmap;
use rustc_errors::{DiagCtxtHandle, FatalError};
use rustc_hir::def_id::LOCAL_CRATE;
use rustc_middle::bug;
use rustc_middle::dep_graph::WorkProduct;
use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel};
use rustc_session::config::{CrateType, Lto};
@@ -37,7 +39,7 @@ use tempfile::{tempdir, TempDir};
use crate::back::write::save_temp_bitcode;
use crate::errors::{DynamicLinkingWithLTO, LtoBitcodeFromRlib, LtoDisallowed, LtoDylib};
use crate::{to_gcc_opt_level, GccCodegenBackend, GccContext};
use crate::{to_gcc_opt_level, GccCodegenBackend, GccContext, SyncContext};
/// We keep track of the computed LTO cache keys from the previous
/// session to determine which CGUs we can reuse.
@@ -128,8 +130,7 @@ fn prepare_lto(
}
let archive_data = unsafe {
Mmap::map(File::open(&path).expect("couldn't open rlib"))
.expect("couldn't map rlib")
Mmap::map(File::open(path).expect("couldn't open rlib")).expect("couldn't map rlib")
};
let archive = ArchiveFile::parse(&*archive_data).expect("wanted an rlib");
let obj_files = archive
@@ -349,6 +350,395 @@ impl ModuleBuffer {
impl ModuleBufferMethods for ModuleBuffer {
fn data(&self) -> &[u8] {
unimplemented!("data not needed for GCC codegen");
&[]
}
}
/// Performs thin LTO by performing necessary global analysis and returning two
/// lists, one of the modules that need optimization and another for modules that
/// can simply be copied over from the incr. comp. cache.
pub(crate) fn run_thin(
cgcx: &CodegenContext<GccCodegenBackend>,
modules: Vec<(String, ThinBuffer)>,
cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
) -> Result<(Vec<LtoModuleCodegen<GccCodegenBackend>>, Vec<WorkProduct>), FatalError> {
let dcx = cgcx.create_dcx();
let dcx = dcx.handle();
let lto_data = prepare_lto(cgcx, dcx)?;
/*let symbols_below_threshold =
symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::<Vec<_>>();*/
if cgcx.opts.cg.linker_plugin_lto.enabled() {
unreachable!(
"We should never reach this case if the LTO step \
is deferred to the linker"
);
}
thin_lto(
cgcx,
dcx,
modules,
lto_data.upstream_modules,
lto_data.tmp_path,
cached_modules, /*, &symbols_below_threshold*/
)
}
pub(crate) fn prepare_thin(
module: ModuleCodegen<GccContext>,
_emit_summary: bool,
) -> (String, ThinBuffer) {
let name = module.name;
//let buffer = ThinBuffer::new(module.module_llvm.context, true, emit_summary);
let buffer = ThinBuffer::new(&module.module_llvm.context);
(name, buffer)
}
/// Prepare "thin" LTO to get run on these modules.
///
/// The general structure of ThinLTO is quite different from the structure of
/// "fat" LTO above. With "fat" LTO all LLVM modules in question are merged into
/// one giant LLVM module, and then we run more optimization passes over this
/// big module after internalizing most symbols. Thin LTO, on the other hand,
/// avoid this large bottleneck through more targeted optimization.
///
/// At a high level Thin LTO looks like:
///
/// 1. Prepare a "summary" of each LLVM module in question which describes
/// the values inside, cost of the values, etc.
/// 2. Merge the summaries of all modules in question into one "index"
/// 3. Perform some global analysis on this index
/// 4. For each module, use the index and analysis calculated previously to
/// perform local transformations on the module, for example inlining
/// small functions from other modules.
/// 5. Run thin-specific optimization passes over each module, and then code
/// generate everything at the end.
///
/// The summary for each module is intended to be quite cheap, and the global
/// index is relatively quite cheap to create as well. As a result, the goal of
/// ThinLTO is to reduce the bottleneck on LTO and enable LTO to be used in more
/// situations. For example one cheap optimization is that we can parallelize
/// all codegen modules, easily making use of all the cores on a machine.
///
/// With all that in mind, the function here is designed at specifically just
/// calculating the *index* for ThinLTO. This index will then be shared amongst
/// all of the `LtoModuleCodegen` units returned below and destroyed once
/// they all go out of scope.
fn thin_lto(
cgcx: &CodegenContext<GccCodegenBackend>,
_dcx: DiagCtxtHandle<'_>,
modules: Vec<(String, ThinBuffer)>,
serialized_modules: Vec<(SerializedModule<ModuleBuffer>, CString)>,
tmp_path: TempDir,
cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
//symbols_below_threshold: &[*const libc::c_char],
) -> Result<(Vec<LtoModuleCodegen<GccCodegenBackend>>, Vec<WorkProduct>), FatalError> {
let _timer = cgcx.prof.generic_activity("LLVM_thin_lto_global_analysis");
info!("going for that thin, thin LTO");
/*let green_modules: FxHashMap<_, _> =
cached_modules.iter().map(|(_, wp)| (wp.cgu_name.clone(), wp.clone())).collect();*/
let full_scope_len = modules.len() + serialized_modules.len() + cached_modules.len();
let mut thin_buffers = Vec::with_capacity(modules.len());
let mut module_names = Vec::with_capacity(full_scope_len);
//let mut thin_modules = Vec::with_capacity(full_scope_len);
for (i, (name, buffer)) in modules.into_iter().enumerate() {
info!("local module: {} - {}", i, name);
let cname = CString::new(name.as_bytes()).unwrap();
/*thin_modules.push(llvm::ThinLTOModule {
identifier: cname.as_ptr(),
data: buffer.data().as_ptr(),
len: buffer.data().len(),
});*/
thin_buffers.push(buffer);
module_names.push(cname);
}
// FIXME: All upstream crates are deserialized internally in the
// function below to extract their summary and modules. Note that
// unlike the loop above we *must* decode and/or read something
// here as these are all just serialized files on disk. An
// improvement, however, to make here would be to store the
// module summary separately from the actual module itself. Right
// now this is store in one large bitcode file, and the entire
// file is deflate-compressed. We could try to bypass some of the
// decompression by storing the index uncompressed and only
// lazily decompressing the bytecode if necessary.
//
// Note that truly taking advantage of this optimization will
// likely be further down the road. We'd have to implement
// incremental ThinLTO first where we could actually avoid
// looking at upstream modules entirely sometimes (the contents,
// we must always unconditionally look at the index).
let mut serialized = Vec::with_capacity(serialized_modules.len() + cached_modules.len());
let cached_modules =
cached_modules.into_iter().map(|(sm, wp)| (sm, CString::new(wp.cgu_name).unwrap()));
for (module, name) in serialized_modules.into_iter().chain(cached_modules) {
info!("upstream or cached module {:?}", name);
/*thin_modules.push(llvm::ThinLTOModule {
identifier: name.as_ptr(),
data: module.data().as_ptr(),
len: module.data().len(),
});*/
match module {
SerializedModule::Local(_) => {
//let path = module_buffer.0.to_str().expect("path");
//let my_path = PathBuf::from(path);
//let exists = my_path.exists();
/*module.module_llvm.should_combine_object_files = true;
module
.module_llvm
.context
.add_driver_option(module_buffer.0.to_str().expect("path"));*/
}
SerializedModule::FromRlib(_) => unimplemented!("from rlib"),
SerializedModule::FromUncompressedFile(_) => {
unimplemented!("from uncompressed file")
}
}
serialized.push(module);
module_names.push(name);
}
// Sanity check
//assert_eq!(thin_modules.len(), module_names.len());
// Delegate to the C++ bindings to create some data here. Once this is a
// tried-and-true interface we may wish to try to upstream some of this
// to LLVM itself, right now we reimplement a lot of what they do
// upstream...
/*let data = llvm::LLVMRustCreateThinLTOData(
thin_modules.as_ptr(),
thin_modules.len() as u32,
symbols_below_threshold.as_ptr(),
symbols_below_threshold.len() as u32,
)
.ok_or_else(|| write::llvm_err(dcx, LlvmError::PrepareThinLtoContext))?;
*/
let data = ThinData; //(Arc::new(tmp_path))/*(data)*/;
info!("thin LTO data created");
/*let (key_map_path, prev_key_map, curr_key_map) =
if let Some(ref incr_comp_session_dir) = cgcx.incr_comp_session_dir {
let path = incr_comp_session_dir.join(THIN_LTO_KEYS_INCR_COMP_FILE_NAME);
// If the previous file was deleted, or we get an IO error
// reading the file, then we'll just use `None` as the
// prev_key_map, which will force the code to be recompiled.
let prev =
if path.exists() { ThinLTOKeysMap::load_from_file(&path).ok() } else { None };
let curr = ThinLTOKeysMap::from_thin_lto_modules(&data, &thin_modules, &module_names);
(Some(path), prev, curr)
}
else {
// If we don't compile incrementally, we don't need to load the
// import data from LLVM.
assert!(green_modules.is_empty());
let curr = ThinLTOKeysMap::default();
(None, None, curr)
};
info!("thin LTO cache key map loaded");
info!("prev_key_map: {:#?}", prev_key_map);
info!("curr_key_map: {:#?}", curr_key_map);*/
// Throw our data in an `Arc` as we'll be sharing it across threads. We
// also put all memory referenced by the C++ data (buffers, ids, etc)
// into the arc as well. After this we'll create a thin module
// codegen per module in this data.
let shared =
Arc::new(ThinShared { data, thin_buffers, serialized_modules: serialized, module_names });
let copy_jobs = vec![];
let mut opt_jobs = vec![];
info!("checking which modules can be-reused and which have to be re-optimized.");
for (module_index, module_name) in shared.module_names.iter().enumerate() {
let module_name = module_name_to_str(module_name);
/*if let (Some(prev_key_map), true) =
(prev_key_map.as_ref(), green_modules.contains_key(module_name))
{
assert!(cgcx.incr_comp_session_dir.is_some());
// If a module exists in both the current and the previous session,
// and has the same LTO cache key in both sessions, then we can re-use it
if prev_key_map.keys.get(module_name) == curr_key_map.keys.get(module_name) {
let work_product = green_modules[module_name].clone();
copy_jobs.push(work_product);
info!(" - {}: re-used", module_name);
assert!(cgcx.incr_comp_session_dir.is_some());
continue;
}
}*/
info!(" - {}: re-compiled", module_name);
opt_jobs
.push(LtoModuleCodegen::Thin(ThinModule { shared: shared.clone(), idx: module_index }));
}
// Save the current ThinLTO import information for the next compilation
// session, overwriting the previous serialized data (if any).
/*if let Some(path) = key_map_path {
if let Err(err) = curr_key_map.save_to_file(&path) {
return Err(write::llvm_err(dcx, LlvmError::WriteThinLtoKey { err }));
}
}*/
// NOTE: save the temporary directory used by LTO so that it gets deleted after linking instead
// of now.
//module.module_llvm.temp_dir = Some(tmp_path);
// TODO: save the directory so that it gets deleted later.
std::mem::forget(tmp_path);
Ok((opt_jobs, copy_jobs))
}
pub unsafe fn optimize_thin_module(
thin_module: ThinModule<GccCodegenBackend>,
_cgcx: &CodegenContext<GccCodegenBackend>,
) -> Result<ModuleCodegen<GccContext>, FatalError> {
//let dcx = cgcx.create_dcx();
//let module_name = &thin_module.shared.module_names[thin_module.idx];
/*let tm_factory_config = TargetMachineFactoryConfig::new(cgcx, module_name.to_str().unwrap());
let tm = (cgcx.tm_factory)(tm_factory_config).map_err(|e| write::llvm_err(&dcx, e))?;*/
// Right now the implementation we've got only works over serialized
// modules, so we create a fresh new LLVM context and parse the module
// into that context. One day, however, we may do this for upstream
// crates but for locally codegened modules we may be able to reuse
// that LLVM Context and Module.
//let llcx = llvm::LLVMRustContextCreate(cgcx.fewer_names);
//let llmod_raw = parse_module(llcx, module_name, thin_module.data(), &dcx)? as *const _;
let mut should_combine_object_files = false;
let context = match thin_module.shared.thin_buffers.get(thin_module.idx) {
Some(thin_buffer) => Arc::clone(&thin_buffer.context),
None => {
let context = Context::default();
let len = thin_module.shared.thin_buffers.len();
let module = &thin_module.shared.serialized_modules[thin_module.idx - len];
match *module {
SerializedModule::Local(ref module_buffer) => {
let path = module_buffer.0.to_str().expect("path");
context.add_driver_option(path);
should_combine_object_files = true;
/*module.module_llvm.should_combine_object_files = true;
module
.module_llvm
.context
.add_driver_option(module_buffer.0.to_str().expect("path"));*/
}
SerializedModule::FromRlib(_) => unimplemented!("from rlib"),
SerializedModule::FromUncompressedFile(_) => {
unimplemented!("from uncompressed file")
}
}
Arc::new(SyncContext::new(context))
}
};
let module = ModuleCodegen {
module_llvm: GccContext { context, should_combine_object_files, temp_dir: None },
name: thin_module.name().to_string(),
kind: ModuleKind::Regular,
};
/*{
let target = &*module.module_llvm.tm;
let llmod = module.module_llvm.llmod();
save_temp_bitcode(cgcx, &module, "thin-lto-input");
// Up next comes the per-module local analyses that we do for Thin LTO.
// Each of these functions is basically copied from the LLVM
// implementation and then tailored to suit this implementation. Ideally
// each of these would be supported by upstream LLVM but that's perhaps
// a patch for another day!
//
// You can find some more comments about these functions in the LLVM
// bindings we've got (currently `PassWrapper.cpp`)
{
let _timer =
cgcx.prof.generic_activity_with_arg("LLVM_thin_lto_rename", thin_module.name());
if !llvm::LLVMRustPrepareThinLTORename(thin_module.shared.data.0, llmod, target) {
return Err(write::llvm_err(&dcx, LlvmError::PrepareThinLtoModule));
}
save_temp_bitcode(cgcx, &module, "thin-lto-after-rename");
}
{
let _timer = cgcx
.prof
.generic_activity_with_arg("LLVM_thin_lto_resolve_weak", thin_module.name());
if !llvm::LLVMRustPrepareThinLTOResolveWeak(thin_module.shared.data.0, llmod) {
return Err(write::llvm_err(&dcx, LlvmError::PrepareThinLtoModule));
}
save_temp_bitcode(cgcx, &module, "thin-lto-after-resolve");
}
{
let _timer = cgcx
.prof
.generic_activity_with_arg("LLVM_thin_lto_internalize", thin_module.name());
if !llvm::LLVMRustPrepareThinLTOInternalize(thin_module.shared.data.0, llmod) {
return Err(write::llvm_err(&dcx, LlvmError::PrepareThinLtoModule));
}
save_temp_bitcode(cgcx, &module, "thin-lto-after-internalize");
}
{
let _timer =
cgcx.prof.generic_activity_with_arg("LLVM_thin_lto_import", thin_module.name());
if !llvm::LLVMRustPrepareThinLTOImport(thin_module.shared.data.0, llmod, target) {
return Err(write::llvm_err(&dcx, LlvmError::PrepareThinLtoModule));
}
save_temp_bitcode(cgcx, &module, "thin-lto-after-import");
}
// Alright now that we've done everything related to the ThinLTO
// analysis it's time to run some optimizations! Here we use the same
// `run_pass_manager` as the "fat" LTO above except that we tell it to
// populate a thin-specific pass manager, which presumably LLVM treats a
// little differently.
{
info!("running thin lto passes over {}", module.name);
run_pass_manager(cgcx, &dcx, &mut module, true)?;
save_temp_bitcode(cgcx, &module, "thin-lto-after-pm");
}
}*/
Ok(module)
}
pub struct ThinBuffer {
context: Arc<SyncContext>,
}
// TODO: check if this makes sense to make ThinBuffer Send and Sync.
unsafe impl Send for ThinBuffer {}
unsafe impl Sync for ThinBuffer {}
impl ThinBuffer {
pub(crate) fn new(context: &Arc<SyncContext>) -> Self {
Self { context: Arc::clone(context) }
}
}
impl ThinBufferMethods for ThinBuffer {
fn data(&self) -> &[u8] {
&[]
}
fn thin_link_data(&self) -> &[u8] {
unimplemented!();
}
}
pub struct ThinData; //(Arc<TempDir>);
fn module_name_to_str(c_str: &CStr) -> &str {
c_str.to_str().unwrap_or_else(|e| {
bug!("Encountered non-utf8 GCC module name `{}`: {}", c_str.to_string_lossy(), e)
})
}

20
compiler/rustc_codegen_gcc/src/back/write.rs
View File

@@ -31,6 +31,7 @@ pub(crate) unsafe fn codegen(
// NOTE: Only generate object files with GIMPLE when this environment variable is set for
// now because this requires a particular setup (same gcc/lto1/lto-wrapper commit as libgccjit).
// TODO: remove this environment variable.
let fat_lto = env::var("EMBED_LTO_BITCODE").as_deref() == Ok("1");
let bc_out = cgcx.output_filenames.temp_path(OutputType::Bitcode, module_name);
@@ -56,6 +57,8 @@ pub(crate) unsafe fn codegen(
.generic_activity_with_arg("GCC_module_codegen_emit_bitcode", &*module.name);
context.add_command_line_option("-flto=auto");
context.add_command_line_option("-flto-partition=one");
// TODO: remove since we don't want fat objects when it is for Bitcode only.
context.add_command_line_option("-ffat-lto-objects");
context
.compile_to_file(OutputKind::ObjectFile, bc_out.to_str().expect("path to str"));
}
@@ -104,7 +107,7 @@ pub(crate) unsafe fn codegen(
// FIXME(antoyo): segfault in dump_reproducer_to_file() might be caused by
// transmuting an rvalue to an lvalue.
// Segfault is actually in gcc::jit::reproducer::get_identifier_as_lvalue
context.dump_reproducer_to_file(&format!("/tmp/reproducers/{}.c", module.name));
context.dump_reproducer_to_file(format!("/tmp/reproducers/{}.c", module.name));
println!("Dumped reproducer {}", module.name);
}
if env::var("CG_GCCJIT_DUMP_TO_FILE").as_deref() == Ok("1") {
@@ -113,17 +116,20 @@ pub(crate) unsafe fn codegen(
context.set_debug_info(true);
context.dump_to_file(path, true);
}
if should_combine_object_files && fat_lto {
context.add_command_line_option("-flto=auto");
context.add_command_line_option("-flto-partition=one");
if should_combine_object_files {
if fat_lto {
context.add_command_line_option("-flto=auto");
context.add_command_line_option("-flto-partition=one");
// NOTE: without -fuse-linker-plugin, we get the following error:
// lto1: internal compiler error: decompressed stream: Destination buffer is too small
context.add_driver_option("-fuse-linker-plugin");
}
context.add_driver_option("-Wl,-r");
// NOTE: we need -nostdlib, otherwise, we get the following error:
// /usr/bin/ld: cannot find -lgcc_s: No such file or directory
context.add_driver_option("-nostdlib");
// NOTE: without -fuse-linker-plugin, we get the following error:
// lto1: internal compiler error: decompressed stream: Destination buffer is too small
context.add_driver_option("-fuse-linker-plugin");
// NOTE: this doesn't actually generate an executable. With the above flags, it combines the .o files together in another .o.
context.compile_to_file(