1use std::collections::BTreeMap;
2use std::ffi::{CStr, CString};
3use std::fs::File;
4use std::path::Path;
5use std::ptr::NonNull;
6use std::sync::Arc;
7use std::{io, iter, slice};
8
9use object::read::archive::ArchiveFile;
10use rustc_codegen_ssa::back::lto::{LtoModuleCodegen, SerializedModule, ThinModule, ThinShared};
11use rustc_codegen_ssa::back::symbol_export;
12use rustc_codegen_ssa::back::write::{CodegenContext, FatLtoInput};
13use rustc_codegen_ssa::traits::*;
14use rustc_codegen_ssa::{ModuleCodegen, ModuleKind, looks_like_rust_object_file};
15use rustc_data_structures::fx::FxHashMap;
16use rustc_data_structures::memmap::Mmap;
17use rustc_errors::{DiagCtxtHandle, FatalError};
18use rustc_hir::def_id::LOCAL_CRATE;
19use rustc_middle::bug;
20use rustc_middle::dep_graph::WorkProduct;
21use rustc_middle::middle::exported_symbols::{SymbolExportInfo, SymbolExportLevel};
22use rustc_session::config::{self, CrateType, Lto};
23use tracing::{debug, info};
24
25use crate::back::write::{
26 self, CodegenDiagnosticsStage, DiagnosticHandlers, bitcode_section_name, save_temp_bitcode,
27};
28use crate::errors::{
29 DynamicLinkingWithLTO, LlvmError, LtoBitcodeFromRlib, LtoDisallowed, LtoDylib, LtoProcMacro,
30};
31use crate::llvm::AttributePlace::Function;
32use crate::llvm::{self, build_string};
33use crate::{LlvmCodegenBackend, ModuleLlvm, SimpleCx, attributes};
34
35const THIN_LTO_KEYS_INCR_COMP_FILE_NAME: &str = "thin-lto-past-keys.bin";
38
39fn crate_type_allows_lto(crate_type: CrateType) -> bool {
40 match crate_type {
41 CrateType::Executable
42 | CrateType::Dylib
43 | CrateType::Staticlib
44 | CrateType::Cdylib
45 | CrateType::ProcMacro
46 | CrateType::Sdylib => true,
47 CrateType::Rlib => false,
48 }
49}
50
51fn prepare_lto(
52 cgcx: &CodegenContext<LlvmCodegenBackend>,
53 dcx: DiagCtxtHandle<'_>,
54) -> Result<(Vec<CString>, Vec<(SerializedModule<ModuleBuffer>, CString)>), FatalError> {
55 let export_threshold = match cgcx.lto {
56 Lto::ThinLocal => SymbolExportLevel::Rust,
58
59 Lto::Fat | Lto::Thin => symbol_export::crates_export_threshold(&cgcx.crate_types),
61
62 Lto::No => panic!("didn't request LTO but we're doing LTO"),
63 };
64
65 let symbol_filter = &|&(ref name, info): &(String, SymbolExportInfo)| {
66 if info.level.is_below_threshold(export_threshold) || info.used {
67 Some(CString::new(name.as_str()).unwrap())
68 } else {
69 None
70 }
71 };
72 let exported_symbols = cgcx.exported_symbols.as_ref().expect("needs exported symbols for LTO");
73 let mut symbols_below_threshold = {
74 let _timer = cgcx.prof.generic_activity("LLVM_lto_generate_symbols_below_threshold");
75 exported_symbols[&LOCAL_CRATE].iter().filter_map(symbol_filter).collect::<Vec<CString>>()
76 };
77 info!("{} symbols to preserve in this crate", symbols_below_threshold.len());
78
79 let mut upstream_modules = Vec::new();
86 if cgcx.lto != Lto::ThinLocal {
87 for crate_type in cgcx.crate_types.iter() {
89 if !crate_type_allows_lto(*crate_type) {
90 dcx.emit_err(LtoDisallowed);
91 return Err(FatalError);
92 } else if *crate_type == CrateType::Dylib {
93 if !cgcx.opts.unstable_opts.dylib_lto {
94 dcx.emit_err(LtoDylib);
95 return Err(FatalError);
96 }
97 } else if *crate_type == CrateType::ProcMacro && !cgcx.opts.unstable_opts.dylib_lto {
98 dcx.emit_err(LtoProcMacro);
99 return Err(FatalError);
100 }
101 }
102
103 if cgcx.opts.cg.prefer_dynamic && !cgcx.opts.unstable_opts.dylib_lto {
104 dcx.emit_err(DynamicLinkingWithLTO);
105 return Err(FatalError);
106 }
107
108 for &(cnum, ref path) in cgcx.each_linked_rlib_for_lto.iter() {
109 let exported_symbols =
110 cgcx.exported_symbols.as_ref().expect("needs exported symbols for LTO");
111 {
112 let _timer =
113 cgcx.prof.generic_activity("LLVM_lto_generate_symbols_below_threshold");
114 symbols_below_threshold
115 .extend(exported_symbols[&cnum].iter().filter_map(symbol_filter));
116 }
117
118 let archive_data = unsafe {
119 Mmap::map(std::fs::File::open(&path).expect("couldn't open rlib"))
120 .expect("couldn't map rlib")
121 };
122 let archive = ArchiveFile::parse(&*archive_data).expect("wanted an rlib");
123 let obj_files = archive
124 .members()
125 .filter_map(|child| {
126 child.ok().and_then(|c| {
127 std::str::from_utf8(c.name()).ok().map(|name| (name.trim(), c))
128 })
129 })
130 .filter(|&(name, _)| looks_like_rust_object_file(name));
131 for (name, child) in obj_files {
132 info!("adding bitcode from {}", name);
133 match get_bitcode_slice_from_object_data(
134 child.data(&*archive_data).expect("corrupt rlib"),
135 cgcx,
136 ) {
137 Ok(data) => {
138 let module = SerializedModule::FromRlib(data.to_vec());
139 upstream_modules.push((module, CString::new(name).unwrap()));
140 }
141 Err(e) => {
142 dcx.emit_err(e);
143 return Err(FatalError);
144 }
145 }
146 }
147 }
148 }
149
150 symbols_below_threshold.push(c"__llvm_profile_counter_bias".to_owned());
154 Ok((symbols_below_threshold, upstream_modules))
155}
156
157fn get_bitcode_slice_from_object_data<'a>(
158 obj: &'a [u8],
159 cgcx: &CodegenContext<LlvmCodegenBackend>,
160) -> Result<&'a [u8], LtoBitcodeFromRlib> {
161 if obj.starts_with(b"\xDE\xC0\x17\x0B") || obj.starts_with(b"BC\xC0\xDE") {
165 return Ok(obj);
166 }
167 let section_name = bitcode_section_name(cgcx).to_str().unwrap().trim_start_matches("__LLVM,");
171 let mut len = 0;
172 let data = unsafe {
173 llvm::LLVMRustGetSliceFromObjectDataByName(
174 obj.as_ptr(),
175 obj.len(),
176 section_name.as_ptr(),
177 section_name.len(),
178 &mut len,
179 )
180 };
181 if !data.is_null() {
182 assert!(len != 0);
183 let bc = unsafe { slice::from_raw_parts(data, len) };
184
185 assert!(obj.as_ptr() <= bc.as_ptr());
187 assert!(bc[bc.len()..bc.len()].as_ptr() <= obj[obj.len()..obj.len()].as_ptr());
188
189 Ok(bc)
190 } else {
191 assert!(len == 0);
192 Err(LtoBitcodeFromRlib {
193 llvm_err: llvm::last_error().unwrap_or_else(|| "unknown LLVM error".to_string()),
194 })
195 }
196}
197
198pub(crate) fn run_fat(
201 cgcx: &CodegenContext<LlvmCodegenBackend>,
202 modules: Vec<FatLtoInput<LlvmCodegenBackend>>,
203 cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
204) -> Result<LtoModuleCodegen<LlvmCodegenBackend>, FatalError> {
205 let dcx = cgcx.create_dcx();
206 let dcx = dcx.handle();
207 let (symbols_below_threshold, upstream_modules) = prepare_lto(cgcx, dcx)?;
208 let symbols_below_threshold =
209 symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::<Vec<_>>();
210 fat_lto(cgcx, dcx, modules, cached_modules, upstream_modules, &symbols_below_threshold)
211}
212
213pub(crate) fn run_thin(
217 cgcx: &CodegenContext<LlvmCodegenBackend>,
218 modules: Vec<(String, ThinBuffer)>,
219 cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
220) -> Result<(Vec<LtoModuleCodegen<LlvmCodegenBackend>>, Vec<WorkProduct>), FatalError> {
221 let dcx = cgcx.create_dcx();
222 let dcx = dcx.handle();
223 let (symbols_below_threshold, upstream_modules) = prepare_lto(cgcx, dcx)?;
224 let symbols_below_threshold =
225 symbols_below_threshold.iter().map(|c| c.as_ptr()).collect::<Vec<_>>();
226 if cgcx.opts.cg.linker_plugin_lto.enabled() {
227 unreachable!(
228 "We should never reach this case if the LTO step \
229 is deferred to the linker"
230 );
231 }
232 thin_lto(cgcx, dcx, modules, upstream_modules, cached_modules, &symbols_below_threshold)
233}
234
235pub(crate) fn prepare_thin(
236 module: ModuleCodegen<ModuleLlvm>,
237 emit_summary: bool,
238) -> (String, ThinBuffer) {
239 let name = module.name;
240 let buffer = ThinBuffer::new(module.module_llvm.llmod(), true, emit_summary);
241 (name, buffer)
242}
243
244fn fat_lto(
245 cgcx: &CodegenContext<LlvmCodegenBackend>,
246 dcx: DiagCtxtHandle<'_>,
247 modules: Vec<FatLtoInput<LlvmCodegenBackend>>,
248 cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
249 mut serialized_modules: Vec<(SerializedModule<ModuleBuffer>, CString)>,
250 symbols_below_threshold: &[*const libc::c_char],
251) -> Result<LtoModuleCodegen<LlvmCodegenBackend>, FatalError> {
252 let _timer = cgcx.prof.generic_activity("LLVM_fat_lto_build_monolithic_module");
253 info!("going for a fat lto");
254
255 let mut in_memory = Vec::new();
266 serialized_modules.extend(cached_modules.into_iter().map(|(buffer, wp)| {
267 info!("pushing cached module {:?}", wp.cgu_name);
268 (buffer, CString::new(wp.cgu_name).unwrap())
269 }));
270 for module in modules {
271 match module {
272 FatLtoInput::InMemory(m) => in_memory.push(m),
273 FatLtoInput::Serialized { name, buffer } => {
274 info!("pushing serialized module {:?}", name);
275 let buffer = SerializedModule::Local(buffer);
276 serialized_modules.push((buffer, CString::new(name).unwrap()));
277 }
278 }
279 }
280
281 let costliest_module = in_memory
291 .iter()
292 .enumerate()
293 .filter(|&(_, module)| module.kind == ModuleKind::Regular)
294 .map(|(i, module)| {
295 let cost = unsafe { llvm::LLVMRustModuleCost(module.module_llvm.llmod()) };
296 (cost, i)
297 })
298 .max();
299
300 let module: ModuleCodegen<ModuleLlvm> = match costliest_module {
306 Some((_cost, i)) => in_memory.remove(i),
307 None => {
308 assert!(!serialized_modules.is_empty(), "must have at least one serialized module");
309 let (buffer, name) = serialized_modules.remove(0);
310 info!("no in-memory regular modules to choose from, parsing {:?}", name);
311 let llvm_module = ModuleLlvm::parse(cgcx, &name, buffer.data(), dcx)?;
312 ModuleCodegen::new_regular(name.into_string().unwrap(), llvm_module)
313 }
314 };
315 {
316 let (llcx, llmod) = {
317 let llvm = &module.module_llvm;
318 (&llvm.llcx, llvm.llmod())
319 };
320 info!("using {:?} as a base module", module.name);
321
322 let _handler =
326 DiagnosticHandlers::new(cgcx, dcx, llcx, &module, CodegenDiagnosticsStage::LTO);
327
328 for module in in_memory {
334 let buffer = ModuleBuffer::new(module.module_llvm.llmod());
335 let llmod_id = CString::new(&module.name[..]).unwrap();
336 serialized_modules.push((SerializedModule::Local(buffer), llmod_id));
337 }
338 serialized_modules.sort_by(|module1, module2| module1.1.cmp(&module2.1));
340
341 let mut linker = Linker::new(llmod);
344 for (bc_decoded, name) in serialized_modules {
345 let _timer = cgcx
346 .prof
347 .generic_activity_with_arg_recorder("LLVM_fat_lto_link_module", |recorder| {
348 recorder.record_arg(format!("{name:?}"))
349 });
350 info!("linking {:?}", name);
351 let data = bc_decoded.data();
352 linker.add(data).map_err(|()| write::llvm_err(dcx, LlvmError::LoadBitcode { name }))?;
353 }
354 drop(linker);
355 save_temp_bitcode(cgcx, &module, "lto.input");
356
357 unsafe {
359 let ptr = symbols_below_threshold.as_ptr();
360 llvm::LLVMRustRunRestrictionPass(
361 llmod,
362 ptr as *const *const libc::c_char,
363 symbols_below_threshold.len() as libc::size_t,
364 );
365 }
366 save_temp_bitcode(cgcx, &module, "lto.after-restriction");
367 }
368
369 Ok(LtoModuleCodegen::Fat(module))
370}
371
372pub(crate) struct Linker<'a>(&'a mut llvm::Linker<'a>);
373
374impl<'a> Linker<'a> {
375 pub(crate) fn new(llmod: &'a llvm::Module) -> Self {
376 unsafe { Linker(llvm::LLVMRustLinkerNew(llmod)) }
377 }
378
379 pub(crate) fn add(&mut self, bytecode: &[u8]) -> Result<(), ()> {
380 unsafe {
381 if llvm::LLVMRustLinkerAdd(
382 self.0,
383 bytecode.as_ptr() as *const libc::c_char,
384 bytecode.len(),
385 ) {
386 Ok(())
387 } else {
388 Err(())
389 }
390 }
391 }
392}
393
394impl Drop for Linker<'_> {
395 fn drop(&mut self) {
396 unsafe {
397 llvm::LLVMRustLinkerFree(&mut *(self.0 as *mut _));
398 }
399 }
400}
401
402fn thin_lto(
433 cgcx: &CodegenContext<LlvmCodegenBackend>,
434 dcx: DiagCtxtHandle<'_>,
435 modules: Vec<(String, ThinBuffer)>,
436 serialized_modules: Vec<(SerializedModule<ModuleBuffer>, CString)>,
437 cached_modules: Vec<(SerializedModule<ModuleBuffer>, WorkProduct)>,
438 symbols_below_threshold: &[*const libc::c_char],
439) -> Result<(Vec<LtoModuleCodegen<LlvmCodegenBackend>>, Vec<WorkProduct>), FatalError> {
440 let _timer = cgcx.prof.generic_activity("LLVM_thin_lto_global_analysis");
441 unsafe {
442 info!("going for that thin, thin LTO");
443
444 let green_modules: FxHashMap<_, _> =
445 cached_modules.iter().map(|(_, wp)| (wp.cgu_name.clone(), wp.clone())).collect();
446
447 let full_scope_len = modules.len() + serialized_modules.len() + cached_modules.len();
448 let mut thin_buffers = Vec::with_capacity(modules.len());
449 let mut module_names = Vec::with_capacity(full_scope_len);
450 let mut thin_modules = Vec::with_capacity(full_scope_len);
451
452 for (i, (name, buffer)) in modules.into_iter().enumerate() {
453 info!("local module: {} - {}", i, name);
454 let cname = CString::new(name.as_bytes()).unwrap();
455 thin_modules.push(llvm::ThinLTOModule {
456 identifier: cname.as_ptr(),
457 data: buffer.data().as_ptr(),
458 len: buffer.data().len(),
459 });
460 thin_buffers.push(buffer);
461 module_names.push(cname);
462 }
463
464 let mut serialized = Vec::with_capacity(serialized_modules.len() + cached_modules.len());
481
482 let cached_modules =
483 cached_modules.into_iter().map(|(sm, wp)| (sm, CString::new(wp.cgu_name).unwrap()));
484
485 for (module, name) in serialized_modules.into_iter().chain(cached_modules) {
486 info!("upstream or cached module {:?}", name);
487 thin_modules.push(llvm::ThinLTOModule {
488 identifier: name.as_ptr(),
489 data: module.data().as_ptr(),
490 len: module.data().len(),
491 });
492 serialized.push(module);
493 module_names.push(name);
494 }
495
496 assert_eq!(thin_modules.len(), module_names.len());
498
499 let data = llvm::LLVMRustCreateThinLTOData(
504 thin_modules.as_ptr(),
505 thin_modules.len(),
506 symbols_below_threshold.as_ptr(),
507 symbols_below_threshold.len(),
508 )
509 .ok_or_else(|| write::llvm_err(dcx, LlvmError::PrepareThinLtoContext))?;
510
511 let data = ThinData(data);
512
513 info!("thin LTO data created");
514
515 let (key_map_path, prev_key_map, curr_key_map) = if let Some(ref incr_comp_session_dir) =
516 cgcx.incr_comp_session_dir
517 {
518 let path = incr_comp_session_dir.join(THIN_LTO_KEYS_INCR_COMP_FILE_NAME);
519 let prev =
523 if path.exists() { ThinLTOKeysMap::load_from_file(&path).ok() } else { None };
524 let curr = ThinLTOKeysMap::from_thin_lto_modules(&data, &thin_modules, &module_names);
525 (Some(path), prev, curr)
526 } else {
527 assert!(green_modules.is_empty());
530 let curr = ThinLTOKeysMap::default();
531 (None, None, curr)
532 };
533 info!("thin LTO cache key map loaded");
534 info!("prev_key_map: {:#?}", prev_key_map);
535 info!("curr_key_map: {:#?}", curr_key_map);
536
537 let shared = Arc::new(ThinShared {
542 data,
543 thin_buffers,
544 serialized_modules: serialized,
545 module_names,
546 });
547
548 let mut copy_jobs = vec![];
549 let mut opt_jobs = vec![];
550
551 info!("checking which modules can be-reused and which have to be re-optimized.");
552 for (module_index, module_name) in shared.module_names.iter().enumerate() {
553 let module_name = module_name_to_str(module_name);
554 if let (Some(prev_key_map), true) =
555 (prev_key_map.as_ref(), green_modules.contains_key(module_name))
556 {
557 assert!(cgcx.incr_comp_session_dir.is_some());
558
559 if prev_key_map.keys.get(module_name) == curr_key_map.keys.get(module_name) {
562 let work_product = green_modules[module_name].clone();
563 copy_jobs.push(work_product);
564 info!(" - {}: re-used", module_name);
565 assert!(cgcx.incr_comp_session_dir.is_some());
566 continue;
567 }
568 }
569
570 info!(" - {}: re-compiled", module_name);
571 opt_jobs.push(LtoModuleCodegen::Thin(ThinModule {
572 shared: Arc::clone(&shared),
573 idx: module_index,
574 }));
575 }
576
577 if let Some(path) = key_map_path {
580 if let Err(err) = curr_key_map.save_to_file(&path) {
581 return Err(write::llvm_err(dcx, LlvmError::WriteThinLtoKey { err }));
582 }
583 }
584
585 Ok((opt_jobs, copy_jobs))
586 }
587}
588
589fn enable_autodiff_settings(ad: &[config::AutoDiff]) {
590 for val in ad {
591 match val {
593 config::AutoDiff::PrintPerf => {
594 llvm::set_print_perf(true);
595 }
596 config::AutoDiff::PrintAA => {
597 llvm::set_print_activity(true);
598 }
599 config::AutoDiff::PrintTA => {
600 llvm::set_print_type(true);
601 }
602 config::AutoDiff::PrintTAFn(fun) => {
603 llvm::set_print_type(true); llvm::set_print_type_fun(&fun); }
606 config::AutoDiff::Inline => {
607 llvm::set_inline(true);
608 }
609 config::AutoDiff::LooseTypes => {
610 llvm::set_loose_types(true);
611 }
612 config::AutoDiff::PrintSteps => {
613 llvm::set_print(true);
614 }
615 config::AutoDiff::PrintPasses => {}
617 config::AutoDiff::PrintModBefore => {}
619 config::AutoDiff::PrintModAfter => {}
621 config::AutoDiff::PrintModFinal => {}
623 config::AutoDiff::Enable => {}
625 config::AutoDiff::NoPostopt => {}
627 }
628 }
629 llvm::set_strict_aliasing(false);
631 llvm::set_rust_rules(true);
633}
634
635pub(crate) fn run_pass_manager(
636 cgcx: &CodegenContext<LlvmCodegenBackend>,
637 dcx: DiagCtxtHandle<'_>,
638 module: &mut ModuleCodegen<ModuleLlvm>,
639 thin: bool,
640) -> Result<(), FatalError> {
641 let _timer = cgcx.prof.generic_activity_with_arg("LLVM_lto_optimize", &*module.name);
642 let config = cgcx.config(module.kind);
643
644 debug!("running the pass manager");
650 let opt_stage = if thin { llvm::OptStage::ThinLTO } else { llvm::OptStage::FatLTO };
651 let opt_level = config.opt_level.unwrap_or(config::OptLevel::No);
652
653 let enable_ad = config.autodiff.contains(&config::AutoDiff::Enable);
660 let stage = if thin {
661 write::AutodiffStage::PreAD
662 } else {
663 if enable_ad { write::AutodiffStage::DuringAD } else { write::AutodiffStage::PostAD }
664 };
665
666 if enable_ad {
667 enable_autodiff_settings(&config.autodiff);
668 }
669
670 unsafe {
671 write::llvm_optimize(cgcx, dcx, module, None, config, opt_level, opt_stage, stage)?;
672 }
673
674 if cfg!(llvm_enzyme) && enable_ad && !thin {
675 let cx =
676 SimpleCx::new(module.module_llvm.llmod(), &module.module_llvm.llcx, cgcx.pointer_size);
677
678 for function in cx.get_functions() {
679 let enzyme_marker = "enzyme_marker";
680 if attributes::has_string_attr(function, enzyme_marker) {
681 assert!(
683 !attributes::has_attr(function, Function, llvm::AttributeKind::NoInline),
684 "Expected __enzyme function to have 'noinline' before adding 'alwaysinline'"
685 );
686
687 attributes::remove_from_llfn(function, Function, llvm::AttributeKind::NoInline);
688 attributes::remove_string_attr_from_llfn(function, enzyme_marker);
689
690 assert!(
691 !attributes::has_string_attr(function, enzyme_marker),
692 "Expected function to not have 'enzyme_marker'"
693 );
694
695 let always_inline = llvm::AttributeKind::AlwaysInline.create_attr(cx.llcx);
696 attributes::apply_to_llfn(function, Function, &[always_inline]);
697 }
698 }
699
700 let opt_stage = llvm::OptStage::FatLTO;
701 let stage = write::AutodiffStage::PostAD;
702 if !config.autodiff.contains(&config::AutoDiff::NoPostopt) {
703 unsafe {
704 write::llvm_optimize(cgcx, dcx, module, None, config, opt_level, opt_stage, stage)?;
705 }
706 }
707
708 if config.autodiff.contains(&config::AutoDiff::PrintModFinal) {
711 unsafe { llvm::LLVMDumpModule(module.module_llvm.llmod()) };
712 }
713 }
714
715 debug!("lto done");
716 Ok(())
717}
718
719pub struct ModuleBuffer(&'static mut llvm::ModuleBuffer);
720
721unsafe impl Send for ModuleBuffer {}
722unsafe impl Sync for ModuleBuffer {}
723
724impl ModuleBuffer {
725 pub(crate) fn new(m: &llvm::Module) -> ModuleBuffer {
726 ModuleBuffer(unsafe { llvm::LLVMRustModuleBufferCreate(m) })
727 }
728}
729
730impl ModuleBufferMethods for ModuleBuffer {
731 fn data(&self) -> &[u8] {
732 unsafe {
733 let ptr = llvm::LLVMRustModuleBufferPtr(self.0);
734 let len = llvm::LLVMRustModuleBufferLen(self.0);
735 slice::from_raw_parts(ptr, len)
736 }
737 }
738}
739
740impl Drop for ModuleBuffer {
741 fn drop(&mut self) {
742 unsafe {
743 llvm::LLVMRustModuleBufferFree(&mut *(self.0 as *mut _));
744 }
745 }
746}
747
748pub struct ThinData(&'static mut llvm::ThinLTOData);
749
750unsafe impl Send for ThinData {}
751unsafe impl Sync for ThinData {}
752
753impl Drop for ThinData {
754 fn drop(&mut self) {
755 unsafe {
756 llvm::LLVMRustFreeThinLTOData(&mut *(self.0 as *mut _));
757 }
758 }
759}
760
761pub struct ThinBuffer(&'static mut llvm::ThinLTOBuffer);
762
763unsafe impl Send for ThinBuffer {}
764unsafe impl Sync for ThinBuffer {}
765
766impl ThinBuffer {
767 pub(crate) fn new(m: &llvm::Module, is_thin: bool, emit_summary: bool) -> ThinBuffer {
768 unsafe {
769 let buffer = llvm::LLVMRustThinLTOBufferCreate(m, is_thin, emit_summary);
770 ThinBuffer(buffer)
771 }
772 }
773
774 pub(crate) unsafe fn from_raw_ptr(ptr: *mut llvm::ThinLTOBuffer) -> ThinBuffer {
775 let mut ptr = NonNull::new(ptr).unwrap();
776 ThinBuffer(unsafe { ptr.as_mut() })
777 }
778}
779
780impl ThinBufferMethods for ThinBuffer {
781 fn data(&self) -> &[u8] {
782 unsafe {
783 let ptr = llvm::LLVMRustThinLTOBufferPtr(self.0) as *const _;
784 let len = llvm::LLVMRustThinLTOBufferLen(self.0);
785 slice::from_raw_parts(ptr, len)
786 }
787 }
788
789 fn thin_link_data(&self) -> &[u8] {
790 unsafe {
791 let ptr = llvm::LLVMRustThinLTOBufferThinLinkDataPtr(self.0) as *const _;
792 let len = llvm::LLVMRustThinLTOBufferThinLinkDataLen(self.0);
793 slice::from_raw_parts(ptr, len)
794 }
795 }
796}
797
798impl Drop for ThinBuffer {
799 fn drop(&mut self) {
800 unsafe {
801 llvm::LLVMRustThinLTOBufferFree(&mut *(self.0 as *mut _));
802 }
803 }
804}
805
806pub(crate) fn optimize_thin_module(
807 thin_module: ThinModule<LlvmCodegenBackend>,
808 cgcx: &CodegenContext<LlvmCodegenBackend>,
809) -> Result<ModuleCodegen<ModuleLlvm>, FatalError> {
810 let dcx = cgcx.create_dcx();
811 let dcx = dcx.handle();
812
813 let module_name = &thin_module.shared.module_names[thin_module.idx];
814
815 let module_llvm = ModuleLlvm::parse(cgcx, module_name, thin_module.data(), dcx)?;
821 let mut module = ModuleCodegen::new_regular(thin_module.name(), module_llvm);
822 if cgcx.config(ModuleKind::Regular).embed_bitcode() {
824 module.thin_lto_buffer = Some(thin_module.data().to_vec());
825 }
826 {
827 let target = &*module.module_llvm.tm;
828 let llmod = module.module_llvm.llmod();
829 save_temp_bitcode(cgcx, &module, "thin-lto-input");
830
831 {
840 let _timer =
841 cgcx.prof.generic_activity_with_arg("LLVM_thin_lto_rename", thin_module.name());
842 unsafe {
843 llvm::LLVMRustPrepareThinLTORename(thin_module.shared.data.0, llmod, target.raw())
844 };
845 save_temp_bitcode(cgcx, &module, "thin-lto-after-rename");
846 }
847
848 {
849 let _timer = cgcx
850 .prof
851 .generic_activity_with_arg("LLVM_thin_lto_resolve_weak", thin_module.name());
852 if unsafe { !llvm::LLVMRustPrepareThinLTOResolveWeak(thin_module.shared.data.0, llmod) }
853 {
854 return Err(write::llvm_err(dcx, LlvmError::PrepareThinLtoModule));
855 }
856 save_temp_bitcode(cgcx, &module, "thin-lto-after-resolve");
857 }
858
859 {
860 let _timer = cgcx
861 .prof
862 .generic_activity_with_arg("LLVM_thin_lto_internalize", thin_module.name());
863 if unsafe { !llvm::LLVMRustPrepareThinLTOInternalize(thin_module.shared.data.0, llmod) }
864 {
865 return Err(write::llvm_err(dcx, LlvmError::PrepareThinLtoModule));
866 }
867 save_temp_bitcode(cgcx, &module, "thin-lto-after-internalize");
868 }
869
870 {
871 let _timer =
872 cgcx.prof.generic_activity_with_arg("LLVM_thin_lto_import", thin_module.name());
873 if unsafe {
874 !llvm::LLVMRustPrepareThinLTOImport(thin_module.shared.data.0, llmod, target.raw())
875 } {
876 return Err(write::llvm_err(dcx, LlvmError::PrepareThinLtoModule));
877 }
878 save_temp_bitcode(cgcx, &module, "thin-lto-after-import");
879 }
880
881 {
887 info!("running thin lto passes over {}", module.name);
888 run_pass_manager(cgcx, dcx, &mut module, true)?;
889 save_temp_bitcode(cgcx, &module, "thin-lto-after-pm");
890 }
891 }
892 Ok(module)
893}
894
895#[derive(Debug, Default)]
897struct ThinLTOKeysMap {
898 keys: BTreeMap<String, String>,
900}
901
902impl ThinLTOKeysMap {
903 fn save_to_file(&self, path: &Path) -> io::Result<()> {
904 use std::io::Write;
905 let mut writer = File::create_buffered(path)?;
906 for (module, key) in &self.keys {
909 writeln!(writer, "{module} {key}")?;
910 }
911 Ok(())
912 }
913
914 fn load_from_file(path: &Path) -> io::Result<Self> {
915 use std::io::BufRead;
916 let mut keys = BTreeMap::default();
917 let file = File::open_buffered(path)?;
918 for line in file.lines() {
919 let line = line?;
920 let mut split = line.split(' ');
921 let module = split.next().unwrap();
922 let key = split.next().unwrap();
923 assert_eq!(split.next(), None, "Expected two space-separated values, found {line:?}");
924 keys.insert(module.to_string(), key.to_string());
925 }
926 Ok(Self { keys })
927 }
928
929 fn from_thin_lto_modules(
930 data: &ThinData,
931 modules: &[llvm::ThinLTOModule],
932 names: &[CString],
933 ) -> Self {
934 let keys = iter::zip(modules, names)
935 .map(|(module, name)| {
936 let key = build_string(|rust_str| unsafe {
937 llvm::LLVMRustComputeLTOCacheKey(rust_str, module.identifier, data.0);
938 })
939 .expect("Invalid ThinLTO module key");
940 (module_name_to_str(name).to_string(), key)
941 })
942 .collect();
943 Self { keys }
944 }
945}
946
947fn module_name_to_str(c_str: &CStr) -> &str {
948 c_str.to_str().unwrap_or_else(|e| {
949 bug!("Encountered non-utf8 LLVM module name `{}`: {}", c_str.to_string_lossy(), e)
950 })
951}
952
953pub(crate) fn parse_module<'a>(
954 cx: &'a llvm::Context,
955 name: &CStr,
956 data: &[u8],
957 dcx: DiagCtxtHandle<'_>,
958) -> Result<&'a llvm::Module, FatalError> {
959 unsafe {
960 llvm::LLVMRustParseBitcodeForLTO(cx, data.as_ptr(), data.len(), name.as_ptr())
961 .ok_or_else(|| write::llvm_err(dcx, LlvmError::ParseBitcode))
962 }
963}