1use std::any::Any;
2use std::assert_matches::assert_matches;
3use std::marker::PhantomData;
4use std::path::{Path, PathBuf};
5use std::sync::Arc;
6use std::sync::mpsc::{Receiver, Sender, channel};
7use std::{fs, io, mem, str, thread};
8
9use rustc_abi::Size;
10use rustc_ast::attr;
11use rustc_ast::expand::autodiff_attrs::AutoDiffItem;
12use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
13use rustc_data_structures::jobserver::{self, Acquired};
14use rustc_data_structures::memmap::Mmap;
15use rustc_data_structures::profiling::{SelfProfilerRef, VerboseTimingGuard};
16use rustc_errors::emitter::Emitter;
17use rustc_errors::translation::Translator;
18use rustc_errors::{
19 Diag, DiagArgMap, DiagCtxt, DiagMessage, ErrCode, FatalError, Level, MultiSpan, Style,
20 Suggestions,
21};
22use rustc_fs_util::link_or_copy;
23use rustc_hir::def_id::{CrateNum, LOCAL_CRATE};
24use rustc_incremental::{
25 copy_cgu_workproduct_to_incr_comp_cache_dir, in_incr_comp_dir, in_incr_comp_dir_sess,
26};
27use rustc_metadata::fs::copy_to_stdout;
28use rustc_middle::bug;
29use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
30use rustc_middle::middle::exported_symbols::SymbolExportInfo;
31use rustc_middle::ty::TyCtxt;
32use rustc_session::Session;
33use rustc_session::config::{
34 self, CrateType, Lto, OutFileName, OutputFilenames, OutputType, Passes, SwitchWithOptPath,
35};
36use rustc_span::source_map::SourceMap;
37use rustc_span::{FileName, InnerSpan, Span, SpanData, sym};
38use rustc_target::spec::{MergeFunctions, SanitizerSet};
39use tracing::debug;
40
41use super::link::{self, ensure_removed};
42use super::lto::{self, SerializedModule};
43use super::symbol_export::symbol_name_for_instance_in_crate;
44use crate::errors::{AutodiffWithoutLto, ErrorCreatingRemarkDir};
45use crate::traits::*;
46use crate::{
47 CachedModuleCodegen, CodegenResults, CompiledModule, CrateInfo, ModuleCodegen, ModuleKind,
48 errors,
49};
50
51const PRE_LTO_BC_EXT: &str = "pre-lto.bc";
52
53#[derive(Clone, Copy, PartialEq)]
55pub enum EmitObj {
56 None,
58
59 Bitcode,
62
63 ObjectCode(BitcodeSection),
65}
66
67#[derive(Clone, Copy, PartialEq)]
69pub enum BitcodeSection {
70 None,
72
73 Full,
75}
76
77pub struct ModuleConfig {
79 pub passes: Vec<String>,
81 pub opt_level: Option<config::OptLevel>,
84
85 pub opt_size: Option<config::OptLevel>,
87
88 pub pgo_gen: SwitchWithOptPath,
89 pub pgo_use: Option<PathBuf>,
90 pub pgo_sample_use: Option<PathBuf>,
91 pub debug_info_for_profiling: bool,
92 pub instrument_coverage: bool,
93
94 pub sanitizer: SanitizerSet,
95 pub sanitizer_recover: SanitizerSet,
96 pub sanitizer_dataflow_abilist: Vec<String>,
97 pub sanitizer_memory_track_origins: usize,
98
99 pub emit_pre_lto_bc: bool,
101 pub emit_no_opt_bc: bool,
102 pub emit_bc: bool,
103 pub emit_ir: bool,
104 pub emit_asm: bool,
105 pub emit_obj: EmitObj,
106 pub emit_thin_lto: bool,
107 pub emit_thin_lto_summary: bool,
108 pub bc_cmdline: String,
109
110 pub verify_llvm_ir: bool,
113 pub lint_llvm_ir: bool,
114 pub no_prepopulate_passes: bool,
115 pub no_builtins: bool,
116 pub time_module: bool,
117 pub vectorize_loop: bool,
118 pub vectorize_slp: bool,
119 pub merge_functions: bool,
120 pub emit_lifetime_markers: bool,
121 pub llvm_plugins: Vec<String>,
122 pub autodiff: Vec<config::AutoDiff>,
123}
124
125impl ModuleConfig {
126 fn new(kind: ModuleKind, tcx: TyCtxt<'_>, no_builtins: bool) -> ModuleConfig {
127 macro_rules! if_regular {
130 ($regular: expr, $other: expr) => {
131 if let ModuleKind::Regular = kind { $regular } else { $other }
132 };
133 }
134
135 let sess = tcx.sess;
136 let opt_level_and_size = if_regular!(Some(sess.opts.optimize), None);
137
138 let save_temps = sess.opts.cg.save_temps;
139
140 let should_emit_obj = sess.opts.output_types.contains_key(&OutputType::Exe)
141 || match kind {
142 ModuleKind::Regular => sess.opts.output_types.contains_key(&OutputType::Object),
143 ModuleKind::Allocator => false,
144 };
145
146 let emit_obj = if !should_emit_obj {
147 EmitObj::None
148 } else if sess.target.obj_is_bitcode
149 || (sess.opts.cg.linker_plugin_lto.enabled() && !no_builtins)
150 {
151 EmitObj::Bitcode
166 } else if need_bitcode_in_object(tcx) {
167 EmitObj::ObjectCode(BitcodeSection::Full)
168 } else {
169 EmitObj::ObjectCode(BitcodeSection::None)
170 };
171
172 ModuleConfig {
173 passes: if_regular!(sess.opts.cg.passes.clone(), vec![]),
174
175 opt_level: opt_level_and_size,
176 opt_size: opt_level_and_size,
177
178 pgo_gen: if_regular!(
179 sess.opts.cg.profile_generate.clone(),
180 SwitchWithOptPath::Disabled
181 ),
182 pgo_use: if_regular!(sess.opts.cg.profile_use.clone(), None),
183 pgo_sample_use: if_regular!(sess.opts.unstable_opts.profile_sample_use.clone(), None),
184 debug_info_for_profiling: sess.opts.unstable_opts.debug_info_for_profiling,
185 instrument_coverage: if_regular!(sess.instrument_coverage(), false),
186
187 sanitizer: if_regular!(sess.opts.unstable_opts.sanitizer, SanitizerSet::empty()),
188 sanitizer_dataflow_abilist: if_regular!(
189 sess.opts.unstable_opts.sanitizer_dataflow_abilist.clone(),
190 Vec::new()
191 ),
192 sanitizer_recover: if_regular!(
193 sess.opts.unstable_opts.sanitizer_recover,
194 SanitizerSet::empty()
195 ),
196 sanitizer_memory_track_origins: if_regular!(
197 sess.opts.unstable_opts.sanitizer_memory_track_origins,
198 0
199 ),
200
201 emit_pre_lto_bc: if_regular!(
202 save_temps || need_pre_lto_bitcode_for_incr_comp(sess),
203 false
204 ),
205 emit_no_opt_bc: if_regular!(save_temps, false),
206 emit_bc: if_regular!(
207 save_temps || sess.opts.output_types.contains_key(&OutputType::Bitcode),
208 save_temps
209 ),
210 emit_ir: if_regular!(
211 sess.opts.output_types.contains_key(&OutputType::LlvmAssembly),
212 false
213 ),
214 emit_asm: if_regular!(
215 sess.opts.output_types.contains_key(&OutputType::Assembly),
216 false
217 ),
218 emit_obj,
219 emit_thin_lto: sess.opts.unstable_opts.emit_thin_lto,
220 emit_thin_lto_summary: if_regular!(
221 sess.opts.output_types.contains_key(&OutputType::ThinLinkBitcode),
222 false
223 ),
224 bc_cmdline: sess.target.bitcode_llvm_cmdline.to_string(),
225
226 verify_llvm_ir: sess.verify_llvm_ir(),
227 lint_llvm_ir: sess.opts.unstable_opts.lint_llvm_ir,
228 no_prepopulate_passes: sess.opts.cg.no_prepopulate_passes,
229 no_builtins: no_builtins || sess.target.no_builtins,
230
231 time_module: if_regular!(true, false),
234
235 vectorize_loop: !sess.opts.cg.no_vectorize_loops
238 && (sess.opts.optimize == config::OptLevel::More
239 || sess.opts.optimize == config::OptLevel::Aggressive),
240 vectorize_slp: !sess.opts.cg.no_vectorize_slp
241 && sess.opts.optimize == config::OptLevel::Aggressive,
242
243 merge_functions: match sess
253 .opts
254 .unstable_opts
255 .merge_functions
256 .unwrap_or(sess.target.merge_functions)
257 {
258 MergeFunctions::Disabled => false,
259 MergeFunctions::Trampolines | MergeFunctions::Aliases => {
260 use config::OptLevel::*;
261 match sess.opts.optimize {
262 Aggressive | More | SizeMin | Size => true,
263 Less | No => false,
264 }
265 }
266 },
267
268 emit_lifetime_markers: sess.emit_lifetime_markers(),
269 llvm_plugins: if_regular!(sess.opts.unstable_opts.llvm_plugins.clone(), vec![]),
270 autodiff: if_regular!(sess.opts.unstable_opts.autodiff.clone(), vec![]),
271 }
272 }
273
274 pub fn bitcode_needed(&self) -> bool {
275 self.emit_bc
276 || self.emit_thin_lto_summary
277 || self.emit_obj == EmitObj::Bitcode
278 || self.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full)
279 }
280
281 pub fn embed_bitcode(&self) -> bool {
282 self.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full)
283 }
284}
285
286pub struct TargetMachineFactoryConfig {
288 pub split_dwarf_file: Option<PathBuf>,
292
293 pub output_obj_file: Option<PathBuf>,
296}
297
298impl TargetMachineFactoryConfig {
299 pub fn new(
300 cgcx: &CodegenContext<impl WriteBackendMethods>,
301 module_name: &str,
302 ) -> TargetMachineFactoryConfig {
303 let split_dwarf_file = if cgcx.target_can_use_split_dwarf {
304 cgcx.output_filenames.split_dwarf_path(
305 cgcx.split_debuginfo,
306 cgcx.split_dwarf_kind,
307 module_name,
308 cgcx.invocation_temp.as_deref(),
309 )
310 } else {
311 None
312 };
313
314 let output_obj_file = Some(cgcx.output_filenames.temp_path_for_cgu(
315 OutputType::Object,
316 module_name,
317 cgcx.invocation_temp.as_deref(),
318 ));
319 TargetMachineFactoryConfig { split_dwarf_file, output_obj_file }
320 }
321}
322
323pub type TargetMachineFactoryFn<B> = Arc<
324 dyn Fn(
325 TargetMachineFactoryConfig,
326 ) -> Result<
327 <B as WriteBackendMethods>::TargetMachine,
328 <B as WriteBackendMethods>::TargetMachineError,
329 > + Send
330 + Sync,
331>;
332
333type ExportedSymbols = FxHashMap<CrateNum, Arc<Vec<(String, SymbolExportInfo)>>>;
334
335#[derive(Clone)]
337pub struct CodegenContext<B: WriteBackendMethods> {
338 pub prof: SelfProfilerRef,
340 pub lto: Lto,
341 pub save_temps: bool,
342 pub fewer_names: bool,
343 pub time_trace: bool,
344 pub exported_symbols: Option<Arc<ExportedSymbols>>,
345 pub opts: Arc<config::Options>,
346 pub crate_types: Vec<CrateType>,
347 pub each_linked_rlib_for_lto: Vec<(CrateNum, PathBuf)>,
348 pub output_filenames: Arc<OutputFilenames>,
349 pub invocation_temp: Option<String>,
350 pub regular_module_config: Arc<ModuleConfig>,
351 pub allocator_module_config: Arc<ModuleConfig>,
352 pub tm_factory: TargetMachineFactoryFn<B>,
353 pub msvc_imps_needed: bool,
354 pub is_pe_coff: bool,
355 pub target_can_use_split_dwarf: bool,
356 pub target_arch: String,
357 pub target_is_like_darwin: bool,
358 pub target_is_like_aix: bool,
359 pub split_debuginfo: rustc_target::spec::SplitDebuginfo,
360 pub split_dwarf_kind: rustc_session::config::SplitDwarfKind,
361 pub pointer_size: Size,
362
363 pub expanded_args: Vec<String>,
368
369 pub diag_emitter: SharedEmitter,
371 pub remark: Passes,
373 pub remark_dir: Option<PathBuf>,
376 pub incr_comp_session_dir: Option<PathBuf>,
379 pub coordinator_send: Sender<Box<dyn Any + Send>>,
381 pub parallel: bool,
385}
386
387impl<B: WriteBackendMethods> CodegenContext<B> {
388 pub fn create_dcx(&self) -> DiagCtxt {
389 DiagCtxt::new(Box::new(self.diag_emitter.clone()))
390 }
391
392 pub fn config(&self, kind: ModuleKind) -> &ModuleConfig {
393 match kind {
394 ModuleKind::Regular => &self.regular_module_config,
395 ModuleKind::Allocator => &self.allocator_module_config,
396 }
397 }
398}
399
400fn generate_lto_work<B: ExtraBackendMethods>(
401 cgcx: &CodegenContext<B>,
402 autodiff: Vec<AutoDiffItem>,
403 needs_fat_lto: Vec<FatLtoInput<B>>,
404 needs_thin_lto: Vec<(String, B::ThinBuffer)>,
405 import_only_modules: Vec<(SerializedModule<B::ModuleBuffer>, WorkProduct)>,
406) -> Vec<(WorkItem<B>, u64)> {
407 let _prof_timer = cgcx.prof.generic_activity("codegen_generate_lto_work");
408
409 if !needs_fat_lto.is_empty() {
410 assert!(needs_thin_lto.is_empty());
411 let mut module =
412 B::run_fat_lto(cgcx, needs_fat_lto, import_only_modules).unwrap_or_else(|e| e.raise());
413 if cgcx.lto == Lto::Fat && !autodiff.is_empty() {
414 let config = cgcx.config(ModuleKind::Regular);
415 module = module.autodiff(cgcx, autodiff, config).unwrap_or_else(|e| e.raise());
416 }
417 vec![(WorkItem::LTO(module), 0)]
419 } else {
420 if !autodiff.is_empty() {
421 let dcx = cgcx.create_dcx();
422 dcx.handle().emit_fatal(AutodiffWithoutLto {});
423 }
424 assert!(needs_fat_lto.is_empty());
425 let (lto_modules, copy_jobs) = B::run_thin_lto(cgcx, needs_thin_lto, import_only_modules)
426 .unwrap_or_else(|e| e.raise());
427 lto_modules
428 .into_iter()
429 .map(|module| {
430 let cost = module.cost();
431 (WorkItem::LTO(module), cost)
432 })
433 .chain(copy_jobs.into_iter().map(|wp| {
434 (
435 WorkItem::CopyPostLtoArtifacts(CachedModuleCodegen {
436 name: wp.cgu_name.clone(),
437 source: wp,
438 }),
439 0, )
441 }))
442 .collect()
443 }
444}
445
446struct CompiledModules {
447 modules: Vec<CompiledModule>,
448 allocator_module: Option<CompiledModule>,
449}
450
451fn need_bitcode_in_object(tcx: TyCtxt<'_>) -> bool {
452 let sess = tcx.sess;
453 sess.opts.cg.embed_bitcode
454 && tcx.crate_types().contains(&CrateType::Rlib)
455 && sess.opts.output_types.contains_key(&OutputType::Exe)
456}
457
458fn need_pre_lto_bitcode_for_incr_comp(sess: &Session) -> bool {
459 if sess.opts.incremental.is_none() {
460 return false;
461 }
462
463 match sess.lto() {
464 Lto::No => false,
465 Lto::Fat | Lto::Thin | Lto::ThinLocal => true,
466 }
467}
468
469pub(crate) fn start_async_codegen<B: ExtraBackendMethods>(
470 backend: B,
471 tcx: TyCtxt<'_>,
472 target_cpu: String,
473) -> OngoingCodegen<B> {
474 let (coordinator_send, coordinator_receive) = channel();
475
476 let crate_attrs = tcx.hir_attrs(rustc_hir::CRATE_HIR_ID);
477 let no_builtins = attr::contains_name(crate_attrs, sym::no_builtins);
478
479 let crate_info = CrateInfo::new(tcx, target_cpu);
480
481 let regular_config = ModuleConfig::new(ModuleKind::Regular, tcx, no_builtins);
482 let allocator_config = ModuleConfig::new(ModuleKind::Allocator, tcx, no_builtins);
483
484 let (shared_emitter, shared_emitter_main) = SharedEmitter::new();
485 let (codegen_worker_send, codegen_worker_receive) = channel();
486
487 let coordinator_thread = start_executing_work(
488 backend.clone(),
489 tcx,
490 &crate_info,
491 shared_emitter,
492 codegen_worker_send,
493 coordinator_receive,
494 Arc::new(regular_config),
495 Arc::new(allocator_config),
496 coordinator_send.clone(),
497 );
498
499 OngoingCodegen {
500 backend,
501 crate_info,
502
503 codegen_worker_receive,
504 shared_emitter_main,
505 coordinator: Coordinator {
506 sender: coordinator_send,
507 future: Some(coordinator_thread),
508 phantom: PhantomData,
509 },
510 output_filenames: Arc::clone(tcx.output_filenames(())),
511 }
512}
513
514fn copy_all_cgu_workproducts_to_incr_comp_cache_dir(
515 sess: &Session,
516 compiled_modules: &CompiledModules,
517) -> FxIndexMap<WorkProductId, WorkProduct> {
518 let mut work_products = FxIndexMap::default();
519
520 if sess.opts.incremental.is_none() {
521 return work_products;
522 }
523
524 let _timer = sess.timer("copy_all_cgu_workproducts_to_incr_comp_cache_dir");
525
526 for module in compiled_modules.modules.iter().filter(|m| m.kind == ModuleKind::Regular) {
527 let mut files = Vec::new();
528 if let Some(object_file_path) = &module.object {
529 files.push((OutputType::Object.extension(), object_file_path.as_path()));
530 }
531 if let Some(dwarf_object_file_path) = &module.dwarf_object {
532 files.push(("dwo", dwarf_object_file_path.as_path()));
533 }
534 if let Some(path) = &module.assembly {
535 files.push((OutputType::Assembly.extension(), path.as_path()));
536 }
537 if let Some(path) = &module.llvm_ir {
538 files.push((OutputType::LlvmAssembly.extension(), path.as_path()));
539 }
540 if let Some(path) = &module.bytecode {
541 files.push((OutputType::Bitcode.extension(), path.as_path()));
542 }
543 if let Some((id, product)) = copy_cgu_workproduct_to_incr_comp_cache_dir(
544 sess,
545 &module.name,
546 files.as_slice(),
547 &module.links_from_incr_cache,
548 ) {
549 work_products.insert(id, product);
550 }
551 }
552
553 work_products
554}
555
556fn produce_final_output_artifacts(
557 sess: &Session,
558 compiled_modules: &CompiledModules,
559 crate_output: &OutputFilenames,
560) {
561 let mut user_wants_bitcode = false;
562 let mut user_wants_objects = false;
563
564 let copy_gracefully = |from: &Path, to: &OutFileName| match to {
566 OutFileName::Stdout if let Err(e) = copy_to_stdout(from) => {
567 sess.dcx().emit_err(errors::CopyPath::new(from, to.as_path(), e));
568 }
569 OutFileName::Real(path) if let Err(e) = fs::copy(from, path) => {
570 sess.dcx().emit_err(errors::CopyPath::new(from, path, e));
571 }
572 _ => {}
573 };
574
575 let copy_if_one_unit = |output_type: OutputType, keep_numbered: bool| {
576 if let [module] = &compiled_modules.modules[..] {
577 let path = crate_output.temp_path_for_cgu(
580 output_type,
581 &module.name,
582 sess.invocation_temp.as_deref(),
583 );
584 let output = crate_output.path(output_type);
585 if !output_type.is_text_output() && output.is_tty() {
586 sess.dcx()
587 .emit_err(errors::BinaryOutputToTty { shorthand: output_type.shorthand() });
588 } else {
589 copy_gracefully(&path, &output);
590 }
591 if !sess.opts.cg.save_temps && !keep_numbered {
592 ensure_removed(sess.dcx(), &path);
594 }
595 } else {
596 if crate_output.outputs.contains_explicit_name(&output_type) {
597 sess.dcx()
600 .emit_warn(errors::IgnoringEmitPath { extension: output_type.extension() });
601 } else if crate_output.single_output_file.is_some() {
602 sess.dcx().emit_warn(errors::IgnoringOutput { extension: output_type.extension() });
605 } else {
606 }
610 }
611 };
612
613 for output_type in crate_output.outputs.keys() {
617 match *output_type {
618 OutputType::Bitcode => {
619 user_wants_bitcode = true;
620 copy_if_one_unit(OutputType::Bitcode, true);
624 }
625 OutputType::ThinLinkBitcode => {
626 copy_if_one_unit(OutputType::ThinLinkBitcode, false);
627 }
628 OutputType::LlvmAssembly => {
629 copy_if_one_unit(OutputType::LlvmAssembly, false);
630 }
631 OutputType::Assembly => {
632 copy_if_one_unit(OutputType::Assembly, false);
633 }
634 OutputType::Object => {
635 user_wants_objects = true;
636 copy_if_one_unit(OutputType::Object, true);
637 }
638 OutputType::Mir | OutputType::Metadata | OutputType::Exe | OutputType::DepInfo => {}
639 }
640 }
641
642 if !sess.opts.cg.save_temps {
655 let needs_crate_object = crate_output.outputs.contains_key(&OutputType::Exe);
671
672 let keep_numbered_bitcode = user_wants_bitcode && sess.codegen_units().as_usize() > 1;
673
674 let keep_numbered_objects =
675 needs_crate_object || (user_wants_objects && sess.codegen_units().as_usize() > 1);
676
677 for module in compiled_modules.modules.iter() {
678 if !keep_numbered_objects {
679 if let Some(ref path) = module.object {
680 ensure_removed(sess.dcx(), path);
681 }
682
683 if let Some(ref path) = module.dwarf_object {
684 ensure_removed(sess.dcx(), path);
685 }
686 }
687
688 if let Some(ref path) = module.bytecode {
689 if !keep_numbered_bitcode {
690 ensure_removed(sess.dcx(), path);
691 }
692 }
693 }
694
695 if !user_wants_bitcode
696 && let Some(ref allocator_module) = compiled_modules.allocator_module
697 && let Some(ref path) = allocator_module.bytecode
698 {
699 ensure_removed(sess.dcx(), path);
700 }
701 }
702
703 if sess.opts.json_artifact_notifications {
704 if let [module] = &compiled_modules.modules[..] {
705 module.for_each_output(|_path, ty| {
706 if sess.opts.output_types.contains_key(&ty) {
707 let descr = ty.shorthand();
708 let path = crate_output.path(ty);
711 sess.dcx().emit_artifact_notification(path.as_path(), descr);
712 }
713 });
714 } else {
715 for module in &compiled_modules.modules {
716 module.for_each_output(|path, ty| {
717 if sess.opts.output_types.contains_key(&ty) {
718 let descr = ty.shorthand();
719 sess.dcx().emit_artifact_notification(&path, descr);
720 }
721 });
722 }
723 }
724 }
725
726 }
732
733pub(crate) enum WorkItem<B: WriteBackendMethods> {
734 Optimize(ModuleCodegen<B::Module>),
736 CopyPostLtoArtifacts(CachedModuleCodegen),
739 LTO(lto::LtoModuleCodegen<B>),
741}
742
743impl<B: WriteBackendMethods> WorkItem<B> {
744 fn module_kind(&self) -> ModuleKind {
745 match *self {
746 WorkItem::Optimize(ref m) => m.kind,
747 WorkItem::CopyPostLtoArtifacts(_) | WorkItem::LTO(_) => ModuleKind::Regular,
748 }
749 }
750
751 fn short_description(&self) -> String {
753 #[cfg(not(windows))]
757 fn desc(short: &str, _long: &str, name: &str) -> String {
758 assert_eq!(short.len(), 3);
778 let name = if let Some(index) = name.find("-cgu.") {
779 &name[index + 1..] } else {
781 name
782 };
783 format!("{short} {name}")
784 }
785
786 #[cfg(windows)]
788 fn desc(_short: &str, long: &str, name: &str) -> String {
789 format!("{long} {name}")
790 }
791
792 match self {
793 WorkItem::Optimize(m) => desc("opt", "optimize module", &m.name),
794 WorkItem::CopyPostLtoArtifacts(m) => desc("cpy", "copy LTO artifacts for", &m.name),
795 WorkItem::LTO(m) => desc("lto", "LTO module", m.name()),
796 }
797 }
798}
799
800pub(crate) enum WorkItemResult<B: WriteBackendMethods> {
802 Finished(CompiledModule),
804
805 NeedsLink(ModuleCodegen<B::Module>),
808
809 NeedsFatLto(FatLtoInput<B>),
812
813 NeedsThinLto(String, B::ThinBuffer),
816}
817
818pub enum FatLtoInput<B: WriteBackendMethods> {
819 Serialized { name: String, buffer: B::ModuleBuffer },
820 InMemory(ModuleCodegen<B::Module>),
821}
822
823pub(crate) enum ComputedLtoType {
825 No,
826 Thin,
827 Fat,
828}
829
830pub(crate) fn compute_per_cgu_lto_type(
831 sess_lto: &Lto,
832 opts: &config::Options,
833 sess_crate_types: &[CrateType],
834 module_kind: ModuleKind,
835) -> ComputedLtoType {
836 let linker_does_lto = opts.cg.linker_plugin_lto.enabled();
840
841 let is_allocator = module_kind == ModuleKind::Allocator;
846
847 let is_rlib = matches!(sess_crate_types, [CrateType::Rlib]);
856
857 match sess_lto {
858 Lto::ThinLocal if !linker_does_lto && !is_allocator => ComputedLtoType::Thin,
859 Lto::Thin if !linker_does_lto && !is_rlib => ComputedLtoType::Thin,
860 Lto::Fat if !is_rlib => ComputedLtoType::Fat,
861 _ => ComputedLtoType::No,
862 }
863}
864
865fn execute_optimize_work_item<B: ExtraBackendMethods>(
866 cgcx: &CodegenContext<B>,
867 mut module: ModuleCodegen<B::Module>,
868 module_config: &ModuleConfig,
869) -> Result<WorkItemResult<B>, FatalError> {
870 let dcx = cgcx.create_dcx();
871 let dcx = dcx.handle();
872
873 B::optimize(cgcx, dcx, &mut module, module_config)?;
874
875 let lto_type = compute_per_cgu_lto_type(&cgcx.lto, &cgcx.opts, &cgcx.crate_types, module.kind);
881
882 let bitcode = if cgcx.config(module.kind).emit_pre_lto_bc {
885 let filename = pre_lto_bitcode_filename(&module.name);
886 cgcx.incr_comp_session_dir.as_ref().map(|path| path.join(&filename))
887 } else {
888 None
889 };
890
891 match lto_type {
892 ComputedLtoType::No => finish_intra_module_work(cgcx, module, module_config),
893 ComputedLtoType::Thin => {
894 let (name, thin_buffer) = B::prepare_thin(module, false);
895 if let Some(path) = bitcode {
896 fs::write(&path, thin_buffer.data()).unwrap_or_else(|e| {
897 panic!("Error writing pre-lto-bitcode file `{}`: {}", path.display(), e);
898 });
899 }
900 Ok(WorkItemResult::NeedsThinLto(name, thin_buffer))
901 }
902 ComputedLtoType::Fat => match bitcode {
903 Some(path) => {
904 let (name, buffer) = B::serialize_module(module);
905 fs::write(&path, buffer.data()).unwrap_or_else(|e| {
906 panic!("Error writing pre-lto-bitcode file `{}`: {}", path.display(), e);
907 });
908 Ok(WorkItemResult::NeedsFatLto(FatLtoInput::Serialized { name, buffer }))
909 }
910 None => Ok(WorkItemResult::NeedsFatLto(FatLtoInput::InMemory(module))),
911 },
912 }
913}
914
915fn execute_copy_from_cache_work_item<B: ExtraBackendMethods>(
916 cgcx: &CodegenContext<B>,
917 module: CachedModuleCodegen,
918 module_config: &ModuleConfig,
919) -> WorkItemResult<B> {
920 let incr_comp_session_dir = cgcx.incr_comp_session_dir.as_ref().unwrap();
921
922 let mut links_from_incr_cache = Vec::new();
923
924 let mut load_from_incr_comp_dir = |output_path: PathBuf, saved_path: &str| {
925 let source_file = in_incr_comp_dir(incr_comp_session_dir, saved_path);
926 debug!(
927 "copying preexisting module `{}` from {:?} to {}",
928 module.name,
929 source_file,
930 output_path.display()
931 );
932 match link_or_copy(&source_file, &output_path) {
933 Ok(_) => {
934 links_from_incr_cache.push(source_file);
935 Some(output_path)
936 }
937 Err(error) => {
938 cgcx.create_dcx().handle().emit_err(errors::CopyPathBuf {
939 source_file,
940 output_path,
941 error,
942 });
943 None
944 }
945 }
946 };
947
948 let dwarf_object =
949 module.source.saved_files.get("dwo").as_ref().and_then(|saved_dwarf_object_file| {
950 let dwarf_obj_out = cgcx
951 .output_filenames
952 .split_dwarf_path(
953 cgcx.split_debuginfo,
954 cgcx.split_dwarf_kind,
955 &module.name,
956 cgcx.invocation_temp.as_deref(),
957 )
958 .expect(
959 "saved dwarf object in work product but `split_dwarf_path` returned `None`",
960 );
961 load_from_incr_comp_dir(dwarf_obj_out, saved_dwarf_object_file)
962 });
963
964 let mut load_from_incr_cache = |perform, output_type: OutputType| {
965 if perform {
966 let saved_file = module.source.saved_files.get(output_type.extension())?;
967 let output_path = cgcx.output_filenames.temp_path_for_cgu(
968 output_type,
969 &module.name,
970 cgcx.invocation_temp.as_deref(),
971 );
972 load_from_incr_comp_dir(output_path, &saved_file)
973 } else {
974 None
975 }
976 };
977
978 let should_emit_obj = module_config.emit_obj != EmitObj::None;
979 let assembly = load_from_incr_cache(module_config.emit_asm, OutputType::Assembly);
980 let llvm_ir = load_from_incr_cache(module_config.emit_ir, OutputType::LlvmAssembly);
981 let bytecode = load_from_incr_cache(module_config.emit_bc, OutputType::Bitcode);
982 let object = load_from_incr_cache(should_emit_obj, OutputType::Object);
983 if should_emit_obj && object.is_none() {
984 cgcx.create_dcx().handle().emit_fatal(errors::NoSavedObjectFile { cgu_name: &module.name })
985 }
986
987 WorkItemResult::Finished(CompiledModule {
988 links_from_incr_cache,
989 name: module.name,
990 kind: ModuleKind::Regular,
991 object,
992 dwarf_object,
993 bytecode,
994 assembly,
995 llvm_ir,
996 })
997}
998
999fn execute_lto_work_item<B: ExtraBackendMethods>(
1000 cgcx: &CodegenContext<B>,
1001 module: lto::LtoModuleCodegen<B>,
1002 module_config: &ModuleConfig,
1003) -> Result<WorkItemResult<B>, FatalError> {
1004 let module = module.optimize(cgcx)?;
1005 finish_intra_module_work(cgcx, module, module_config)
1006}
1007
1008fn finish_intra_module_work<B: ExtraBackendMethods>(
1009 cgcx: &CodegenContext<B>,
1010 module: ModuleCodegen<B::Module>,
1011 module_config: &ModuleConfig,
1012) -> Result<WorkItemResult<B>, FatalError> {
1013 let dcx = cgcx.create_dcx();
1014 let dcx = dcx.handle();
1015
1016 if !cgcx.opts.unstable_opts.combine_cgu || module.kind == ModuleKind::Allocator {
1017 let module = B::codegen(cgcx, dcx, module, module_config)?;
1018 Ok(WorkItemResult::Finished(module))
1019 } else {
1020 Ok(WorkItemResult::NeedsLink(module))
1021 }
1022}
1023
1024pub(crate) enum Message<B: WriteBackendMethods> {
1026 Token(io::Result<Acquired>),
1029
1030 WorkItem { result: Result<WorkItemResult<B>, Option<WorkerFatalError>>, worker_id: usize },
1033
1034 AddAutoDiffItems(Vec<AutoDiffItem>),
1036
1037 CodegenDone { llvm_work_item: WorkItem<B>, cost: u64 },
1041
1042 AddImportOnlyModule {
1045 module_data: SerializedModule<B::ModuleBuffer>,
1046 work_product: WorkProduct,
1047 },
1048
1049 CodegenComplete,
1052
1053 CodegenAborted,
1056}
1057
1058pub struct CguMessage;
1061
1062struct Diagnostic {
1072 level: Level,
1073 messages: Vec<(DiagMessage, Style)>,
1074 code: Option<ErrCode>,
1075 children: Vec<Subdiagnostic>,
1076 args: DiagArgMap,
1077}
1078
1079pub(crate) struct Subdiagnostic {
1083 level: Level,
1084 messages: Vec<(DiagMessage, Style)>,
1085}
1086
1087#[derive(PartialEq, Clone, Copy, Debug)]
1088enum MainThreadState {
1089 Idle,
1091
1092 Codegenning,
1094
1095 Lending,
1097}
1098
1099fn start_executing_work<B: ExtraBackendMethods>(
1100 backend: B,
1101 tcx: TyCtxt<'_>,
1102 crate_info: &CrateInfo,
1103 shared_emitter: SharedEmitter,
1104 codegen_worker_send: Sender<CguMessage>,
1105 coordinator_receive: Receiver<Box<dyn Any + Send>>,
1106 regular_config: Arc<ModuleConfig>,
1107 allocator_config: Arc<ModuleConfig>,
1108 tx_to_llvm_workers: Sender<Box<dyn Any + Send>>,
1109) -> thread::JoinHandle<Result<CompiledModules, ()>> {
1110 let coordinator_send = tx_to_llvm_workers;
1111 let sess = tcx.sess;
1112
1113 let mut each_linked_rlib_for_lto = Vec::new();
1114 drop(link::each_linked_rlib(crate_info, None, &mut |cnum, path| {
1115 if link::ignored_for_lto(sess, crate_info, cnum) {
1116 return;
1117 }
1118 each_linked_rlib_for_lto.push((cnum, path.to_path_buf()));
1119 }));
1120
1121 let exported_symbols = {
1123 let mut exported_symbols = FxHashMap::default();
1124
1125 let copy_symbols = |cnum| {
1126 let symbols = tcx
1127 .exported_symbols(cnum)
1128 .iter()
1129 .map(|&(s, lvl)| (symbol_name_for_instance_in_crate(tcx, s, cnum), lvl))
1130 .collect();
1131 Arc::new(symbols)
1132 };
1133
1134 match sess.lto() {
1135 Lto::No => None,
1136 Lto::ThinLocal => {
1137 exported_symbols.insert(LOCAL_CRATE, copy_symbols(LOCAL_CRATE));
1138 Some(Arc::new(exported_symbols))
1139 }
1140 Lto::Fat | Lto::Thin => {
1141 exported_symbols.insert(LOCAL_CRATE, copy_symbols(LOCAL_CRATE));
1142 for &(cnum, ref _path) in &each_linked_rlib_for_lto {
1143 exported_symbols.insert(cnum, copy_symbols(cnum));
1144 }
1145 Some(Arc::new(exported_symbols))
1146 }
1147 }
1148 };
1149
1150 let coordinator_send2 = coordinator_send.clone();
1156 let helper = jobserver::client()
1157 .into_helper_thread(move |token| {
1158 drop(coordinator_send2.send(Box::new(Message::Token::<B>(token))));
1159 })
1160 .expect("failed to spawn helper thread");
1161
1162 let ol =
1163 if tcx.sess.opts.unstable_opts.no_codegen || !tcx.sess.opts.output_types.should_codegen() {
1164 config::OptLevel::No
1166 } else {
1167 tcx.backend_optimization_level(())
1168 };
1169 let backend_features = tcx.global_backend_features(());
1170
1171 let remark_dir = if let Some(ref dir) = sess.opts.unstable_opts.remark_dir {
1172 let result = fs::create_dir_all(dir).and_then(|_| dir.canonicalize());
1173 match result {
1174 Ok(dir) => Some(dir),
1175 Err(error) => sess.dcx().emit_fatal(ErrorCreatingRemarkDir { error }),
1176 }
1177 } else {
1178 None
1179 };
1180
1181 let cgcx = CodegenContext::<B> {
1182 crate_types: tcx.crate_types().to_vec(),
1183 each_linked_rlib_for_lto,
1184 lto: sess.lto(),
1185 fewer_names: sess.fewer_names(),
1186 save_temps: sess.opts.cg.save_temps,
1187 time_trace: sess.opts.unstable_opts.llvm_time_trace,
1188 opts: Arc::new(sess.opts.clone()),
1189 prof: sess.prof.clone(),
1190 exported_symbols,
1191 remark: sess.opts.cg.remark.clone(),
1192 remark_dir,
1193 incr_comp_session_dir: sess.incr_comp_session_dir_opt().map(|r| r.clone()),
1194 coordinator_send,
1195 expanded_args: tcx.sess.expanded_args.clone(),
1196 diag_emitter: shared_emitter.clone(),
1197 output_filenames: Arc::clone(tcx.output_filenames(())),
1198 regular_module_config: regular_config,
1199 allocator_module_config: allocator_config,
1200 tm_factory: backend.target_machine_factory(tcx.sess, ol, backend_features),
1201 msvc_imps_needed: msvc_imps_needed(tcx),
1202 is_pe_coff: tcx.sess.target.is_like_windows,
1203 target_can_use_split_dwarf: tcx.sess.target_can_use_split_dwarf(),
1204 target_arch: tcx.sess.target.arch.to_string(),
1205 target_is_like_darwin: tcx.sess.target.is_like_darwin,
1206 target_is_like_aix: tcx.sess.target.is_like_aix,
1207 split_debuginfo: tcx.sess.split_debuginfo(),
1208 split_dwarf_kind: tcx.sess.opts.unstable_opts.split_dwarf_kind,
1209 parallel: backend.supports_parallel() && !sess.opts.unstable_opts.no_parallel_backend,
1210 pointer_size: tcx.data_layout.pointer_size,
1211 invocation_temp: sess.invocation_temp.clone(),
1212 };
1213
1214 return B::spawn_named_thread(cgcx.time_trace, "coordinator".to_string(), move || {
1350 let mut worker_id_counter = 0;
1351 let mut free_worker_ids = Vec::new();
1352 let mut get_worker_id = |free_worker_ids: &mut Vec<usize>| {
1353 if let Some(id) = free_worker_ids.pop() {
1354 id
1355 } else {
1356 let id = worker_id_counter;
1357 worker_id_counter += 1;
1358 id
1359 }
1360 };
1361
1362 let mut autodiff_items = Vec::new();
1365 let mut compiled_modules = vec![];
1366 let mut compiled_allocator_module = None;
1367 let mut needs_link = Vec::new();
1368 let mut needs_fat_lto = Vec::new();
1369 let mut needs_thin_lto = Vec::new();
1370 let mut lto_import_only_modules = Vec::new();
1371 let mut started_lto = false;
1372
1373 #[derive(Debug, PartialEq)]
1378 enum CodegenState {
1379 Ongoing,
1380 Completed,
1381 Aborted,
1382 }
1383 use CodegenState::*;
1384 let mut codegen_state = Ongoing;
1385
1386 let mut work_items = Vec::<(WorkItem<B>, u64)>::new();
1388
1389 let mut tokens = Vec::new();
1392
1393 let mut main_thread_state = MainThreadState::Idle;
1394
1395 let mut running_with_own_token = 0;
1398
1399 let running_with_any_token = |main_thread_state, running_with_own_token| {
1402 running_with_own_token
1403 + if main_thread_state == MainThreadState::Lending { 1 } else { 0 }
1404 };
1405
1406 let mut llvm_start_time: Option<VerboseTimingGuard<'_>> = None;
1407
1408 loop {
1414 if codegen_state == Ongoing {
1418 if main_thread_state == MainThreadState::Idle {
1419 let extra_tokens = tokens.len().checked_sub(running_with_own_token).unwrap();
1427 let additional_running = std::cmp::min(extra_tokens, work_items.len());
1428 let anticipated_running = running_with_own_token + additional_running + 1;
1429
1430 if !queue_full_enough(work_items.len(), anticipated_running) {
1431 if codegen_worker_send.send(CguMessage).is_err() {
1433 panic!("Could not send CguMessage to main thread")
1434 }
1435 main_thread_state = MainThreadState::Codegenning;
1436 } else {
1437 let (item, _) =
1441 work_items.pop().expect("queue empty - queue_full_enough() broken?");
1442 main_thread_state = MainThreadState::Lending;
1443 spawn_work(
1444 &cgcx,
1445 &mut llvm_start_time,
1446 get_worker_id(&mut free_worker_ids),
1447 item,
1448 );
1449 }
1450 }
1451 } else if codegen_state == Completed {
1452 if running_with_any_token(main_thread_state, running_with_own_token) == 0
1453 && work_items.is_empty()
1454 {
1455 if needs_fat_lto.is_empty()
1457 && needs_thin_lto.is_empty()
1458 && lto_import_only_modules.is_empty()
1459 {
1460 break;
1462 }
1463
1464 assert!(!started_lto);
1470 started_lto = true;
1471
1472 let needs_fat_lto = mem::take(&mut needs_fat_lto);
1473 let needs_thin_lto = mem::take(&mut needs_thin_lto);
1474 let import_only_modules = mem::take(&mut lto_import_only_modules);
1475
1476 for (work, cost) in generate_lto_work(
1477 &cgcx,
1478 autodiff_items.clone(),
1479 needs_fat_lto,
1480 needs_thin_lto,
1481 import_only_modules,
1482 ) {
1483 let insertion_index = work_items
1484 .binary_search_by_key(&cost, |&(_, cost)| cost)
1485 .unwrap_or_else(|e| e);
1486 work_items.insert(insertion_index, (work, cost));
1487 if cgcx.parallel {
1488 helper.request_token();
1489 }
1490 }
1491 }
1492
1493 match main_thread_state {
1497 MainThreadState::Idle => {
1498 if let Some((item, _)) = work_items.pop() {
1499 main_thread_state = MainThreadState::Lending;
1500 spawn_work(
1501 &cgcx,
1502 &mut llvm_start_time,
1503 get_worker_id(&mut free_worker_ids),
1504 item,
1505 );
1506 } else {
1507 assert!(running_with_own_token > 0);
1514 running_with_own_token -= 1;
1515 main_thread_state = MainThreadState::Lending;
1516 }
1517 }
1518 MainThreadState::Codegenning => bug!(
1519 "codegen worker should not be codegenning after \
1520 codegen was already completed"
1521 ),
1522 MainThreadState::Lending => {
1523 }
1525 }
1526 } else {
1527 assert!(codegen_state == Aborted);
1530 if running_with_any_token(main_thread_state, running_with_own_token) == 0 {
1531 break;
1532 }
1533 }
1534
1535 if codegen_state != Aborted {
1538 while running_with_own_token < tokens.len()
1539 && let Some((item, _)) = work_items.pop()
1540 {
1541 spawn_work(
1542 &cgcx,
1543 &mut llvm_start_time,
1544 get_worker_id(&mut free_worker_ids),
1545 item,
1546 );
1547 running_with_own_token += 1;
1548 }
1549 }
1550
1551 tokens.truncate(running_with_own_token);
1553
1554 let mut free_worker = |worker_id| {
1560 if main_thread_state == MainThreadState::Lending {
1561 main_thread_state = MainThreadState::Idle;
1562 } else {
1563 running_with_own_token -= 1;
1564 }
1565
1566 free_worker_ids.push(worker_id);
1567 };
1568
1569 let msg = coordinator_receive.recv().unwrap();
1570 match *msg.downcast::<Message<B>>().ok().unwrap() {
1571 Message::Token(token) => {
1575 match token {
1576 Ok(token) => {
1577 tokens.push(token);
1578
1579 if main_thread_state == MainThreadState::Lending {
1580 main_thread_state = MainThreadState::Idle;
1585 running_with_own_token += 1;
1586 }
1587 }
1588 Err(e) => {
1589 let msg = &format!("failed to acquire jobserver token: {e}");
1590 shared_emitter.fatal(msg);
1591 codegen_state = Aborted;
1592 }
1593 }
1594 }
1595
1596 Message::CodegenDone { llvm_work_item, cost } => {
1597 let insertion_index = work_items.binary_search_by_key(&cost, |&(_, cost)| cost);
1606 let insertion_index = match insertion_index {
1607 Ok(idx) | Err(idx) => idx,
1608 };
1609 work_items.insert(insertion_index, (llvm_work_item, cost));
1610
1611 if cgcx.parallel {
1612 helper.request_token();
1613 }
1614 assert_eq!(main_thread_state, MainThreadState::Codegenning);
1615 main_thread_state = MainThreadState::Idle;
1616 }
1617
1618 Message::AddAutoDiffItems(mut items) => {
1619 autodiff_items.append(&mut items);
1620 }
1621
1622 Message::CodegenComplete => {
1623 if codegen_state != Aborted {
1624 codegen_state = Completed;
1625 }
1626 assert_eq!(main_thread_state, MainThreadState::Codegenning);
1627 main_thread_state = MainThreadState::Idle;
1628 }
1629
1630 Message::CodegenAborted => {
1638 codegen_state = Aborted;
1639 }
1640
1641 Message::WorkItem { result, worker_id } => {
1642 free_worker(worker_id);
1643
1644 match result {
1645 Ok(WorkItemResult::Finished(compiled_module)) => {
1646 match compiled_module.kind {
1647 ModuleKind::Regular => {
1648 assert!(needs_link.is_empty());
1649 compiled_modules.push(compiled_module);
1650 }
1651 ModuleKind::Allocator => {
1652 assert!(compiled_allocator_module.is_none());
1653 compiled_allocator_module = Some(compiled_module);
1654 }
1655 }
1656 }
1657 Ok(WorkItemResult::NeedsLink(module)) => {
1658 assert!(compiled_modules.is_empty());
1659 needs_link.push(module);
1660 }
1661 Ok(WorkItemResult::NeedsFatLto(fat_lto_input)) => {
1662 assert!(!started_lto);
1663 assert!(needs_thin_lto.is_empty());
1664 needs_fat_lto.push(fat_lto_input);
1665 }
1666 Ok(WorkItemResult::NeedsThinLto(name, thin_buffer)) => {
1667 assert!(!started_lto);
1668 assert!(needs_fat_lto.is_empty());
1669 needs_thin_lto.push((name, thin_buffer));
1670 }
1671 Err(Some(WorkerFatalError)) => {
1672 codegen_state = Aborted;
1674 }
1675 Err(None) => {
1676 bug!("worker thread panicked");
1679 }
1680 }
1681 }
1682
1683 Message::AddImportOnlyModule { module_data, work_product } => {
1684 assert!(!started_lto);
1685 assert_eq!(codegen_state, Ongoing);
1686 assert_eq!(main_thread_state, MainThreadState::Codegenning);
1687 lto_import_only_modules.push((module_data, work_product));
1688 main_thread_state = MainThreadState::Idle;
1689 }
1690 }
1691 }
1692
1693 if codegen_state == Aborted {
1694 return Err(());
1695 }
1696
1697 let needs_link = mem::take(&mut needs_link);
1698 if !needs_link.is_empty() {
1699 assert!(compiled_modules.is_empty());
1700 let dcx = cgcx.create_dcx();
1701 let dcx = dcx.handle();
1702 let module = B::run_link(&cgcx, dcx, needs_link).map_err(|_| ())?;
1703 let module =
1704 B::codegen(&cgcx, dcx, module, cgcx.config(ModuleKind::Regular)).map_err(|_| ())?;
1705 compiled_modules.push(module);
1706 }
1707
1708 drop(llvm_start_time);
1710
1711 compiled_modules.sort_by(|a, b| a.name.cmp(&b.name));
1715
1716 Ok(CompiledModules {
1717 modules: compiled_modules,
1718 allocator_module: compiled_allocator_module,
1719 })
1720 })
1721 .expect("failed to spawn coordinator thread");
1722
1723 fn queue_full_enough(items_in_queue: usize, workers_running: usize) -> bool {
1726 let quarter_of_workers = workers_running - 3 * workers_running / 4;
1777 items_in_queue > 0 && items_in_queue >= quarter_of_workers
1778 }
1779}
1780
1781#[must_use]
1783pub(crate) struct WorkerFatalError;
1784
1785fn spawn_work<'a, B: ExtraBackendMethods>(
1786 cgcx: &'a CodegenContext<B>,
1787 llvm_start_time: &mut Option<VerboseTimingGuard<'a>>,
1788 worker_id: usize,
1789 work: WorkItem<B>,
1790) {
1791 if cgcx.config(work.module_kind()).time_module && llvm_start_time.is_none() {
1792 *llvm_start_time = Some(cgcx.prof.verbose_generic_activity("LLVM_passes"));
1793 }
1794
1795 let cgcx = cgcx.clone();
1796
1797 B::spawn_named_thread(cgcx.time_trace, work.short_description(), move || {
1798 struct Bomb<B: ExtraBackendMethods> {
1801 coordinator_send: Sender<Box<dyn Any + Send>>,
1802 result: Option<Result<WorkItemResult<B>, FatalError>>,
1803 worker_id: usize,
1804 }
1805 impl<B: ExtraBackendMethods> Drop for Bomb<B> {
1806 fn drop(&mut self) {
1807 let worker_id = self.worker_id;
1808 let msg = match self.result.take() {
1809 Some(Ok(result)) => Message::WorkItem::<B> { result: Ok(result), worker_id },
1810 Some(Err(FatalError)) => {
1811 Message::WorkItem::<B> { result: Err(Some(WorkerFatalError)), worker_id }
1812 }
1813 None => Message::WorkItem::<B> { result: Err(None), worker_id },
1814 };
1815 drop(self.coordinator_send.send(Box::new(msg)));
1816 }
1817 }
1818
1819 let mut bomb =
1820 Bomb::<B> { coordinator_send: cgcx.coordinator_send.clone(), result: None, worker_id };
1821
1822 bomb.result = {
1829 let module_config = cgcx.config(work.module_kind());
1830
1831 Some(match work {
1832 WorkItem::Optimize(m) => {
1833 let _timer =
1834 cgcx.prof.generic_activity_with_arg("codegen_module_optimize", &*m.name);
1835 execute_optimize_work_item(&cgcx, m, module_config)
1836 }
1837 WorkItem::CopyPostLtoArtifacts(m) => {
1838 let _timer = cgcx.prof.generic_activity_with_arg(
1839 "codegen_copy_artifacts_from_incr_cache",
1840 &*m.name,
1841 );
1842 Ok(execute_copy_from_cache_work_item(&cgcx, m, module_config))
1843 }
1844 WorkItem::LTO(m) => {
1845 let _timer =
1846 cgcx.prof.generic_activity_with_arg("codegen_module_perform_lto", m.name());
1847 execute_lto_work_item(&cgcx, m, module_config)
1848 }
1849 })
1850 };
1851 })
1852 .expect("failed to spawn work thread");
1853}
1854
1855enum SharedEmitterMessage {
1856 Diagnostic(Diagnostic),
1857 InlineAsmError(SpanData, String, Level, Option<(String, Vec<InnerSpan>)>),
1858 Fatal(String),
1859}
1860
1861#[derive(Clone)]
1862pub struct SharedEmitter {
1863 sender: Sender<SharedEmitterMessage>,
1864}
1865
1866pub struct SharedEmitterMain {
1867 receiver: Receiver<SharedEmitterMessage>,
1868}
1869
1870impl SharedEmitter {
1871 fn new() -> (SharedEmitter, SharedEmitterMain) {
1872 let (sender, receiver) = channel();
1873
1874 (SharedEmitter { sender }, SharedEmitterMain { receiver })
1875 }
1876
1877 pub fn inline_asm_error(
1878 &self,
1879 span: SpanData,
1880 msg: String,
1881 level: Level,
1882 source: Option<(String, Vec<InnerSpan>)>,
1883 ) {
1884 drop(self.sender.send(SharedEmitterMessage::InlineAsmError(span, msg, level, source)));
1885 }
1886
1887 fn fatal(&self, msg: &str) {
1888 drop(self.sender.send(SharedEmitterMessage::Fatal(msg.to_string())));
1889 }
1890}
1891
1892impl Emitter for SharedEmitter {
1893 fn emit_diagnostic(
1894 &mut self,
1895 mut diag: rustc_errors::DiagInner,
1896 _registry: &rustc_errors::registry::Registry,
1897 ) {
1898 assert_eq!(diag.span, MultiSpan::new());
1901 assert_eq!(diag.suggestions, Suggestions::Enabled(vec![]));
1902 assert_eq!(diag.sort_span, rustc_span::DUMMY_SP);
1903 assert_eq!(diag.is_lint, None);
1904 let args = mem::replace(&mut diag.args, DiagArgMap::default());
1907 drop(
1908 self.sender.send(SharedEmitterMessage::Diagnostic(Diagnostic {
1909 level: diag.level(),
1910 messages: diag.messages,
1911 code: diag.code,
1912 children: diag
1913 .children
1914 .into_iter()
1915 .map(|child| Subdiagnostic { level: child.level, messages: child.messages })
1916 .collect(),
1917 args,
1918 })),
1919 );
1920 }
1921
1922 fn source_map(&self) -> Option<&SourceMap> {
1923 None
1924 }
1925
1926 fn translator(&self) -> &Translator {
1927 panic!("shared emitter attempted to translate a diagnostic");
1928 }
1929}
1930
1931impl SharedEmitterMain {
1932 fn check(&self, sess: &Session, blocking: bool) {
1933 loop {
1934 let message = if blocking {
1935 match self.receiver.recv() {
1936 Ok(message) => Ok(message),
1937 Err(_) => Err(()),
1938 }
1939 } else {
1940 match self.receiver.try_recv() {
1941 Ok(message) => Ok(message),
1942 Err(_) => Err(()),
1943 }
1944 };
1945
1946 match message {
1947 Ok(SharedEmitterMessage::Diagnostic(diag)) => {
1948 let dcx = sess.dcx();
1951 let mut d =
1952 rustc_errors::DiagInner::new_with_messages(diag.level, diag.messages);
1953 d.code = diag.code; d.children = diag
1955 .children
1956 .into_iter()
1957 .map(|sub| rustc_errors::Subdiag {
1958 level: sub.level,
1959 messages: sub.messages,
1960 span: MultiSpan::new(),
1961 })
1962 .collect();
1963 d.args = diag.args;
1964 dcx.emit_diagnostic(d);
1965 sess.dcx().abort_if_errors();
1966 }
1967 Ok(SharedEmitterMessage::InlineAsmError(span, msg, level, source)) => {
1968 assert_matches!(level, Level::Error | Level::Warning | Level::Note);
1969 let mut err = Diag::<()>::new(sess.dcx(), level, msg);
1970 if !span.is_dummy() {
1971 err.span(span.span());
1972 }
1973
1974 if let Some((buffer, spans)) = source {
1976 let source = sess
1977 .source_map()
1978 .new_source_file(FileName::inline_asm_source_code(&buffer), buffer);
1979 let spans: Vec<_> = spans
1980 .iter()
1981 .map(|sp| {
1982 Span::with_root_ctxt(
1983 source.normalized_byte_pos(sp.start as u32),
1984 source.normalized_byte_pos(sp.end as u32),
1985 )
1986 })
1987 .collect();
1988 err.span_note(spans, "instantiated into assembly here");
1989 }
1990
1991 err.emit();
1992 }
1993 Ok(SharedEmitterMessage::Fatal(msg)) => {
1994 sess.dcx().fatal(msg);
1995 }
1996 Err(_) => {
1997 break;
1998 }
1999 }
2000 }
2001 }
2002}
2003
2004pub struct Coordinator<B: ExtraBackendMethods> {
2005 pub sender: Sender<Box<dyn Any + Send>>,
2006 future: Option<thread::JoinHandle<Result<CompiledModules, ()>>>,
2007 phantom: PhantomData<B>,
2009}
2010
2011impl<B: ExtraBackendMethods> Coordinator<B> {
2012 fn join(mut self) -> std::thread::Result<Result<CompiledModules, ()>> {
2013 self.future.take().unwrap().join()
2014 }
2015}
2016
2017impl<B: ExtraBackendMethods> Drop for Coordinator<B> {
2018 fn drop(&mut self) {
2019 if let Some(future) = self.future.take() {
2020 drop(self.sender.send(Box::new(Message::CodegenAborted::<B>)));
2023 drop(future.join());
2024 }
2025 }
2026}
2027
2028pub struct OngoingCodegen<B: ExtraBackendMethods> {
2029 pub backend: B,
2030 pub crate_info: CrateInfo,
2031 pub codegen_worker_receive: Receiver<CguMessage>,
2032 pub shared_emitter_main: SharedEmitterMain,
2033 pub output_filenames: Arc<OutputFilenames>,
2034 pub coordinator: Coordinator<B>,
2035}
2036
2037impl<B: ExtraBackendMethods> OngoingCodegen<B> {
2038 pub fn join(self, sess: &Session) -> (CodegenResults, FxIndexMap<WorkProductId, WorkProduct>) {
2039 self.shared_emitter_main.check(sess, true);
2040 let compiled_modules = sess.time("join_worker_thread", || match self.coordinator.join() {
2041 Ok(Ok(compiled_modules)) => compiled_modules,
2042 Ok(Err(())) => {
2043 sess.dcx().abort_if_errors();
2044 panic!("expected abort due to worker thread errors")
2045 }
2046 Err(_) => {
2047 bug!("panic during codegen/LLVM phase");
2048 }
2049 });
2050
2051 sess.dcx().abort_if_errors();
2052
2053 let work_products =
2054 copy_all_cgu_workproducts_to_incr_comp_cache_dir(sess, &compiled_modules);
2055 produce_final_output_artifacts(sess, &compiled_modules, &self.output_filenames);
2056
2057 if sess.codegen_units().as_usize() == 1 && sess.opts.unstable_opts.time_llvm_passes {
2060 self.backend.print_pass_timings()
2061 }
2062
2063 if sess.print_llvm_stats() {
2064 self.backend.print_statistics()
2065 }
2066
2067 (
2068 CodegenResults {
2069 crate_info: self.crate_info,
2070
2071 modules: compiled_modules.modules,
2072 allocator_module: compiled_modules.allocator_module,
2073 },
2074 work_products,
2075 )
2076 }
2077
2078 pub(crate) fn codegen_finished(&self, tcx: TyCtxt<'_>) {
2079 self.wait_for_signal_to_codegen_item();
2080 self.check_for_errors(tcx.sess);
2081 drop(self.coordinator.sender.send(Box::new(Message::CodegenComplete::<B>)));
2082 }
2083
2084 pub(crate) fn submit_autodiff_items(&self, items: Vec<AutoDiffItem>) {
2085 drop(self.coordinator.sender.send(Box::new(Message::<B>::AddAutoDiffItems(items))));
2086 }
2087
2088 pub(crate) fn check_for_errors(&self, sess: &Session) {
2089 self.shared_emitter_main.check(sess, false);
2090 }
2091
2092 pub(crate) fn wait_for_signal_to_codegen_item(&self) {
2093 match self.codegen_worker_receive.recv() {
2094 Ok(CguMessage) => {
2095 }
2097 Err(_) => {
2098 }
2101 }
2102 }
2103}
2104
2105pub(crate) fn submit_codegened_module_to_llvm<B: ExtraBackendMethods>(
2106 _backend: &B,
2107 tx_to_llvm_workers: &Sender<Box<dyn Any + Send>>,
2108 module: ModuleCodegen<B::Module>,
2109 cost: u64,
2110) {
2111 let llvm_work_item = WorkItem::Optimize(module);
2112 drop(tx_to_llvm_workers.send(Box::new(Message::CodegenDone::<B> { llvm_work_item, cost })));
2113}
2114
2115pub(crate) fn submit_post_lto_module_to_llvm<B: ExtraBackendMethods>(
2116 _backend: &B,
2117 tx_to_llvm_workers: &Sender<Box<dyn Any + Send>>,
2118 module: CachedModuleCodegen,
2119) {
2120 let llvm_work_item = WorkItem::CopyPostLtoArtifacts(module);
2121 drop(tx_to_llvm_workers.send(Box::new(Message::CodegenDone::<B> { llvm_work_item, cost: 0 })));
2122}
2123
2124pub(crate) fn submit_pre_lto_module_to_llvm<B: ExtraBackendMethods>(
2125 _backend: &B,
2126 tcx: TyCtxt<'_>,
2127 tx_to_llvm_workers: &Sender<Box<dyn Any + Send>>,
2128 module: CachedModuleCodegen,
2129) {
2130 let filename = pre_lto_bitcode_filename(&module.name);
2131 let bc_path = in_incr_comp_dir_sess(tcx.sess, &filename);
2132 let file = fs::File::open(&bc_path)
2133 .unwrap_or_else(|e| panic!("failed to open bitcode file `{}`: {}", bc_path.display(), e));
2134
2135 let mmap = unsafe {
2136 Mmap::map(file).unwrap_or_else(|e| {
2137 panic!("failed to mmap bitcode file `{}`: {}", bc_path.display(), e)
2138 })
2139 };
2140 drop(tx_to_llvm_workers.send(Box::new(Message::AddImportOnlyModule::<B> {
2142 module_data: SerializedModule::FromUncompressedFile(mmap),
2143 work_product: module.source,
2144 })));
2145}
2146
2147fn pre_lto_bitcode_filename(module_name: &str) -> String {
2148 format!("{module_name}.{PRE_LTO_BC_EXT}")
2149}
2150
2151fn msvc_imps_needed(tcx: TyCtxt<'_>) -> bool {
2152 assert!(
2155 !(tcx.sess.opts.cg.linker_plugin_lto.enabled()
2156 && tcx.sess.target.is_like_windows
2157 && tcx.sess.opts.cg.prefer_dynamic)
2158 );
2159
2160 let can_have_static_objects =
2164 tcx.sess.lto() == Lto::Thin || tcx.crate_types().contains(&CrateType::Rlib);
2165
2166 tcx.sess.target.is_like_windows &&
2167 can_have_static_objects &&
2168 !tcx.sess.opts.cg.linker_plugin_lto.enabled()
2172}