1use std::any::Any;
2use std::assert_matches::assert_matches;
3use std::marker::PhantomData;
4use std::path::{Path, PathBuf};
5use std::sync::Arc;
6use std::sync::mpsc::{Receiver, Sender, channel};
7use std::{fs, io, mem, str, thread};
8
9use rustc_abi::Size;
10use rustc_ast::attr;
11use rustc_ast::expand::autodiff_attrs::AutoDiffItem;
12use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
13use rustc_data_structures::jobserver::{self, Acquired};
14use rustc_data_structures::memmap::Mmap;
15use rustc_data_structures::profiling::{SelfProfilerRef, VerboseTimingGuard};
16use rustc_errors::emitter::Emitter;
17use rustc_errors::translation::Translator;
18use rustc_errors::{
19 Diag, DiagArgMap, DiagCtxt, DiagMessage, ErrCode, FatalError, Level, MultiSpan, Style,
20 Suggestions,
21};
22use rustc_fs_util::link_or_copy;
23use rustc_hir::def_id::{CrateNum, LOCAL_CRATE};
24use rustc_incremental::{
25 copy_cgu_workproduct_to_incr_comp_cache_dir, in_incr_comp_dir, in_incr_comp_dir_sess,
26};
27use rustc_metadata::fs::copy_to_stdout;
28use rustc_middle::bug;
29use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
30use rustc_middle::middle::exported_symbols::SymbolExportInfo;
31use rustc_middle::ty::TyCtxt;
32use rustc_session::Session;
33use rustc_session::config::{
34 self, CrateType, Lto, OutFileName, OutputFilenames, OutputType, Passes, SwitchWithOptPath,
35};
36use rustc_span::source_map::SourceMap;
37use rustc_span::{FileName, InnerSpan, Span, SpanData, sym};
38use rustc_target::spec::{MergeFunctions, SanitizerSet};
39use tracing::debug;
40
41use super::link::{self, ensure_removed};
42use super::lto::{self, SerializedModule};
43use super::symbol_export::symbol_name_for_instance_in_crate;
44use crate::errors::{AutodiffWithoutLto, ErrorCreatingRemarkDir};
45use crate::traits::*;
46use crate::{
47 CachedModuleCodegen, CodegenResults, CompiledModule, CrateInfo, ModuleCodegen, ModuleKind,
48 errors,
49};
50
51const PRE_LTO_BC_EXT: &str = "pre-lto.bc";
52
53#[derive(Clone, Copy, PartialEq)]
55pub enum EmitObj {
56 None,
58
59 Bitcode,
62
63 ObjectCode(BitcodeSection),
65}
66
67#[derive(Clone, Copy, PartialEq)]
69pub enum BitcodeSection {
70 None,
72
73 Full,
75}
76
77pub struct ModuleConfig {
79 pub passes: Vec<String>,
81 pub opt_level: Option<config::OptLevel>,
84
85 pub opt_size: Option<config::OptLevel>,
87
88 pub pgo_gen: SwitchWithOptPath,
89 pub pgo_use: Option<PathBuf>,
90 pub pgo_sample_use: Option<PathBuf>,
91 pub debug_info_for_profiling: bool,
92 pub instrument_coverage: bool,
93
94 pub sanitizer: SanitizerSet,
95 pub sanitizer_recover: SanitizerSet,
96 pub sanitizer_dataflow_abilist: Vec<String>,
97 pub sanitizer_memory_track_origins: usize,
98
99 pub emit_pre_lto_bc: bool,
101 pub emit_no_opt_bc: bool,
102 pub emit_bc: bool,
103 pub emit_ir: bool,
104 pub emit_asm: bool,
105 pub emit_obj: EmitObj,
106 pub emit_thin_lto: bool,
107 pub emit_thin_lto_summary: bool,
108 pub bc_cmdline: String,
109
110 pub verify_llvm_ir: bool,
113 pub lint_llvm_ir: bool,
114 pub no_prepopulate_passes: bool,
115 pub no_builtins: bool,
116 pub time_module: bool,
117 pub vectorize_loop: bool,
118 pub vectorize_slp: bool,
119 pub merge_functions: bool,
120 pub emit_lifetime_markers: bool,
121 pub llvm_plugins: Vec<String>,
122 pub autodiff: Vec<config::AutoDiff>,
123}
124
125impl ModuleConfig {
126 fn new(kind: ModuleKind, tcx: TyCtxt<'_>, no_builtins: bool) -> ModuleConfig {
127 macro_rules! if_regular {
130 ($regular: expr, $other: expr) => {
131 if let ModuleKind::Regular = kind { $regular } else { $other }
132 };
133 }
134
135 let sess = tcx.sess;
136 let opt_level_and_size = if_regular!(Some(sess.opts.optimize), None);
137
138 let save_temps = sess.opts.cg.save_temps;
139
140 let should_emit_obj = sess.opts.output_types.contains_key(&OutputType::Exe)
141 || match kind {
142 ModuleKind::Regular => sess.opts.output_types.contains_key(&OutputType::Object),
143 ModuleKind::Allocator => false,
144 };
145
146 let emit_obj = if !should_emit_obj {
147 EmitObj::None
148 } else if sess.target.obj_is_bitcode
149 || (sess.opts.cg.linker_plugin_lto.enabled() && !no_builtins)
150 {
151 EmitObj::Bitcode
166 } else if need_bitcode_in_object(tcx) {
167 EmitObj::ObjectCode(BitcodeSection::Full)
168 } else {
169 EmitObj::ObjectCode(BitcodeSection::None)
170 };
171
172 ModuleConfig {
173 passes: if_regular!(sess.opts.cg.passes.clone(), vec![]),
174
175 opt_level: opt_level_and_size,
176 opt_size: opt_level_and_size,
177
178 pgo_gen: if_regular!(
179 sess.opts.cg.profile_generate.clone(),
180 SwitchWithOptPath::Disabled
181 ),
182 pgo_use: if_regular!(sess.opts.cg.profile_use.clone(), None),
183 pgo_sample_use: if_regular!(sess.opts.unstable_opts.profile_sample_use.clone(), None),
184 debug_info_for_profiling: sess.opts.unstable_opts.debug_info_for_profiling,
185 instrument_coverage: if_regular!(sess.instrument_coverage(), false),
186
187 sanitizer: if_regular!(sess.opts.unstable_opts.sanitizer, SanitizerSet::empty()),
188 sanitizer_dataflow_abilist: if_regular!(
189 sess.opts.unstable_opts.sanitizer_dataflow_abilist.clone(),
190 Vec::new()
191 ),
192 sanitizer_recover: if_regular!(
193 sess.opts.unstable_opts.sanitizer_recover,
194 SanitizerSet::empty()
195 ),
196 sanitizer_memory_track_origins: if_regular!(
197 sess.opts.unstable_opts.sanitizer_memory_track_origins,
198 0
199 ),
200
201 emit_pre_lto_bc: if_regular!(
202 save_temps || need_pre_lto_bitcode_for_incr_comp(sess),
203 false
204 ),
205 emit_no_opt_bc: if_regular!(save_temps, false),
206 emit_bc: if_regular!(
207 save_temps || sess.opts.output_types.contains_key(&OutputType::Bitcode),
208 save_temps
209 ),
210 emit_ir: if_regular!(
211 sess.opts.output_types.contains_key(&OutputType::LlvmAssembly),
212 false
213 ),
214 emit_asm: if_regular!(
215 sess.opts.output_types.contains_key(&OutputType::Assembly),
216 false
217 ),
218 emit_obj,
219 emit_thin_lto: sess.opts.unstable_opts.emit_thin_lto,
220 emit_thin_lto_summary: if_regular!(
221 sess.opts.output_types.contains_key(&OutputType::ThinLinkBitcode),
222 false
223 ),
224 bc_cmdline: sess.target.bitcode_llvm_cmdline.to_string(),
225
226 verify_llvm_ir: sess.verify_llvm_ir(),
227 lint_llvm_ir: sess.opts.unstable_opts.lint_llvm_ir,
228 no_prepopulate_passes: sess.opts.cg.no_prepopulate_passes,
229 no_builtins: no_builtins || sess.target.no_builtins,
230
231 time_module: if_regular!(true, false),
234
235 vectorize_loop: !sess.opts.cg.no_vectorize_loops
238 && (sess.opts.optimize == config::OptLevel::More
239 || sess.opts.optimize == config::OptLevel::Aggressive),
240 vectorize_slp: !sess.opts.cg.no_vectorize_slp
241 && sess.opts.optimize == config::OptLevel::Aggressive,
242
243 merge_functions: match sess
253 .opts
254 .unstable_opts
255 .merge_functions
256 .unwrap_or(sess.target.merge_functions)
257 {
258 MergeFunctions::Disabled => false,
259 MergeFunctions::Trampolines | MergeFunctions::Aliases => {
260 use config::OptLevel::*;
261 match sess.opts.optimize {
262 Aggressive | More | SizeMin | Size => true,
263 Less | No => false,
264 }
265 }
266 },
267
268 emit_lifetime_markers: sess.emit_lifetime_markers(),
269 llvm_plugins: if_regular!(sess.opts.unstable_opts.llvm_plugins.clone(), vec![]),
270 autodiff: if_regular!(sess.opts.unstable_opts.autodiff.clone(), vec![]),
271 }
272 }
273
274 pub fn bitcode_needed(&self) -> bool {
275 self.emit_bc
276 || self.emit_thin_lto_summary
277 || self.emit_obj == EmitObj::Bitcode
278 || self.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full)
279 }
280
281 pub fn embed_bitcode(&self) -> bool {
282 self.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full)
283 }
284}
285
286pub struct TargetMachineFactoryConfig {
288 pub split_dwarf_file: Option<PathBuf>,
292
293 pub output_obj_file: Option<PathBuf>,
296}
297
298impl TargetMachineFactoryConfig {
299 pub fn new(
300 cgcx: &CodegenContext<impl WriteBackendMethods>,
301 module_name: &str,
302 ) -> TargetMachineFactoryConfig {
303 let split_dwarf_file = if cgcx.target_can_use_split_dwarf {
304 cgcx.output_filenames.split_dwarf_path(
305 cgcx.split_debuginfo,
306 cgcx.split_dwarf_kind,
307 module_name,
308 cgcx.invocation_temp.as_deref(),
309 )
310 } else {
311 None
312 };
313
314 let output_obj_file = Some(cgcx.output_filenames.temp_path_for_cgu(
315 OutputType::Object,
316 module_name,
317 cgcx.invocation_temp.as_deref(),
318 ));
319 TargetMachineFactoryConfig { split_dwarf_file, output_obj_file }
320 }
321}
322
323pub type TargetMachineFactoryFn<B> = Arc<
324 dyn Fn(
325 TargetMachineFactoryConfig,
326 ) -> Result<
327 <B as WriteBackendMethods>::TargetMachine,
328 <B as WriteBackendMethods>::TargetMachineError,
329 > + Send
330 + Sync,
331>;
332
333type ExportedSymbols = FxHashMap<CrateNum, Arc<Vec<(String, SymbolExportInfo)>>>;
334
335#[derive(Clone)]
337pub struct CodegenContext<B: WriteBackendMethods> {
338 pub prof: SelfProfilerRef,
340 pub lto: Lto,
341 pub save_temps: bool,
342 pub fewer_names: bool,
343 pub time_trace: bool,
344 pub exported_symbols: Option<Arc<ExportedSymbols>>,
345 pub opts: Arc<config::Options>,
346 pub crate_types: Vec<CrateType>,
347 pub each_linked_rlib_for_lto: Vec<(CrateNum, PathBuf)>,
348 pub output_filenames: Arc<OutputFilenames>,
349 pub invocation_temp: Option<String>,
350 pub regular_module_config: Arc<ModuleConfig>,
351 pub allocator_module_config: Arc<ModuleConfig>,
352 pub tm_factory: TargetMachineFactoryFn<B>,
353 pub msvc_imps_needed: bool,
354 pub is_pe_coff: bool,
355 pub target_can_use_split_dwarf: bool,
356 pub target_arch: String,
357 pub target_is_like_darwin: bool,
358 pub target_is_like_aix: bool,
359 pub split_debuginfo: rustc_target::spec::SplitDebuginfo,
360 pub split_dwarf_kind: rustc_session::config::SplitDwarfKind,
361 pub pointer_size: Size,
362
363 pub expanded_args: Vec<String>,
368
369 pub diag_emitter: SharedEmitter,
371 pub remark: Passes,
373 pub remark_dir: Option<PathBuf>,
376 pub incr_comp_session_dir: Option<PathBuf>,
379 pub coordinator_send: Sender<Box<dyn Any + Send>>,
381 pub parallel: bool,
385}
386
387impl<B: WriteBackendMethods> CodegenContext<B> {
388 pub fn create_dcx(&self) -> DiagCtxt {
389 DiagCtxt::new(Box::new(self.diag_emitter.clone()))
390 }
391
392 pub fn config(&self, kind: ModuleKind) -> &ModuleConfig {
393 match kind {
394 ModuleKind::Regular => &self.regular_module_config,
395 ModuleKind::Allocator => &self.allocator_module_config,
396 }
397 }
398}
399
400fn generate_lto_work<B: ExtraBackendMethods>(
401 cgcx: &CodegenContext<B>,
402 autodiff: Vec<AutoDiffItem>,
403 needs_fat_lto: Vec<FatLtoInput<B>>,
404 needs_thin_lto: Vec<(String, B::ThinBuffer)>,
405 import_only_modules: Vec<(SerializedModule<B::ModuleBuffer>, WorkProduct)>,
406) -> Vec<(WorkItem<B>, u64)> {
407 let _prof_timer = cgcx.prof.generic_activity("codegen_generate_lto_work");
408
409 if !needs_fat_lto.is_empty() {
410 assert!(needs_thin_lto.is_empty());
411 let mut module =
412 B::run_fat_lto(cgcx, needs_fat_lto, import_only_modules).unwrap_or_else(|e| e.raise());
413 if cgcx.lto == Lto::Fat && !autodiff.is_empty() {
414 let config = cgcx.config(ModuleKind::Regular);
415 module = module.autodiff(cgcx, autodiff, config).unwrap_or_else(|e| e.raise());
416 }
417 vec![(WorkItem::LTO(module), 0)]
419 } else {
420 if !autodiff.is_empty() {
421 let dcx = cgcx.create_dcx();
422 dcx.handle().emit_fatal(AutodiffWithoutLto {});
423 }
424 assert!(needs_fat_lto.is_empty());
425 let (lto_modules, copy_jobs) = B::run_thin_lto(cgcx, needs_thin_lto, import_only_modules)
426 .unwrap_or_else(|e| e.raise());
427 lto_modules
428 .into_iter()
429 .map(|module| {
430 let cost = module.cost();
431 (WorkItem::LTO(module), cost)
432 })
433 .chain(copy_jobs.into_iter().map(|wp| {
434 (
435 WorkItem::CopyPostLtoArtifacts(CachedModuleCodegen {
436 name: wp.cgu_name.clone(),
437 source: wp,
438 }),
439 0, )
441 }))
442 .collect()
443 }
444}
445
446struct CompiledModules {
447 modules: Vec<CompiledModule>,
448 allocator_module: Option<CompiledModule>,
449}
450
451fn need_bitcode_in_object(tcx: TyCtxt<'_>) -> bool {
452 let sess = tcx.sess;
453 sess.opts.cg.embed_bitcode
454 && tcx.crate_types().contains(&CrateType::Rlib)
455 && sess.opts.output_types.contains_key(&OutputType::Exe)
456}
457
458fn need_pre_lto_bitcode_for_incr_comp(sess: &Session) -> bool {
459 if sess.opts.incremental.is_none() {
460 return false;
461 }
462
463 match sess.lto() {
464 Lto::No => false,
465 Lto::Fat | Lto::Thin | Lto::ThinLocal => true,
466 }
467}
468
469pub(crate) fn start_async_codegen<B: ExtraBackendMethods>(
470 backend: B,
471 tcx: TyCtxt<'_>,
472 target_cpu: String,
473) -> OngoingCodegen<B> {
474 let (coordinator_send, coordinator_receive) = channel();
475
476 let crate_attrs = tcx.hir_attrs(rustc_hir::CRATE_HIR_ID);
477 let no_builtins = attr::contains_name(crate_attrs, sym::no_builtins);
478
479 let crate_info = CrateInfo::new(tcx, target_cpu);
480
481 let regular_config = ModuleConfig::new(ModuleKind::Regular, tcx, no_builtins);
482 let allocator_config = ModuleConfig::new(ModuleKind::Allocator, tcx, no_builtins);
483
484 let (shared_emitter, shared_emitter_main) = SharedEmitter::new();
485 let (codegen_worker_send, codegen_worker_receive) = channel();
486
487 let coordinator_thread = start_executing_work(
488 backend.clone(),
489 tcx,
490 &crate_info,
491 shared_emitter,
492 codegen_worker_send,
493 coordinator_receive,
494 Arc::new(regular_config),
495 Arc::new(allocator_config),
496 coordinator_send.clone(),
497 );
498
499 OngoingCodegen {
500 backend,
501 crate_info,
502
503 codegen_worker_receive,
504 shared_emitter_main,
505 coordinator: Coordinator {
506 sender: coordinator_send,
507 future: Some(coordinator_thread),
508 phantom: PhantomData,
509 },
510 output_filenames: Arc::clone(tcx.output_filenames(())),
511 }
512}
513
514fn copy_all_cgu_workproducts_to_incr_comp_cache_dir(
515 sess: &Session,
516 compiled_modules: &CompiledModules,
517) -> FxIndexMap<WorkProductId, WorkProduct> {
518 let mut work_products = FxIndexMap::default();
519
520 if sess.opts.incremental.is_none() {
521 return work_products;
522 }
523
524 let _timer = sess.timer("copy_all_cgu_workproducts_to_incr_comp_cache_dir");
525
526 for module in compiled_modules.modules.iter().filter(|m| m.kind == ModuleKind::Regular) {
527 let mut files = Vec::new();
528 if let Some(object_file_path) = &module.object {
529 files.push((OutputType::Object.extension(), object_file_path.as_path()));
530 }
531 if let Some(dwarf_object_file_path) = &module.dwarf_object {
532 files.push(("dwo", dwarf_object_file_path.as_path()));
533 }
534 if let Some(path) = &module.assembly {
535 files.push((OutputType::Assembly.extension(), path.as_path()));
536 }
537 if let Some(path) = &module.llvm_ir {
538 files.push((OutputType::LlvmAssembly.extension(), path.as_path()));
539 }
540 if let Some(path) = &module.bytecode {
541 files.push((OutputType::Bitcode.extension(), path.as_path()));
542 }
543 if let Some((id, product)) = copy_cgu_workproduct_to_incr_comp_cache_dir(
544 sess,
545 &module.name,
546 files.as_slice(),
547 &module.links_from_incr_cache,
548 ) {
549 work_products.insert(id, product);
550 }
551 }
552
553 work_products
554}
555
556fn produce_final_output_artifacts(
557 sess: &Session,
558 compiled_modules: &CompiledModules,
559 crate_output: &OutputFilenames,
560) {
561 let mut user_wants_bitcode = false;
562 let mut user_wants_objects = false;
563
564 let copy_gracefully = |from: &Path, to: &OutFileName| match to {
566 OutFileName::Stdout if let Err(e) = copy_to_stdout(from) => {
567 sess.dcx().emit_err(errors::CopyPath::new(from, to.as_path(), e));
568 }
569 OutFileName::Real(path) if let Err(e) = fs::copy(from, path) => {
570 sess.dcx().emit_err(errors::CopyPath::new(from, path, e));
571 }
572 _ => {}
573 };
574
575 let copy_if_one_unit = |output_type: OutputType, keep_numbered: bool| {
576 if let [module] = &compiled_modules.modules[..] {
577 let path = crate_output.temp_path_for_cgu(
580 output_type,
581 &module.name,
582 sess.invocation_temp.as_deref(),
583 );
584 let output = crate_output.path(output_type);
585 if !output_type.is_text_output() && output.is_tty() {
586 sess.dcx()
587 .emit_err(errors::BinaryOutputToTty { shorthand: output_type.shorthand() });
588 } else {
589 copy_gracefully(&path, &output);
590 }
591 if !sess.opts.cg.save_temps && !keep_numbered {
592 ensure_removed(sess.dcx(), &path);
594 }
595 } else {
596 if crate_output.outputs.contains_explicit_name(&output_type) {
597 sess.dcx()
600 .emit_warn(errors::IgnoringEmitPath { extension: output_type.extension() });
601 } else if crate_output.single_output_file.is_some() {
602 sess.dcx().emit_warn(errors::IgnoringOutput { extension: output_type.extension() });
605 } else {
606 }
610 }
611 };
612
613 for output_type in crate_output.outputs.keys() {
617 match *output_type {
618 OutputType::Bitcode => {
619 user_wants_bitcode = true;
620 copy_if_one_unit(OutputType::Bitcode, true);
624 }
625 OutputType::ThinLinkBitcode => {
626 copy_if_one_unit(OutputType::ThinLinkBitcode, false);
627 }
628 OutputType::LlvmAssembly => {
629 copy_if_one_unit(OutputType::LlvmAssembly, false);
630 }
631 OutputType::Assembly => {
632 copy_if_one_unit(OutputType::Assembly, false);
633 }
634 OutputType::Object => {
635 user_wants_objects = true;
636 copy_if_one_unit(OutputType::Object, true);
637 }
638 OutputType::Mir | OutputType::Metadata | OutputType::Exe | OutputType::DepInfo => {}
639 }
640 }
641
642 if !sess.opts.cg.save_temps {
655 let needs_crate_object = crate_output.outputs.contains_key(&OutputType::Exe);
671
672 let keep_numbered_bitcode = user_wants_bitcode && sess.codegen_units().as_usize() > 1;
673
674 let keep_numbered_objects =
675 needs_crate_object || (user_wants_objects && sess.codegen_units().as_usize() > 1);
676
677 for module in compiled_modules.modules.iter() {
678 if !keep_numbered_objects {
679 if let Some(ref path) = module.object {
680 ensure_removed(sess.dcx(), path);
681 }
682
683 if let Some(ref path) = module.dwarf_object {
684 ensure_removed(sess.dcx(), path);
685 }
686 }
687
688 if let Some(ref path) = module.bytecode {
689 if !keep_numbered_bitcode {
690 ensure_removed(sess.dcx(), path);
691 }
692 }
693 }
694
695 if !user_wants_bitcode
696 && let Some(ref allocator_module) = compiled_modules.allocator_module
697 && let Some(ref path) = allocator_module.bytecode
698 {
699 ensure_removed(sess.dcx(), path);
700 }
701 }
702
703 if sess.opts.json_artifact_notifications {
704 if let [module] = &compiled_modules.modules[..] {
705 module.for_each_output(|_path, ty| {
706 if sess.opts.output_types.contains_key(&ty) {
707 let descr = ty.shorthand();
708 let path = crate_output.path(ty);
711 sess.dcx().emit_artifact_notification(path.as_path(), descr);
712 }
713 });
714 } else {
715 for module in &compiled_modules.modules {
716 module.for_each_output(|path, ty| {
717 if sess.opts.output_types.contains_key(&ty) {
718 let descr = ty.shorthand();
719 sess.dcx().emit_artifact_notification(&path, descr);
720 }
721 });
722 }
723 }
724 }
725
726 }
732
733pub(crate) enum WorkItem<B: WriteBackendMethods> {
734 Optimize(ModuleCodegen<B::Module>),
736 CopyPostLtoArtifacts(CachedModuleCodegen),
739 LTO(lto::LtoModuleCodegen<B>),
741}
742
743impl<B: WriteBackendMethods> WorkItem<B> {
744 fn module_kind(&self) -> ModuleKind {
745 match *self {
746 WorkItem::Optimize(ref m) => m.kind,
747 WorkItem::CopyPostLtoArtifacts(_) | WorkItem::LTO(_) => ModuleKind::Regular,
748 }
749 }
750
751 fn short_description(&self) -> String {
753 #[cfg(not(windows))]
757 fn desc(short: &str, _long: &str, name: &str) -> String {
758 assert_eq!(short.len(), 3);
778 let name = if let Some(index) = name.find("-cgu.") {
779 &name[index + 1..] } else {
781 name
782 };
783 format!("{short} {name}")
784 }
785
786 #[cfg(windows)]
788 fn desc(_short: &str, long: &str, name: &str) -> String {
789 format!("{long} {name}")
790 }
791
792 match self {
793 WorkItem::Optimize(m) => desc("opt", "optimize module", &m.name),
794 WorkItem::CopyPostLtoArtifacts(m) => desc("cpy", "copy LTO artifacts for", &m.name),
795 WorkItem::LTO(m) => desc("lto", "LTO module", m.name()),
796 }
797 }
798}
799
800pub(crate) enum WorkItemResult<B: WriteBackendMethods> {
802 Finished(CompiledModule),
804
805 NeedsLink(ModuleCodegen<B::Module>),
808
809 NeedsFatLto(FatLtoInput<B>),
812
813 NeedsThinLto(String, B::ThinBuffer),
816}
817
818pub enum FatLtoInput<B: WriteBackendMethods> {
819 Serialized { name: String, buffer: B::ModuleBuffer },
820 InMemory(ModuleCodegen<B::Module>),
821}
822
823pub(crate) enum ComputedLtoType {
825 No,
826 Thin,
827 Fat,
828}
829
830pub(crate) fn compute_per_cgu_lto_type(
831 sess_lto: &Lto,
832 opts: &config::Options,
833 sess_crate_types: &[CrateType],
834 module_kind: ModuleKind,
835) -> ComputedLtoType {
836 let linker_does_lto = opts.cg.linker_plugin_lto.enabled();
840
841 let is_allocator = module_kind == ModuleKind::Allocator;
846
847 let is_rlib = matches!(sess_crate_types, [CrateType::Rlib]);
856
857 match sess_lto {
858 Lto::ThinLocal if !linker_does_lto && !is_allocator => ComputedLtoType::Thin,
859 Lto::Thin if !linker_does_lto && !is_rlib => ComputedLtoType::Thin,
860 Lto::Fat if !is_rlib => ComputedLtoType::Fat,
861 _ => ComputedLtoType::No,
862 }
863}
864
865fn execute_optimize_work_item<B: ExtraBackendMethods>(
866 cgcx: &CodegenContext<B>,
867 mut module: ModuleCodegen<B::Module>,
868 module_config: &ModuleConfig,
869) -> Result<WorkItemResult<B>, FatalError> {
870 let dcx = cgcx.create_dcx();
871 let dcx = dcx.handle();
872
873 B::optimize(cgcx, dcx, &mut module, module_config)?;
874
875 let lto_type = compute_per_cgu_lto_type(&cgcx.lto, &cgcx.opts, &cgcx.crate_types, module.kind);
881
882 let bitcode = if cgcx.config(module.kind).emit_pre_lto_bc {
885 let filename = pre_lto_bitcode_filename(&module.name);
886 cgcx.incr_comp_session_dir.as_ref().map(|path| path.join(&filename))
887 } else {
888 None
889 };
890
891 match lto_type {
892 ComputedLtoType::No => finish_intra_module_work(cgcx, module, module_config),
893 ComputedLtoType::Thin => {
894 let (name, thin_buffer) = B::prepare_thin(module, false);
895 if let Some(path) = bitcode {
896 fs::write(&path, thin_buffer.data()).unwrap_or_else(|e| {
897 panic!("Error writing pre-lto-bitcode file `{}`: {}", path.display(), e);
898 });
899 }
900 Ok(WorkItemResult::NeedsThinLto(name, thin_buffer))
901 }
902 ComputedLtoType::Fat => match bitcode {
903 Some(path) => {
904 let (name, buffer) = B::serialize_module(module);
905 fs::write(&path, buffer.data()).unwrap_or_else(|e| {
906 panic!("Error writing pre-lto-bitcode file `{}`: {}", path.display(), e);
907 });
908 Ok(WorkItemResult::NeedsFatLto(FatLtoInput::Serialized { name, buffer }))
909 }
910 None => Ok(WorkItemResult::NeedsFatLto(FatLtoInput::InMemory(module))),
911 },
912 }
913}
914
915fn execute_copy_from_cache_work_item<B: ExtraBackendMethods>(
916 cgcx: &CodegenContext<B>,
917 module: CachedModuleCodegen,
918 module_config: &ModuleConfig,
919) -> WorkItemResult<B> {
920 let incr_comp_session_dir = cgcx.incr_comp_session_dir.as_ref().unwrap();
921
922 let mut links_from_incr_cache = Vec::new();
923
924 let mut load_from_incr_comp_dir = |output_path: PathBuf, saved_path: &str| {
925 let source_file = in_incr_comp_dir(incr_comp_session_dir, saved_path);
926 debug!(
927 "copying preexisting module `{}` from {:?} to {}",
928 module.name,
929 source_file,
930 output_path.display()
931 );
932 match link_or_copy(&source_file, &output_path) {
933 Ok(_) => {
934 links_from_incr_cache.push(source_file);
935 Some(output_path)
936 }
937 Err(error) => {
938 cgcx.create_dcx().handle().emit_err(errors::CopyPathBuf {
939 source_file,
940 output_path,
941 error,
942 });
943 None
944 }
945 }
946 };
947
948 let dwarf_object =
949 module.source.saved_files.get("dwo").as_ref().and_then(|saved_dwarf_object_file| {
950 let dwarf_obj_out = cgcx
951 .output_filenames
952 .split_dwarf_path(
953 cgcx.split_debuginfo,
954 cgcx.split_dwarf_kind,
955 &module.name,
956 cgcx.invocation_temp.as_deref(),
957 )
958 .expect(
959 "saved dwarf object in work product but `split_dwarf_path` returned `None`",
960 );
961 load_from_incr_comp_dir(dwarf_obj_out, saved_dwarf_object_file)
962 });
963
964 let mut load_from_incr_cache = |perform, output_type: OutputType| {
965 if perform {
966 let saved_file = module.source.saved_files.get(output_type.extension())?;
967 let output_path = cgcx.output_filenames.temp_path_for_cgu(
968 output_type,
969 &module.name,
970 cgcx.invocation_temp.as_deref(),
971 );
972 load_from_incr_comp_dir(output_path, &saved_file)
973 } else {
974 None
975 }
976 };
977
978 let should_emit_obj = module_config.emit_obj != EmitObj::None;
979 let assembly = load_from_incr_cache(module_config.emit_asm, OutputType::Assembly);
980 let llvm_ir = load_from_incr_cache(module_config.emit_ir, OutputType::LlvmAssembly);
981 let bytecode = load_from_incr_cache(module_config.emit_bc, OutputType::Bitcode);
982 let object = load_from_incr_cache(should_emit_obj, OutputType::Object);
983 if should_emit_obj && object.is_none() {
984 cgcx.create_dcx().handle().emit_fatal(errors::NoSavedObjectFile { cgu_name: &module.name })
985 }
986
987 WorkItemResult::Finished(CompiledModule {
988 links_from_incr_cache,
989 name: module.name,
990 kind: ModuleKind::Regular,
991 object,
992 dwarf_object,
993 bytecode,
994 assembly,
995 llvm_ir,
996 })
997}
998
999fn execute_lto_work_item<B: ExtraBackendMethods>(
1000 cgcx: &CodegenContext<B>,
1001 module: lto::LtoModuleCodegen<B>,
1002 module_config: &ModuleConfig,
1003) -> Result<WorkItemResult<B>, FatalError> {
1004 let module = module.optimize(cgcx)?;
1005 finish_intra_module_work(cgcx, module, module_config)
1006}
1007
1008fn finish_intra_module_work<B: ExtraBackendMethods>(
1009 cgcx: &CodegenContext<B>,
1010 module: ModuleCodegen<B::Module>,
1011 module_config: &ModuleConfig,
1012) -> Result<WorkItemResult<B>, FatalError> {
1013 let dcx = cgcx.create_dcx();
1014 let dcx = dcx.handle();
1015
1016 if !cgcx.opts.unstable_opts.combine_cgu || module.kind == ModuleKind::Allocator {
1017 let module = B::codegen(cgcx, dcx, module, module_config)?;
1018 Ok(WorkItemResult::Finished(module))
1019 } else {
1020 Ok(WorkItemResult::NeedsLink(module))
1021 }
1022}
1023
1024pub(crate) enum Message<B: WriteBackendMethods> {
1026 Token(io::Result<Acquired>),
1029
1030 WorkItem { result: Result<WorkItemResult<B>, Option<WorkerFatalError>>, worker_id: usize },
1033
1034 AddAutoDiffItems(Vec<AutoDiffItem>),
1036
1037 CodegenDone { llvm_work_item: WorkItem<B>, cost: u64 },
1041
1042 AddImportOnlyModule {
1045 module_data: SerializedModule<B::ModuleBuffer>,
1046 work_product: WorkProduct,
1047 },
1048
1049 CodegenComplete,
1052
1053 CodegenAborted,
1056}
1057
1058pub struct CguMessage;
1061
1062struct Diagnostic {
1072 level: Level,
1073 messages: Vec<(DiagMessage, Style)>,
1074 code: Option<ErrCode>,
1075 children: Vec<Subdiagnostic>,
1076 args: DiagArgMap,
1077}
1078
1079pub(crate) struct Subdiagnostic {
1083 level: Level,
1084 messages: Vec<(DiagMessage, Style)>,
1085}
1086
1087#[derive(PartialEq, Clone, Copy, Debug)]
1088enum MainThreadState {
1089 Idle,
1091
1092 Codegenning,
1094
1095 Lending,
1097}
1098
1099fn start_executing_work<B: ExtraBackendMethods>(
1100 backend: B,
1101 tcx: TyCtxt<'_>,
1102 crate_info: &CrateInfo,
1103 shared_emitter: SharedEmitter,
1104 codegen_worker_send: Sender<CguMessage>,
1105 coordinator_receive: Receiver<Box<dyn Any + Send>>,
1106 regular_config: Arc<ModuleConfig>,
1107 allocator_config: Arc<ModuleConfig>,
1108 tx_to_llvm_workers: Sender<Box<dyn Any + Send>>,
1109) -> thread::JoinHandle<Result<CompiledModules, ()>> {
1110 let coordinator_send = tx_to_llvm_workers;
1111 let sess = tcx.sess;
1112
1113 let mut each_linked_rlib_for_lto = Vec::new();
1114 drop(link::each_linked_rlib(crate_info, None, &mut |cnum, path| {
1115 if link::ignored_for_lto(sess, crate_info, cnum) {
1116 return;
1117 }
1118 each_linked_rlib_for_lto.push((cnum, path.to_path_buf()));
1119 }));
1120
1121 let exported_symbols = {
1123 let mut exported_symbols = FxHashMap::default();
1124
1125 let copy_symbols = |cnum| {
1126 let symbols = tcx
1127 .exported_non_generic_symbols(cnum)
1128 .iter()
1129 .chain(tcx.exported_generic_symbols(cnum))
1130 .map(|&(s, lvl)| (symbol_name_for_instance_in_crate(tcx, s, cnum), lvl))
1131 .collect();
1132 Arc::new(symbols)
1133 };
1134
1135 match sess.lto() {
1136 Lto::No => None,
1137 Lto::ThinLocal => {
1138 exported_symbols.insert(LOCAL_CRATE, copy_symbols(LOCAL_CRATE));
1139 Some(Arc::new(exported_symbols))
1140 }
1141 Lto::Fat | Lto::Thin => {
1142 exported_symbols.insert(LOCAL_CRATE, copy_symbols(LOCAL_CRATE));
1143 for &(cnum, ref _path) in &each_linked_rlib_for_lto {
1144 exported_symbols.insert(cnum, copy_symbols(cnum));
1145 }
1146 Some(Arc::new(exported_symbols))
1147 }
1148 }
1149 };
1150
1151 let coordinator_send2 = coordinator_send.clone();
1157 let helper = jobserver::client()
1158 .into_helper_thread(move |token| {
1159 drop(coordinator_send2.send(Box::new(Message::Token::<B>(token))));
1160 })
1161 .expect("failed to spawn helper thread");
1162
1163 let ol =
1164 if tcx.sess.opts.unstable_opts.no_codegen || !tcx.sess.opts.output_types.should_codegen() {
1165 config::OptLevel::No
1167 } else {
1168 tcx.backend_optimization_level(())
1169 };
1170 let backend_features = tcx.global_backend_features(());
1171
1172 let remark_dir = if let Some(ref dir) = sess.opts.unstable_opts.remark_dir {
1173 let result = fs::create_dir_all(dir).and_then(|_| dir.canonicalize());
1174 match result {
1175 Ok(dir) => Some(dir),
1176 Err(error) => sess.dcx().emit_fatal(ErrorCreatingRemarkDir { error }),
1177 }
1178 } else {
1179 None
1180 };
1181
1182 let cgcx = CodegenContext::<B> {
1183 crate_types: tcx.crate_types().to_vec(),
1184 each_linked_rlib_for_lto,
1185 lto: sess.lto(),
1186 fewer_names: sess.fewer_names(),
1187 save_temps: sess.opts.cg.save_temps,
1188 time_trace: sess.opts.unstable_opts.llvm_time_trace,
1189 opts: Arc::new(sess.opts.clone()),
1190 prof: sess.prof.clone(),
1191 exported_symbols,
1192 remark: sess.opts.cg.remark.clone(),
1193 remark_dir,
1194 incr_comp_session_dir: sess.incr_comp_session_dir_opt().map(|r| r.clone()),
1195 coordinator_send,
1196 expanded_args: tcx.sess.expanded_args.clone(),
1197 diag_emitter: shared_emitter.clone(),
1198 output_filenames: Arc::clone(tcx.output_filenames(())),
1199 regular_module_config: regular_config,
1200 allocator_module_config: allocator_config,
1201 tm_factory: backend.target_machine_factory(tcx.sess, ol, backend_features),
1202 msvc_imps_needed: msvc_imps_needed(tcx),
1203 is_pe_coff: tcx.sess.target.is_like_windows,
1204 target_can_use_split_dwarf: tcx.sess.target_can_use_split_dwarf(),
1205 target_arch: tcx.sess.target.arch.to_string(),
1206 target_is_like_darwin: tcx.sess.target.is_like_darwin,
1207 target_is_like_aix: tcx.sess.target.is_like_aix,
1208 split_debuginfo: tcx.sess.split_debuginfo(),
1209 split_dwarf_kind: tcx.sess.opts.unstable_opts.split_dwarf_kind,
1210 parallel: backend.supports_parallel() && !sess.opts.unstable_opts.no_parallel_backend,
1211 pointer_size: tcx.data_layout.pointer_size(),
1212 invocation_temp: sess.invocation_temp.clone(),
1213 };
1214
1215 return B::spawn_named_thread(cgcx.time_trace, "coordinator".to_string(), move || {
1351 let mut worker_id_counter = 0;
1352 let mut free_worker_ids = Vec::new();
1353 let mut get_worker_id = |free_worker_ids: &mut Vec<usize>| {
1354 if let Some(id) = free_worker_ids.pop() {
1355 id
1356 } else {
1357 let id = worker_id_counter;
1358 worker_id_counter += 1;
1359 id
1360 }
1361 };
1362
1363 let mut autodiff_items = Vec::new();
1366 let mut compiled_modules = vec![];
1367 let mut compiled_allocator_module = None;
1368 let mut needs_link = Vec::new();
1369 let mut needs_fat_lto = Vec::new();
1370 let mut needs_thin_lto = Vec::new();
1371 let mut lto_import_only_modules = Vec::new();
1372 let mut started_lto = false;
1373
1374 #[derive(Debug, PartialEq)]
1379 enum CodegenState {
1380 Ongoing,
1381 Completed,
1382 Aborted,
1383 }
1384 use CodegenState::*;
1385 let mut codegen_state = Ongoing;
1386
1387 let mut work_items = Vec::<(WorkItem<B>, u64)>::new();
1389
1390 let mut tokens = Vec::new();
1393
1394 let mut main_thread_state = MainThreadState::Idle;
1395
1396 let mut running_with_own_token = 0;
1399
1400 let running_with_any_token = |main_thread_state, running_with_own_token| {
1403 running_with_own_token
1404 + if main_thread_state == MainThreadState::Lending { 1 } else { 0 }
1405 };
1406
1407 let mut llvm_start_time: Option<VerboseTimingGuard<'_>> = None;
1408
1409 loop {
1415 if codegen_state == Ongoing {
1419 if main_thread_state == MainThreadState::Idle {
1420 let extra_tokens = tokens.len().checked_sub(running_with_own_token).unwrap();
1428 let additional_running = std::cmp::min(extra_tokens, work_items.len());
1429 let anticipated_running = running_with_own_token + additional_running + 1;
1430
1431 if !queue_full_enough(work_items.len(), anticipated_running) {
1432 if codegen_worker_send.send(CguMessage).is_err() {
1434 panic!("Could not send CguMessage to main thread")
1435 }
1436 main_thread_state = MainThreadState::Codegenning;
1437 } else {
1438 let (item, _) =
1442 work_items.pop().expect("queue empty - queue_full_enough() broken?");
1443 main_thread_state = MainThreadState::Lending;
1444 spawn_work(
1445 &cgcx,
1446 &mut llvm_start_time,
1447 get_worker_id(&mut free_worker_ids),
1448 item,
1449 );
1450 }
1451 }
1452 } else if codegen_state == Completed {
1453 if running_with_any_token(main_thread_state, running_with_own_token) == 0
1454 && work_items.is_empty()
1455 {
1456 if needs_fat_lto.is_empty()
1458 && needs_thin_lto.is_empty()
1459 && lto_import_only_modules.is_empty()
1460 {
1461 break;
1463 }
1464
1465 assert!(!started_lto);
1471 started_lto = true;
1472
1473 let needs_fat_lto = mem::take(&mut needs_fat_lto);
1474 let needs_thin_lto = mem::take(&mut needs_thin_lto);
1475 let import_only_modules = mem::take(&mut lto_import_only_modules);
1476
1477 for (work, cost) in generate_lto_work(
1478 &cgcx,
1479 autodiff_items.clone(),
1480 needs_fat_lto,
1481 needs_thin_lto,
1482 import_only_modules,
1483 ) {
1484 let insertion_index = work_items
1485 .binary_search_by_key(&cost, |&(_, cost)| cost)
1486 .unwrap_or_else(|e| e);
1487 work_items.insert(insertion_index, (work, cost));
1488 if cgcx.parallel {
1489 helper.request_token();
1490 }
1491 }
1492 }
1493
1494 match main_thread_state {
1498 MainThreadState::Idle => {
1499 if let Some((item, _)) = work_items.pop() {
1500 main_thread_state = MainThreadState::Lending;
1501 spawn_work(
1502 &cgcx,
1503 &mut llvm_start_time,
1504 get_worker_id(&mut free_worker_ids),
1505 item,
1506 );
1507 } else {
1508 assert!(running_with_own_token > 0);
1515 running_with_own_token -= 1;
1516 main_thread_state = MainThreadState::Lending;
1517 }
1518 }
1519 MainThreadState::Codegenning => bug!(
1520 "codegen worker should not be codegenning after \
1521 codegen was already completed"
1522 ),
1523 MainThreadState::Lending => {
1524 }
1526 }
1527 } else {
1528 assert!(codegen_state == Aborted);
1531 if running_with_any_token(main_thread_state, running_with_own_token) == 0 {
1532 break;
1533 }
1534 }
1535
1536 if codegen_state != Aborted {
1539 while running_with_own_token < tokens.len()
1540 && let Some((item, _)) = work_items.pop()
1541 {
1542 spawn_work(
1543 &cgcx,
1544 &mut llvm_start_time,
1545 get_worker_id(&mut free_worker_ids),
1546 item,
1547 );
1548 running_with_own_token += 1;
1549 }
1550 }
1551
1552 tokens.truncate(running_with_own_token);
1554
1555 let mut free_worker = |worker_id| {
1561 if main_thread_state == MainThreadState::Lending {
1562 main_thread_state = MainThreadState::Idle;
1563 } else {
1564 running_with_own_token -= 1;
1565 }
1566
1567 free_worker_ids.push(worker_id);
1568 };
1569
1570 let msg = coordinator_receive.recv().unwrap();
1571 match *msg.downcast::<Message<B>>().ok().unwrap() {
1572 Message::Token(token) => {
1576 match token {
1577 Ok(token) => {
1578 tokens.push(token);
1579
1580 if main_thread_state == MainThreadState::Lending {
1581 main_thread_state = MainThreadState::Idle;
1586 running_with_own_token += 1;
1587 }
1588 }
1589 Err(e) => {
1590 let msg = &format!("failed to acquire jobserver token: {e}");
1591 shared_emitter.fatal(msg);
1592 codegen_state = Aborted;
1593 }
1594 }
1595 }
1596
1597 Message::CodegenDone { llvm_work_item, cost } => {
1598 let insertion_index = work_items.binary_search_by_key(&cost, |&(_, cost)| cost);
1607 let insertion_index = match insertion_index {
1608 Ok(idx) | Err(idx) => idx,
1609 };
1610 work_items.insert(insertion_index, (llvm_work_item, cost));
1611
1612 if cgcx.parallel {
1613 helper.request_token();
1614 }
1615 assert_eq!(main_thread_state, MainThreadState::Codegenning);
1616 main_thread_state = MainThreadState::Idle;
1617 }
1618
1619 Message::AddAutoDiffItems(mut items) => {
1620 autodiff_items.append(&mut items);
1621 }
1622
1623 Message::CodegenComplete => {
1624 if codegen_state != Aborted {
1625 codegen_state = Completed;
1626 }
1627 assert_eq!(main_thread_state, MainThreadState::Codegenning);
1628 main_thread_state = MainThreadState::Idle;
1629 }
1630
1631 Message::CodegenAborted => {
1639 codegen_state = Aborted;
1640 }
1641
1642 Message::WorkItem { result, worker_id } => {
1643 free_worker(worker_id);
1644
1645 match result {
1646 Ok(WorkItemResult::Finished(compiled_module)) => {
1647 match compiled_module.kind {
1648 ModuleKind::Regular => {
1649 assert!(needs_link.is_empty());
1650 compiled_modules.push(compiled_module);
1651 }
1652 ModuleKind::Allocator => {
1653 assert!(compiled_allocator_module.is_none());
1654 compiled_allocator_module = Some(compiled_module);
1655 }
1656 }
1657 }
1658 Ok(WorkItemResult::NeedsLink(module)) => {
1659 assert!(compiled_modules.is_empty());
1660 needs_link.push(module);
1661 }
1662 Ok(WorkItemResult::NeedsFatLto(fat_lto_input)) => {
1663 assert!(!started_lto);
1664 assert!(needs_thin_lto.is_empty());
1665 needs_fat_lto.push(fat_lto_input);
1666 }
1667 Ok(WorkItemResult::NeedsThinLto(name, thin_buffer)) => {
1668 assert!(!started_lto);
1669 assert!(needs_fat_lto.is_empty());
1670 needs_thin_lto.push((name, thin_buffer));
1671 }
1672 Err(Some(WorkerFatalError)) => {
1673 codegen_state = Aborted;
1675 }
1676 Err(None) => {
1677 bug!("worker thread panicked");
1680 }
1681 }
1682 }
1683
1684 Message::AddImportOnlyModule { module_data, work_product } => {
1685 assert!(!started_lto);
1686 assert_eq!(codegen_state, Ongoing);
1687 assert_eq!(main_thread_state, MainThreadState::Codegenning);
1688 lto_import_only_modules.push((module_data, work_product));
1689 main_thread_state = MainThreadState::Idle;
1690 }
1691 }
1692 }
1693
1694 if codegen_state == Aborted {
1695 return Err(());
1696 }
1697
1698 let needs_link = mem::take(&mut needs_link);
1699 if !needs_link.is_empty() {
1700 assert!(compiled_modules.is_empty());
1701 let dcx = cgcx.create_dcx();
1702 let dcx = dcx.handle();
1703 let module = B::run_link(&cgcx, dcx, needs_link).map_err(|_| ())?;
1704 let module =
1705 B::codegen(&cgcx, dcx, module, cgcx.config(ModuleKind::Regular)).map_err(|_| ())?;
1706 compiled_modules.push(module);
1707 }
1708
1709 drop(llvm_start_time);
1711
1712 compiled_modules.sort_by(|a, b| a.name.cmp(&b.name));
1716
1717 Ok(CompiledModules {
1718 modules: compiled_modules,
1719 allocator_module: compiled_allocator_module,
1720 })
1721 })
1722 .expect("failed to spawn coordinator thread");
1723
1724 fn queue_full_enough(items_in_queue: usize, workers_running: usize) -> bool {
1727 let quarter_of_workers = workers_running - 3 * workers_running / 4;
1778 items_in_queue > 0 && items_in_queue >= quarter_of_workers
1779 }
1780}
1781
1782#[must_use]
1784pub(crate) struct WorkerFatalError;
1785
1786fn spawn_work<'a, B: ExtraBackendMethods>(
1787 cgcx: &'a CodegenContext<B>,
1788 llvm_start_time: &mut Option<VerboseTimingGuard<'a>>,
1789 worker_id: usize,
1790 work: WorkItem<B>,
1791) {
1792 if cgcx.config(work.module_kind()).time_module && llvm_start_time.is_none() {
1793 *llvm_start_time = Some(cgcx.prof.verbose_generic_activity("LLVM_passes"));
1794 }
1795
1796 let cgcx = cgcx.clone();
1797
1798 B::spawn_named_thread(cgcx.time_trace, work.short_description(), move || {
1799 struct Bomb<B: ExtraBackendMethods> {
1802 coordinator_send: Sender<Box<dyn Any + Send>>,
1803 result: Option<Result<WorkItemResult<B>, FatalError>>,
1804 worker_id: usize,
1805 }
1806 impl<B: ExtraBackendMethods> Drop for Bomb<B> {
1807 fn drop(&mut self) {
1808 let worker_id = self.worker_id;
1809 let msg = match self.result.take() {
1810 Some(Ok(result)) => Message::WorkItem::<B> { result: Ok(result), worker_id },
1811 Some(Err(FatalError)) => {
1812 Message::WorkItem::<B> { result: Err(Some(WorkerFatalError)), worker_id }
1813 }
1814 None => Message::WorkItem::<B> { result: Err(None), worker_id },
1815 };
1816 drop(self.coordinator_send.send(Box::new(msg)));
1817 }
1818 }
1819
1820 let mut bomb =
1821 Bomb::<B> { coordinator_send: cgcx.coordinator_send.clone(), result: None, worker_id };
1822
1823 bomb.result = {
1830 let module_config = cgcx.config(work.module_kind());
1831
1832 Some(match work {
1833 WorkItem::Optimize(m) => {
1834 let _timer =
1835 cgcx.prof.generic_activity_with_arg("codegen_module_optimize", &*m.name);
1836 execute_optimize_work_item(&cgcx, m, module_config)
1837 }
1838 WorkItem::CopyPostLtoArtifacts(m) => {
1839 let _timer = cgcx.prof.generic_activity_with_arg(
1840 "codegen_copy_artifacts_from_incr_cache",
1841 &*m.name,
1842 );
1843 Ok(execute_copy_from_cache_work_item(&cgcx, m, module_config))
1844 }
1845 WorkItem::LTO(m) => {
1846 let _timer =
1847 cgcx.prof.generic_activity_with_arg("codegen_module_perform_lto", m.name());
1848 execute_lto_work_item(&cgcx, m, module_config)
1849 }
1850 })
1851 };
1852 })
1853 .expect("failed to spawn work thread");
1854}
1855
1856enum SharedEmitterMessage {
1857 Diagnostic(Diagnostic),
1858 InlineAsmError(SpanData, String, Level, Option<(String, Vec<InnerSpan>)>),
1859 Fatal(String),
1860}
1861
1862#[derive(Clone)]
1863pub struct SharedEmitter {
1864 sender: Sender<SharedEmitterMessage>,
1865}
1866
1867pub struct SharedEmitterMain {
1868 receiver: Receiver<SharedEmitterMessage>,
1869}
1870
1871impl SharedEmitter {
1872 fn new() -> (SharedEmitter, SharedEmitterMain) {
1873 let (sender, receiver) = channel();
1874
1875 (SharedEmitter { sender }, SharedEmitterMain { receiver })
1876 }
1877
1878 pub fn inline_asm_error(
1879 &self,
1880 span: SpanData,
1881 msg: String,
1882 level: Level,
1883 source: Option<(String, Vec<InnerSpan>)>,
1884 ) {
1885 drop(self.sender.send(SharedEmitterMessage::InlineAsmError(span, msg, level, source)));
1886 }
1887
1888 fn fatal(&self, msg: &str) {
1889 drop(self.sender.send(SharedEmitterMessage::Fatal(msg.to_string())));
1890 }
1891}
1892
1893impl Emitter for SharedEmitter {
1894 fn emit_diagnostic(
1895 &mut self,
1896 mut diag: rustc_errors::DiagInner,
1897 _registry: &rustc_errors::registry::Registry,
1898 ) {
1899 assert_eq!(diag.span, MultiSpan::new());
1902 assert_eq!(diag.suggestions, Suggestions::Enabled(vec![]));
1903 assert_eq!(diag.sort_span, rustc_span::DUMMY_SP);
1904 assert_eq!(diag.is_lint, None);
1905 let args = mem::replace(&mut diag.args, DiagArgMap::default());
1908 drop(
1909 self.sender.send(SharedEmitterMessage::Diagnostic(Diagnostic {
1910 level: diag.level(),
1911 messages: diag.messages,
1912 code: diag.code,
1913 children: diag
1914 .children
1915 .into_iter()
1916 .map(|child| Subdiagnostic { level: child.level, messages: child.messages })
1917 .collect(),
1918 args,
1919 })),
1920 );
1921 }
1922
1923 fn source_map(&self) -> Option<&SourceMap> {
1924 None
1925 }
1926
1927 fn translator(&self) -> &Translator {
1928 panic!("shared emitter attempted to translate a diagnostic");
1929 }
1930}
1931
1932impl SharedEmitterMain {
1933 fn check(&self, sess: &Session, blocking: bool) {
1934 loop {
1935 let message = if blocking {
1936 match self.receiver.recv() {
1937 Ok(message) => Ok(message),
1938 Err(_) => Err(()),
1939 }
1940 } else {
1941 match self.receiver.try_recv() {
1942 Ok(message) => Ok(message),
1943 Err(_) => Err(()),
1944 }
1945 };
1946
1947 match message {
1948 Ok(SharedEmitterMessage::Diagnostic(diag)) => {
1949 let dcx = sess.dcx();
1952 let mut d =
1953 rustc_errors::DiagInner::new_with_messages(diag.level, diag.messages);
1954 d.code = diag.code; d.children = diag
1956 .children
1957 .into_iter()
1958 .map(|sub| rustc_errors::Subdiag {
1959 level: sub.level,
1960 messages: sub.messages,
1961 span: MultiSpan::new(),
1962 })
1963 .collect();
1964 d.args = diag.args;
1965 dcx.emit_diagnostic(d);
1966 sess.dcx().abort_if_errors();
1967 }
1968 Ok(SharedEmitterMessage::InlineAsmError(span, msg, level, source)) => {
1969 assert_matches!(level, Level::Error | Level::Warning | Level::Note);
1970 let mut err = Diag::<()>::new(sess.dcx(), level, msg);
1971 if !span.is_dummy() {
1972 err.span(span.span());
1973 }
1974
1975 if let Some((buffer, spans)) = source {
1977 let source = sess
1978 .source_map()
1979 .new_source_file(FileName::inline_asm_source_code(&buffer), buffer);
1980 let spans: Vec<_> = spans
1981 .iter()
1982 .map(|sp| {
1983 Span::with_root_ctxt(
1984 source.normalized_byte_pos(sp.start as u32),
1985 source.normalized_byte_pos(sp.end as u32),
1986 )
1987 })
1988 .collect();
1989 err.span_note(spans, "instantiated into assembly here");
1990 }
1991
1992 err.emit();
1993 }
1994 Ok(SharedEmitterMessage::Fatal(msg)) => {
1995 sess.dcx().fatal(msg);
1996 }
1997 Err(_) => {
1998 break;
1999 }
2000 }
2001 }
2002 }
2003}
2004
2005pub struct Coordinator<B: ExtraBackendMethods> {
2006 pub sender: Sender<Box<dyn Any + Send>>,
2007 future: Option<thread::JoinHandle<Result<CompiledModules, ()>>>,
2008 phantom: PhantomData<B>,
2010}
2011
2012impl<B: ExtraBackendMethods> Coordinator<B> {
2013 fn join(mut self) -> std::thread::Result<Result<CompiledModules, ()>> {
2014 self.future.take().unwrap().join()
2015 }
2016}
2017
2018impl<B: ExtraBackendMethods> Drop for Coordinator<B> {
2019 fn drop(&mut self) {
2020 if let Some(future) = self.future.take() {
2021 drop(self.sender.send(Box::new(Message::CodegenAborted::<B>)));
2024 drop(future.join());
2025 }
2026 }
2027}
2028
2029pub struct OngoingCodegen<B: ExtraBackendMethods> {
2030 pub backend: B,
2031 pub crate_info: CrateInfo,
2032 pub codegen_worker_receive: Receiver<CguMessage>,
2033 pub shared_emitter_main: SharedEmitterMain,
2034 pub output_filenames: Arc<OutputFilenames>,
2035 pub coordinator: Coordinator<B>,
2036}
2037
2038impl<B: ExtraBackendMethods> OngoingCodegen<B> {
2039 pub fn join(self, sess: &Session) -> (CodegenResults, FxIndexMap<WorkProductId, WorkProduct>) {
2040 self.shared_emitter_main.check(sess, true);
2041 let compiled_modules = sess.time("join_worker_thread", || match self.coordinator.join() {
2042 Ok(Ok(compiled_modules)) => compiled_modules,
2043 Ok(Err(())) => {
2044 sess.dcx().abort_if_errors();
2045 panic!("expected abort due to worker thread errors")
2046 }
2047 Err(_) => {
2048 bug!("panic during codegen/LLVM phase");
2049 }
2050 });
2051
2052 sess.dcx().abort_if_errors();
2053
2054 let work_products =
2055 copy_all_cgu_workproducts_to_incr_comp_cache_dir(sess, &compiled_modules);
2056 produce_final_output_artifacts(sess, &compiled_modules, &self.output_filenames);
2057
2058 if sess.codegen_units().as_usize() == 1 && sess.opts.unstable_opts.time_llvm_passes {
2061 self.backend.print_pass_timings()
2062 }
2063
2064 if sess.print_llvm_stats() {
2065 self.backend.print_statistics()
2066 }
2067
2068 (
2069 CodegenResults {
2070 crate_info: self.crate_info,
2071
2072 modules: compiled_modules.modules,
2073 allocator_module: compiled_modules.allocator_module,
2074 },
2075 work_products,
2076 )
2077 }
2078
2079 pub(crate) fn codegen_finished(&self, tcx: TyCtxt<'_>) {
2080 self.wait_for_signal_to_codegen_item();
2081 self.check_for_errors(tcx.sess);
2082 drop(self.coordinator.sender.send(Box::new(Message::CodegenComplete::<B>)));
2083 }
2084
2085 pub(crate) fn submit_autodiff_items(&self, items: Vec<AutoDiffItem>) {
2086 drop(self.coordinator.sender.send(Box::new(Message::<B>::AddAutoDiffItems(items))));
2087 }
2088
2089 pub(crate) fn check_for_errors(&self, sess: &Session) {
2090 self.shared_emitter_main.check(sess, false);
2091 }
2092
2093 pub(crate) fn wait_for_signal_to_codegen_item(&self) {
2094 match self.codegen_worker_receive.recv() {
2095 Ok(CguMessage) => {
2096 }
2098 Err(_) => {
2099 }
2102 }
2103 }
2104}
2105
2106pub(crate) fn submit_codegened_module_to_llvm<B: ExtraBackendMethods>(
2107 _backend: &B,
2108 tx_to_llvm_workers: &Sender<Box<dyn Any + Send>>,
2109 module: ModuleCodegen<B::Module>,
2110 cost: u64,
2111) {
2112 let llvm_work_item = WorkItem::Optimize(module);
2113 drop(tx_to_llvm_workers.send(Box::new(Message::CodegenDone::<B> { llvm_work_item, cost })));
2114}
2115
2116pub(crate) fn submit_post_lto_module_to_llvm<B: ExtraBackendMethods>(
2117 _backend: &B,
2118 tx_to_llvm_workers: &Sender<Box<dyn Any + Send>>,
2119 module: CachedModuleCodegen,
2120) {
2121 let llvm_work_item = WorkItem::CopyPostLtoArtifacts(module);
2122 drop(tx_to_llvm_workers.send(Box::new(Message::CodegenDone::<B> { llvm_work_item, cost: 0 })));
2123}
2124
2125pub(crate) fn submit_pre_lto_module_to_llvm<B: ExtraBackendMethods>(
2126 _backend: &B,
2127 tcx: TyCtxt<'_>,
2128 tx_to_llvm_workers: &Sender<Box<dyn Any + Send>>,
2129 module: CachedModuleCodegen,
2130) {
2131 let filename = pre_lto_bitcode_filename(&module.name);
2132 let bc_path = in_incr_comp_dir_sess(tcx.sess, &filename);
2133 let file = fs::File::open(&bc_path)
2134 .unwrap_or_else(|e| panic!("failed to open bitcode file `{}`: {}", bc_path.display(), e));
2135
2136 let mmap = unsafe {
2137 Mmap::map(file).unwrap_or_else(|e| {
2138 panic!("failed to mmap bitcode file `{}`: {}", bc_path.display(), e)
2139 })
2140 };
2141 drop(tx_to_llvm_workers.send(Box::new(Message::AddImportOnlyModule::<B> {
2143 module_data: SerializedModule::FromUncompressedFile(mmap),
2144 work_product: module.source,
2145 })));
2146}
2147
2148fn pre_lto_bitcode_filename(module_name: &str) -> String {
2149 format!("{module_name}.{PRE_LTO_BC_EXT}")
2150}
2151
2152fn msvc_imps_needed(tcx: TyCtxt<'_>) -> bool {
2153 assert!(
2156 !(tcx.sess.opts.cg.linker_plugin_lto.enabled()
2157 && tcx.sess.target.is_like_windows
2158 && tcx.sess.opts.cg.prefer_dynamic)
2159 );
2160
2161 let can_have_static_objects =
2165 tcx.sess.lto() == Lto::Thin || tcx.crate_types().contains(&CrateType::Rlib);
2166
2167 tcx.sess.target.is_like_windows &&
2168 can_have_static_objects &&
2169 !tcx.sess.opts.cg.linker_plugin_lto.enabled()
2173}