1use std::any::Any;
2use std::assert_matches::assert_matches;
3use std::marker::PhantomData;
4use std::path::{Path, PathBuf};
5use std::sync::Arc;
6use std::sync::mpsc::{Receiver, Sender, channel};
7use std::{fs, io, mem, str, thread};
8
9use rustc_abi::Size;
10use rustc_ast::attr;
11use rustc_ast::expand::autodiff_attrs::AutoDiffItem;
12use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
13use rustc_data_structures::jobserver::{self, Acquired};
14use rustc_data_structures::memmap::Mmap;
15use rustc_data_structures::profiling::{SelfProfilerRef, VerboseTimingGuard};
16use rustc_errors::emitter::Emitter;
17use rustc_errors::translation::Translate;
18use rustc_errors::{
19 Diag, DiagArgMap, DiagCtxt, DiagMessage, ErrCode, FatalError, FluentBundle, Level, MultiSpan,
20 Style, Suggestions,
21};
22use rustc_fs_util::link_or_copy;
23use rustc_hir::def_id::{CrateNum, LOCAL_CRATE};
24use rustc_incremental::{
25 copy_cgu_workproduct_to_incr_comp_cache_dir, in_incr_comp_dir, in_incr_comp_dir_sess,
26};
27use rustc_metadata::EncodedMetadata;
28use rustc_metadata::fs::copy_to_stdout;
29use rustc_middle::bug;
30use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
31use rustc_middle::middle::exported_symbols::SymbolExportInfo;
32use rustc_middle::ty::TyCtxt;
33use rustc_session::Session;
34use rustc_session::config::{
35 self, CrateType, Lto, OutFileName, OutputFilenames, OutputType, Passes, SwitchWithOptPath,
36};
37use rustc_span::source_map::SourceMap;
38use rustc_span::{FileName, InnerSpan, Span, SpanData, sym};
39use rustc_target::spec::{MergeFunctions, SanitizerSet};
40use tracing::debug;
41
42use super::link::{self, ensure_removed};
43use super::lto::{self, SerializedModule};
44use super::symbol_export::symbol_name_for_instance_in_crate;
45use crate::errors::{AutodiffWithoutLto, ErrorCreatingRemarkDir};
46use crate::traits::*;
47use crate::{
48 CachedModuleCodegen, CodegenResults, CompiledModule, CrateInfo, ModuleCodegen, ModuleKind,
49 errors,
50};
51
52const PRE_LTO_BC_EXT: &str = "pre-lto.bc";
53
54#[derive(Clone, Copy, PartialEq)]
56pub enum EmitObj {
57 None,
59
60 Bitcode,
63
64 ObjectCode(BitcodeSection),
66}
67
68#[derive(Clone, Copy, PartialEq)]
70pub enum BitcodeSection {
71 None,
73
74 Full,
76}
77
78pub struct ModuleConfig {
80 pub passes: Vec<String>,
82 pub opt_level: Option<config::OptLevel>,
85
86 pub opt_size: Option<config::OptLevel>,
88
89 pub pgo_gen: SwitchWithOptPath,
90 pub pgo_use: Option<PathBuf>,
91 pub pgo_sample_use: Option<PathBuf>,
92 pub debug_info_for_profiling: bool,
93 pub instrument_coverage: bool,
94
95 pub sanitizer: SanitizerSet,
96 pub sanitizer_recover: SanitizerSet,
97 pub sanitizer_dataflow_abilist: Vec<String>,
98 pub sanitizer_memory_track_origins: usize,
99
100 pub emit_pre_lto_bc: bool,
102 pub emit_no_opt_bc: bool,
103 pub emit_bc: bool,
104 pub emit_ir: bool,
105 pub emit_asm: bool,
106 pub emit_obj: EmitObj,
107 pub emit_thin_lto: bool,
108 pub emit_thin_lto_summary: bool,
109 pub bc_cmdline: String,
110
111 pub verify_llvm_ir: bool,
114 pub lint_llvm_ir: bool,
115 pub no_prepopulate_passes: bool,
116 pub no_builtins: bool,
117 pub time_module: bool,
118 pub vectorize_loop: bool,
119 pub vectorize_slp: bool,
120 pub merge_functions: bool,
121 pub emit_lifetime_markers: bool,
122 pub llvm_plugins: Vec<String>,
123 pub autodiff: Vec<config::AutoDiff>,
124}
125
126impl ModuleConfig {
127 fn new(kind: ModuleKind, tcx: TyCtxt<'_>, no_builtins: bool) -> ModuleConfig {
128 macro_rules! if_regular {
131 ($regular: expr, $other: expr) => {
132 if let ModuleKind::Regular = kind { $regular } else { $other }
133 };
134 }
135
136 let sess = tcx.sess;
137 let opt_level_and_size = if_regular!(Some(sess.opts.optimize), None);
138
139 let save_temps = sess.opts.cg.save_temps;
140
141 let should_emit_obj = sess.opts.output_types.contains_key(&OutputType::Exe)
142 || match kind {
143 ModuleKind::Regular => sess.opts.output_types.contains_key(&OutputType::Object),
144 ModuleKind::Allocator => false,
145 ModuleKind::Metadata => sess.opts.output_types.contains_key(&OutputType::Metadata),
146 };
147
148 let emit_obj = if !should_emit_obj {
149 EmitObj::None
150 } else if sess.target.obj_is_bitcode
151 || (sess.opts.cg.linker_plugin_lto.enabled() && !no_builtins)
152 {
153 EmitObj::Bitcode
168 } else if need_bitcode_in_object(tcx) {
169 EmitObj::ObjectCode(BitcodeSection::Full)
170 } else {
171 EmitObj::ObjectCode(BitcodeSection::None)
172 };
173
174 ModuleConfig {
175 passes: if_regular!(sess.opts.cg.passes.clone(), vec![]),
176
177 opt_level: opt_level_and_size,
178 opt_size: opt_level_and_size,
179
180 pgo_gen: if_regular!(
181 sess.opts.cg.profile_generate.clone(),
182 SwitchWithOptPath::Disabled
183 ),
184 pgo_use: if_regular!(sess.opts.cg.profile_use.clone(), None),
185 pgo_sample_use: if_regular!(sess.opts.unstable_opts.profile_sample_use.clone(), None),
186 debug_info_for_profiling: sess.opts.unstable_opts.debug_info_for_profiling,
187 instrument_coverage: if_regular!(sess.instrument_coverage(), false),
188
189 sanitizer: if_regular!(sess.opts.unstable_opts.sanitizer, SanitizerSet::empty()),
190 sanitizer_dataflow_abilist: if_regular!(
191 sess.opts.unstable_opts.sanitizer_dataflow_abilist.clone(),
192 Vec::new()
193 ),
194 sanitizer_recover: if_regular!(
195 sess.opts.unstable_opts.sanitizer_recover,
196 SanitizerSet::empty()
197 ),
198 sanitizer_memory_track_origins: if_regular!(
199 sess.opts.unstable_opts.sanitizer_memory_track_origins,
200 0
201 ),
202
203 emit_pre_lto_bc: if_regular!(
204 save_temps || need_pre_lto_bitcode_for_incr_comp(sess),
205 false
206 ),
207 emit_no_opt_bc: if_regular!(save_temps, false),
208 emit_bc: if_regular!(
209 save_temps || sess.opts.output_types.contains_key(&OutputType::Bitcode),
210 save_temps
211 ),
212 emit_ir: if_regular!(
213 sess.opts.output_types.contains_key(&OutputType::LlvmAssembly),
214 false
215 ),
216 emit_asm: if_regular!(
217 sess.opts.output_types.contains_key(&OutputType::Assembly),
218 false
219 ),
220 emit_obj,
221 emit_thin_lto: sess.opts.unstable_opts.emit_thin_lto,
222 emit_thin_lto_summary: if_regular!(
223 sess.opts.output_types.contains_key(&OutputType::ThinLinkBitcode),
224 false
225 ),
226 bc_cmdline: sess.target.bitcode_llvm_cmdline.to_string(),
227
228 verify_llvm_ir: sess.verify_llvm_ir(),
229 lint_llvm_ir: sess.opts.unstable_opts.lint_llvm_ir,
230 no_prepopulate_passes: sess.opts.cg.no_prepopulate_passes,
231 no_builtins: no_builtins || sess.target.no_builtins,
232
233 time_module: if_regular!(true, false),
236
237 vectorize_loop: !sess.opts.cg.no_vectorize_loops
240 && (sess.opts.optimize == config::OptLevel::More
241 || sess.opts.optimize == config::OptLevel::Aggressive),
242 vectorize_slp: !sess.opts.cg.no_vectorize_slp
243 && sess.opts.optimize == config::OptLevel::Aggressive,
244
245 merge_functions: match sess
255 .opts
256 .unstable_opts
257 .merge_functions
258 .unwrap_or(sess.target.merge_functions)
259 {
260 MergeFunctions::Disabled => false,
261 MergeFunctions::Trampolines | MergeFunctions::Aliases => {
262 use config::OptLevel::*;
263 match sess.opts.optimize {
264 Aggressive | More | SizeMin | Size => true,
265 Less | No => false,
266 }
267 }
268 },
269
270 emit_lifetime_markers: sess.emit_lifetime_markers(),
271 llvm_plugins: if_regular!(sess.opts.unstable_opts.llvm_plugins.clone(), vec![]),
272 autodiff: if_regular!(sess.opts.unstable_opts.autodiff.clone(), vec![]),
273 }
274 }
275
276 pub fn bitcode_needed(&self) -> bool {
277 self.emit_bc
278 || self.emit_thin_lto_summary
279 || self.emit_obj == EmitObj::Bitcode
280 || self.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full)
281 }
282
283 pub fn embed_bitcode(&self) -> bool {
284 self.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full)
285 }
286}
287
288pub struct TargetMachineFactoryConfig {
290 pub split_dwarf_file: Option<PathBuf>,
294
295 pub output_obj_file: Option<PathBuf>,
298}
299
300impl TargetMachineFactoryConfig {
301 pub fn new(
302 cgcx: &CodegenContext<impl WriteBackendMethods>,
303 module_name: &str,
304 ) -> TargetMachineFactoryConfig {
305 let split_dwarf_file = if cgcx.target_can_use_split_dwarf {
306 cgcx.output_filenames.split_dwarf_path(
307 cgcx.split_debuginfo,
308 cgcx.split_dwarf_kind,
309 Some(module_name),
310 )
311 } else {
312 None
313 };
314
315 let output_obj_file =
316 Some(cgcx.output_filenames.temp_path(OutputType::Object, Some(module_name)));
317 TargetMachineFactoryConfig { split_dwarf_file, output_obj_file }
318 }
319}
320
321pub type TargetMachineFactoryFn<B> = Arc<
322 dyn Fn(
323 TargetMachineFactoryConfig,
324 ) -> Result<
325 <B as WriteBackendMethods>::TargetMachine,
326 <B as WriteBackendMethods>::TargetMachineError,
327 > + Send
328 + Sync,
329>;
330
331type ExportedSymbols = FxHashMap<CrateNum, Arc<Vec<(String, SymbolExportInfo)>>>;
332
333#[derive(Clone)]
335pub struct CodegenContext<B: WriteBackendMethods> {
336 pub prof: SelfProfilerRef,
338 pub lto: Lto,
339 pub save_temps: bool,
340 pub fewer_names: bool,
341 pub time_trace: bool,
342 pub exported_symbols: Option<Arc<ExportedSymbols>>,
343 pub opts: Arc<config::Options>,
344 pub crate_types: Vec<CrateType>,
345 pub each_linked_rlib_for_lto: Vec<(CrateNum, PathBuf)>,
346 pub output_filenames: Arc<OutputFilenames>,
347 pub regular_module_config: Arc<ModuleConfig>,
348 pub metadata_module_config: Arc<ModuleConfig>,
349 pub allocator_module_config: Arc<ModuleConfig>,
350 pub tm_factory: TargetMachineFactoryFn<B>,
351 pub msvc_imps_needed: bool,
352 pub is_pe_coff: bool,
353 pub target_can_use_split_dwarf: bool,
354 pub target_arch: String,
355 pub target_is_like_osx: bool,
356 pub target_is_like_aix: bool,
357 pub split_debuginfo: rustc_target::spec::SplitDebuginfo,
358 pub split_dwarf_kind: rustc_session::config::SplitDwarfKind,
359 pub pointer_size: Size,
360
361 pub expanded_args: Vec<String>,
366
367 pub diag_emitter: SharedEmitter,
369 pub remark: Passes,
371 pub remark_dir: Option<PathBuf>,
374 pub incr_comp_session_dir: Option<PathBuf>,
377 pub coordinator_send: Sender<Box<dyn Any + Send>>,
379 pub parallel: bool,
383}
384
385impl<B: WriteBackendMethods> CodegenContext<B> {
386 pub fn create_dcx(&self) -> DiagCtxt {
387 DiagCtxt::new(Box::new(self.diag_emitter.clone()))
388 }
389
390 pub fn config(&self, kind: ModuleKind) -> &ModuleConfig {
391 match kind {
392 ModuleKind::Regular => &self.regular_module_config,
393 ModuleKind::Metadata => &self.metadata_module_config,
394 ModuleKind::Allocator => &self.allocator_module_config,
395 }
396 }
397}
398
399fn generate_lto_work<B: ExtraBackendMethods>(
400 cgcx: &CodegenContext<B>,
401 autodiff: Vec<AutoDiffItem>,
402 needs_fat_lto: Vec<FatLtoInput<B>>,
403 needs_thin_lto: Vec<(String, B::ThinBuffer)>,
404 import_only_modules: Vec<(SerializedModule<B::ModuleBuffer>, WorkProduct)>,
405) -> Vec<(WorkItem<B>, u64)> {
406 let _prof_timer = cgcx.prof.generic_activity("codegen_generate_lto_work");
407
408 if !needs_fat_lto.is_empty() {
409 assert!(needs_thin_lto.is_empty());
410 let mut module =
411 B::run_fat_lto(cgcx, needs_fat_lto, import_only_modules).unwrap_or_else(|e| e.raise());
412 if cgcx.lto == Lto::Fat && !autodiff.is_empty() {
413 let config = cgcx.config(ModuleKind::Regular);
414 module =
415 unsafe { module.autodiff(cgcx, autodiff, config).unwrap_or_else(|e| e.raise()) };
416 }
417 vec![(WorkItem::LTO(module), 0)]
419 } else {
420 if !autodiff.is_empty() {
421 let dcx = cgcx.create_dcx();
422 dcx.handle().emit_fatal(AutodiffWithoutLto {});
423 }
424 assert!(needs_fat_lto.is_empty());
425 let (lto_modules, copy_jobs) = B::run_thin_lto(cgcx, needs_thin_lto, import_only_modules)
426 .unwrap_or_else(|e| e.raise());
427 lto_modules
428 .into_iter()
429 .map(|module| {
430 let cost = module.cost();
431 (WorkItem::LTO(module), cost)
432 })
433 .chain(copy_jobs.into_iter().map(|wp| {
434 (
435 WorkItem::CopyPostLtoArtifacts(CachedModuleCodegen {
436 name: wp.cgu_name.clone(),
437 source: wp,
438 }),
439 0, )
441 }))
442 .collect()
443 }
444}
445
446struct CompiledModules {
447 modules: Vec<CompiledModule>,
448 allocator_module: Option<CompiledModule>,
449}
450
451fn need_bitcode_in_object(tcx: TyCtxt<'_>) -> bool {
452 let sess = tcx.sess;
453 sess.opts.cg.embed_bitcode
454 && tcx.crate_types().contains(&CrateType::Rlib)
455 && sess.opts.output_types.contains_key(&OutputType::Exe)
456}
457
458fn need_pre_lto_bitcode_for_incr_comp(sess: &Session) -> bool {
459 if sess.opts.incremental.is_none() {
460 return false;
461 }
462
463 match sess.lto() {
464 Lto::No => false,
465 Lto::Fat | Lto::Thin | Lto::ThinLocal => true,
466 }
467}
468
469pub(crate) fn start_async_codegen<B: ExtraBackendMethods>(
470 backend: B,
471 tcx: TyCtxt<'_>,
472 target_cpu: String,
473 metadata: EncodedMetadata,
474 metadata_module: Option<CompiledModule>,
475) -> OngoingCodegen<B> {
476 let (coordinator_send, coordinator_receive) = channel();
477
478 let crate_attrs = tcx.hir_attrs(rustc_hir::CRATE_HIR_ID);
479 let no_builtins = attr::contains_name(crate_attrs, sym::no_builtins);
480
481 let crate_info = CrateInfo::new(tcx, target_cpu);
482
483 let regular_config = ModuleConfig::new(ModuleKind::Regular, tcx, no_builtins);
484 let metadata_config = ModuleConfig::new(ModuleKind::Metadata, tcx, no_builtins);
485 let allocator_config = ModuleConfig::new(ModuleKind::Allocator, tcx, no_builtins);
486
487 let (shared_emitter, shared_emitter_main) = SharedEmitter::new();
488 let (codegen_worker_send, codegen_worker_receive) = channel();
489
490 let coordinator_thread = start_executing_work(
491 backend.clone(),
492 tcx,
493 &crate_info,
494 shared_emitter,
495 codegen_worker_send,
496 coordinator_receive,
497 Arc::new(regular_config),
498 Arc::new(metadata_config),
499 Arc::new(allocator_config),
500 coordinator_send.clone(),
501 );
502
503 OngoingCodegen {
504 backend,
505 metadata,
506 metadata_module,
507 crate_info,
508
509 codegen_worker_receive,
510 shared_emitter_main,
511 coordinator: Coordinator {
512 sender: coordinator_send,
513 future: Some(coordinator_thread),
514 phantom: PhantomData,
515 },
516 output_filenames: Arc::clone(tcx.output_filenames(())),
517 }
518}
519
520fn copy_all_cgu_workproducts_to_incr_comp_cache_dir(
521 sess: &Session,
522 compiled_modules: &CompiledModules,
523) -> FxIndexMap<WorkProductId, WorkProduct> {
524 let mut work_products = FxIndexMap::default();
525
526 if sess.opts.incremental.is_none() {
527 return work_products;
528 }
529
530 let _timer = sess.timer("copy_all_cgu_workproducts_to_incr_comp_cache_dir");
531
532 for module in compiled_modules.modules.iter().filter(|m| m.kind == ModuleKind::Regular) {
533 let mut files = Vec::new();
534 if let Some(object_file_path) = &module.object {
535 files.push((OutputType::Object.extension(), object_file_path.as_path()));
536 }
537 if let Some(dwarf_object_file_path) = &module.dwarf_object {
538 files.push(("dwo", dwarf_object_file_path.as_path()));
539 }
540 if let Some(path) = &module.assembly {
541 files.push((OutputType::Assembly.extension(), path.as_path()));
542 }
543 if let Some(path) = &module.llvm_ir {
544 files.push((OutputType::LlvmAssembly.extension(), path.as_path()));
545 }
546 if let Some(path) = &module.bytecode {
547 files.push((OutputType::Bitcode.extension(), path.as_path()));
548 }
549 if let Some((id, product)) = copy_cgu_workproduct_to_incr_comp_cache_dir(
550 sess,
551 &module.name,
552 files.as_slice(),
553 &module.links_from_incr_cache,
554 ) {
555 work_products.insert(id, product);
556 }
557 }
558
559 work_products
560}
561
562fn produce_final_output_artifacts(
563 sess: &Session,
564 compiled_modules: &CompiledModules,
565 crate_output: &OutputFilenames,
566) {
567 let mut user_wants_bitcode = false;
568 let mut user_wants_objects = false;
569
570 let copy_gracefully = |from: &Path, to: &OutFileName| match to {
572 OutFileName::Stdout if let Err(e) = copy_to_stdout(from) => {
573 sess.dcx().emit_err(errors::CopyPath::new(from, to.as_path(), e));
574 }
575 OutFileName::Real(path) if let Err(e) = fs::copy(from, path) => {
576 sess.dcx().emit_err(errors::CopyPath::new(from, path, e));
577 }
578 _ => {}
579 };
580
581 let copy_if_one_unit = |output_type: OutputType, keep_numbered: bool| {
582 if let [module] = &compiled_modules.modules[..] {
583 let module_name = Some(&module.name[..]);
586 let path = crate_output.temp_path(output_type, module_name);
587 let output = crate_output.path(output_type);
588 if !output_type.is_text_output() && output.is_tty() {
589 sess.dcx()
590 .emit_err(errors::BinaryOutputToTty { shorthand: output_type.shorthand() });
591 } else {
592 copy_gracefully(&path, &output);
593 }
594 if !sess.opts.cg.save_temps && !keep_numbered {
595 ensure_removed(sess.dcx(), &path);
597 }
598 } else {
599 let extension = crate_output
600 .temp_path(output_type, None)
601 .extension()
602 .unwrap()
603 .to_str()
604 .unwrap()
605 .to_owned();
606
607 if crate_output.outputs.contains_explicit_name(&output_type) {
608 sess.dcx().emit_warn(errors::IgnoringEmitPath { extension });
611 } else if crate_output.single_output_file.is_some() {
612 sess.dcx().emit_warn(errors::IgnoringOutput { extension });
615 } else {
616 }
620 }
621 };
622
623 for output_type in crate_output.outputs.keys() {
627 match *output_type {
628 OutputType::Bitcode => {
629 user_wants_bitcode = true;
630 copy_if_one_unit(OutputType::Bitcode, true);
634 }
635 OutputType::ThinLinkBitcode => {
636 copy_if_one_unit(OutputType::ThinLinkBitcode, false);
637 }
638 OutputType::LlvmAssembly => {
639 copy_if_one_unit(OutputType::LlvmAssembly, false);
640 }
641 OutputType::Assembly => {
642 copy_if_one_unit(OutputType::Assembly, false);
643 }
644 OutputType::Object => {
645 user_wants_objects = true;
646 copy_if_one_unit(OutputType::Object, true);
647 }
648 OutputType::Mir | OutputType::Metadata | OutputType::Exe | OutputType::DepInfo => {}
649 }
650 }
651
652 if !sess.opts.cg.save_temps {
665 let needs_crate_object = crate_output.outputs.contains_key(&OutputType::Exe);
681
682 let keep_numbered_bitcode = user_wants_bitcode && sess.codegen_units().as_usize() > 1;
683
684 let keep_numbered_objects =
685 needs_crate_object || (user_wants_objects && sess.codegen_units().as_usize() > 1);
686
687 for module in compiled_modules.modules.iter() {
688 if !keep_numbered_objects {
689 if let Some(ref path) = module.object {
690 ensure_removed(sess.dcx(), path);
691 }
692
693 if let Some(ref path) = module.dwarf_object {
694 ensure_removed(sess.dcx(), path);
695 }
696 }
697
698 if let Some(ref path) = module.bytecode {
699 if !keep_numbered_bitcode {
700 ensure_removed(sess.dcx(), path);
701 }
702 }
703 }
704
705 if !user_wants_bitcode
706 && let Some(ref allocator_module) = compiled_modules.allocator_module
707 && let Some(ref path) = allocator_module.bytecode
708 {
709 ensure_removed(sess.dcx(), path);
710 }
711 }
712
713 if sess.opts.json_artifact_notifications {
714 if let [module] = &compiled_modules.modules[..] {
715 module.for_each_output(|_path, ty| {
716 if sess.opts.output_types.contains_key(&ty) {
717 let descr = ty.shorthand();
718 let path = crate_output.path(ty);
721 sess.dcx().emit_artifact_notification(path.as_path(), descr);
722 }
723 });
724 } else {
725 for module in &compiled_modules.modules {
726 module.for_each_output(|path, ty| {
727 if sess.opts.output_types.contains_key(&ty) {
728 let descr = ty.shorthand();
729 sess.dcx().emit_artifact_notification(&path, descr);
730 }
731 });
732 }
733 }
734 }
735
736 }
742
743pub(crate) enum WorkItem<B: WriteBackendMethods> {
744 Optimize(ModuleCodegen<B::Module>),
746 CopyPostLtoArtifacts(CachedModuleCodegen),
749 LTO(lto::LtoModuleCodegen<B>),
751}
752
753impl<B: WriteBackendMethods> WorkItem<B> {
754 fn module_kind(&self) -> ModuleKind {
755 match *self {
756 WorkItem::Optimize(ref m) => m.kind,
757 WorkItem::CopyPostLtoArtifacts(_) | WorkItem::LTO(_) => ModuleKind::Regular,
758 }
759 }
760
761 fn short_description(&self) -> String {
763 #[cfg(not(windows))]
767 fn desc(short: &str, _long: &str, name: &str) -> String {
768 assert_eq!(short.len(), 3);
788 let name = if let Some(index) = name.find("-cgu.") {
789 &name[index + 1..] } else {
791 name
792 };
793 format!("{short} {name}")
794 }
795
796 #[cfg(windows)]
798 fn desc(_short: &str, long: &str, name: &str) -> String {
799 format!("{long} {name}")
800 }
801
802 match self {
803 WorkItem::Optimize(m) => desc("opt", "optimize module", &m.name),
804 WorkItem::CopyPostLtoArtifacts(m) => desc("cpy", "copy LTO artifacts for", &m.name),
805 WorkItem::LTO(m) => desc("lto", "LTO module", m.name()),
806 }
807 }
808}
809
810pub(crate) enum WorkItemResult<B: WriteBackendMethods> {
812 Finished(CompiledModule),
814
815 NeedsLink(ModuleCodegen<B::Module>),
818
819 NeedsFatLto(FatLtoInput<B>),
822
823 NeedsThinLto(String, B::ThinBuffer),
826}
827
828pub enum FatLtoInput<B: WriteBackendMethods> {
829 Serialized { name: String, buffer: B::ModuleBuffer },
830 InMemory(ModuleCodegen<B::Module>),
831}
832
833pub(crate) enum ComputedLtoType {
835 No,
836 Thin,
837 Fat,
838}
839
840pub(crate) fn compute_per_cgu_lto_type(
841 sess_lto: &Lto,
842 opts: &config::Options,
843 sess_crate_types: &[CrateType],
844 module_kind: ModuleKind,
845) -> ComputedLtoType {
846 if module_kind == ModuleKind::Metadata {
849 return ComputedLtoType::No;
850 }
851
852 let linker_does_lto = opts.cg.linker_plugin_lto.enabled();
856
857 let is_allocator = module_kind == ModuleKind::Allocator;
862
863 let is_rlib = matches!(sess_crate_types, [CrateType::Rlib]);
872
873 match sess_lto {
874 Lto::ThinLocal if !linker_does_lto && !is_allocator => ComputedLtoType::Thin,
875 Lto::Thin if !linker_does_lto && !is_rlib => ComputedLtoType::Thin,
876 Lto::Fat if !is_rlib => ComputedLtoType::Fat,
877 _ => ComputedLtoType::No,
878 }
879}
880
881fn execute_optimize_work_item<B: ExtraBackendMethods>(
882 cgcx: &CodegenContext<B>,
883 mut module: ModuleCodegen<B::Module>,
884 module_config: &ModuleConfig,
885) -> Result<WorkItemResult<B>, FatalError> {
886 let dcx = cgcx.create_dcx();
887 let dcx = dcx.handle();
888
889 unsafe {
890 B::optimize(cgcx, dcx, &mut module, module_config)?;
891 }
892
893 let lto_type = compute_per_cgu_lto_type(&cgcx.lto, &cgcx.opts, &cgcx.crate_types, module.kind);
899
900 let bitcode = if cgcx.config(module.kind).emit_pre_lto_bc {
903 let filename = pre_lto_bitcode_filename(&module.name);
904 cgcx.incr_comp_session_dir.as_ref().map(|path| path.join(&filename))
905 } else {
906 None
907 };
908
909 match lto_type {
910 ComputedLtoType::No => finish_intra_module_work(cgcx, module, module_config),
911 ComputedLtoType::Thin => {
912 let (name, thin_buffer) = B::prepare_thin(module, false);
913 if let Some(path) = bitcode {
914 fs::write(&path, thin_buffer.data()).unwrap_or_else(|e| {
915 panic!("Error writing pre-lto-bitcode file `{}`: {}", path.display(), e);
916 });
917 }
918 Ok(WorkItemResult::NeedsThinLto(name, thin_buffer))
919 }
920 ComputedLtoType::Fat => match bitcode {
921 Some(path) => {
922 let (name, buffer) = B::serialize_module(module);
923 fs::write(&path, buffer.data()).unwrap_or_else(|e| {
924 panic!("Error writing pre-lto-bitcode file `{}`: {}", path.display(), e);
925 });
926 Ok(WorkItemResult::NeedsFatLto(FatLtoInput::Serialized { name, buffer }))
927 }
928 None => Ok(WorkItemResult::NeedsFatLto(FatLtoInput::InMemory(module))),
929 },
930 }
931}
932
933fn execute_copy_from_cache_work_item<B: ExtraBackendMethods>(
934 cgcx: &CodegenContext<B>,
935 module: CachedModuleCodegen,
936 module_config: &ModuleConfig,
937) -> WorkItemResult<B> {
938 let incr_comp_session_dir = cgcx.incr_comp_session_dir.as_ref().unwrap();
939
940 let mut links_from_incr_cache = Vec::new();
941
942 let mut load_from_incr_comp_dir = |output_path: PathBuf, saved_path: &str| {
943 let source_file = in_incr_comp_dir(incr_comp_session_dir, saved_path);
944 debug!(
945 "copying preexisting module `{}` from {:?} to {}",
946 module.name,
947 source_file,
948 output_path.display()
949 );
950 match link_or_copy(&source_file, &output_path) {
951 Ok(_) => {
952 links_from_incr_cache.push(source_file);
953 Some(output_path)
954 }
955 Err(error) => {
956 cgcx.create_dcx().handle().emit_err(errors::CopyPathBuf {
957 source_file,
958 output_path,
959 error,
960 });
961 None
962 }
963 }
964 };
965
966 let dwarf_object =
967 module.source.saved_files.get("dwo").as_ref().and_then(|saved_dwarf_object_file| {
968 let dwarf_obj_out = cgcx
969 .output_filenames
970 .split_dwarf_path(cgcx.split_debuginfo, cgcx.split_dwarf_kind, Some(&module.name))
971 .expect(
972 "saved dwarf object in work product but `split_dwarf_path` returned `None`",
973 );
974 load_from_incr_comp_dir(dwarf_obj_out, saved_dwarf_object_file)
975 });
976
977 let mut load_from_incr_cache = |perform, output_type: OutputType| {
978 if perform {
979 let saved_file = module.source.saved_files.get(output_type.extension())?;
980 let output_path = cgcx.output_filenames.temp_path(output_type, Some(&module.name));
981 load_from_incr_comp_dir(output_path, &saved_file)
982 } else {
983 None
984 }
985 };
986
987 let should_emit_obj = module_config.emit_obj != EmitObj::None;
988 let assembly = load_from_incr_cache(module_config.emit_asm, OutputType::Assembly);
989 let llvm_ir = load_from_incr_cache(module_config.emit_ir, OutputType::LlvmAssembly);
990 let bytecode = load_from_incr_cache(module_config.emit_bc, OutputType::Bitcode);
991 let object = load_from_incr_cache(should_emit_obj, OutputType::Object);
992 if should_emit_obj && object.is_none() {
993 cgcx.create_dcx().handle().emit_fatal(errors::NoSavedObjectFile { cgu_name: &module.name })
994 }
995
996 WorkItemResult::Finished(CompiledModule {
997 links_from_incr_cache,
998 name: module.name,
999 kind: ModuleKind::Regular,
1000 object,
1001 dwarf_object,
1002 bytecode,
1003 assembly,
1004 llvm_ir,
1005 })
1006}
1007
1008fn execute_lto_work_item<B: ExtraBackendMethods>(
1009 cgcx: &CodegenContext<B>,
1010 module: lto::LtoModuleCodegen<B>,
1011 module_config: &ModuleConfig,
1012) -> Result<WorkItemResult<B>, FatalError> {
1013 let module = unsafe { module.optimize(cgcx)? };
1014 finish_intra_module_work(cgcx, module, module_config)
1015}
1016
1017fn finish_intra_module_work<B: ExtraBackendMethods>(
1018 cgcx: &CodegenContext<B>,
1019 module: ModuleCodegen<B::Module>,
1020 module_config: &ModuleConfig,
1021) -> Result<WorkItemResult<B>, FatalError> {
1022 let dcx = cgcx.create_dcx();
1023 let dcx = dcx.handle();
1024
1025 if !cgcx.opts.unstable_opts.combine_cgu
1026 || module.kind == ModuleKind::Metadata
1027 || module.kind == ModuleKind::Allocator
1028 {
1029 let module = unsafe { B::codegen(cgcx, dcx, module, module_config)? };
1030 Ok(WorkItemResult::Finished(module))
1031 } else {
1032 Ok(WorkItemResult::NeedsLink(module))
1033 }
1034}
1035
1036pub(crate) enum Message<B: WriteBackendMethods> {
1038 Token(io::Result<Acquired>),
1041
1042 WorkItem { result: Result<WorkItemResult<B>, Option<WorkerFatalError>>, worker_id: usize },
1045
1046 AddAutoDiffItems(Vec<AutoDiffItem>),
1048
1049 CodegenDone { llvm_work_item: WorkItem<B>, cost: u64 },
1053
1054 AddImportOnlyModule {
1057 module_data: SerializedModule<B::ModuleBuffer>,
1058 work_product: WorkProduct,
1059 },
1060
1061 CodegenComplete,
1064
1065 CodegenAborted,
1068}
1069
1070pub struct CguMessage;
1073
1074struct Diagnostic {
1084 level: Level,
1085 messages: Vec<(DiagMessage, Style)>,
1086 code: Option<ErrCode>,
1087 children: Vec<Subdiagnostic>,
1088 args: DiagArgMap,
1089}
1090
1091pub(crate) struct Subdiagnostic {
1095 level: Level,
1096 messages: Vec<(DiagMessage, Style)>,
1097}
1098
1099#[derive(PartialEq, Clone, Copy, Debug)]
1100enum MainThreadState {
1101 Idle,
1103
1104 Codegenning,
1106
1107 Lending,
1109}
1110
1111fn start_executing_work<B: ExtraBackendMethods>(
1112 backend: B,
1113 tcx: TyCtxt<'_>,
1114 crate_info: &CrateInfo,
1115 shared_emitter: SharedEmitter,
1116 codegen_worker_send: Sender<CguMessage>,
1117 coordinator_receive: Receiver<Box<dyn Any + Send>>,
1118 regular_config: Arc<ModuleConfig>,
1119 metadata_config: Arc<ModuleConfig>,
1120 allocator_config: Arc<ModuleConfig>,
1121 tx_to_llvm_workers: Sender<Box<dyn Any + Send>>,
1122) -> thread::JoinHandle<Result<CompiledModules, ()>> {
1123 let coordinator_send = tx_to_llvm_workers;
1124 let sess = tcx.sess;
1125
1126 let mut each_linked_rlib_for_lto = Vec::new();
1127 drop(link::each_linked_rlib(crate_info, None, &mut |cnum, path| {
1128 if link::ignored_for_lto(sess, crate_info, cnum) {
1129 return;
1130 }
1131 each_linked_rlib_for_lto.push((cnum, path.to_path_buf()));
1132 }));
1133
1134 let exported_symbols = {
1136 let mut exported_symbols = FxHashMap::default();
1137
1138 let copy_symbols = |cnum| {
1139 let symbols = tcx
1140 .exported_symbols(cnum)
1141 .iter()
1142 .map(|&(s, lvl)| (symbol_name_for_instance_in_crate(tcx, s, cnum), lvl))
1143 .collect();
1144 Arc::new(symbols)
1145 };
1146
1147 match sess.lto() {
1148 Lto::No => None,
1149 Lto::ThinLocal => {
1150 exported_symbols.insert(LOCAL_CRATE, copy_symbols(LOCAL_CRATE));
1151 Some(Arc::new(exported_symbols))
1152 }
1153 Lto::Fat | Lto::Thin => {
1154 exported_symbols.insert(LOCAL_CRATE, copy_symbols(LOCAL_CRATE));
1155 for &(cnum, ref _path) in &each_linked_rlib_for_lto {
1156 exported_symbols.insert(cnum, copy_symbols(cnum));
1157 }
1158 Some(Arc::new(exported_symbols))
1159 }
1160 }
1161 };
1162
1163 let coordinator_send2 = coordinator_send.clone();
1169 let helper = jobserver::client()
1170 .into_helper_thread(move |token| {
1171 drop(coordinator_send2.send(Box::new(Message::Token::<B>(token))));
1172 })
1173 .expect("failed to spawn helper thread");
1174
1175 let ol =
1176 if tcx.sess.opts.unstable_opts.no_codegen || !tcx.sess.opts.output_types.should_codegen() {
1177 config::OptLevel::No
1179 } else {
1180 tcx.backend_optimization_level(())
1181 };
1182 let backend_features = tcx.global_backend_features(());
1183
1184 let remark_dir = if let Some(ref dir) = sess.opts.unstable_opts.remark_dir {
1185 let result = fs::create_dir_all(dir).and_then(|_| dir.canonicalize());
1186 match result {
1187 Ok(dir) => Some(dir),
1188 Err(error) => sess.dcx().emit_fatal(ErrorCreatingRemarkDir { error }),
1189 }
1190 } else {
1191 None
1192 };
1193
1194 let cgcx = CodegenContext::<B> {
1195 crate_types: tcx.crate_types().to_vec(),
1196 each_linked_rlib_for_lto,
1197 lto: sess.lto(),
1198 fewer_names: sess.fewer_names(),
1199 save_temps: sess.opts.cg.save_temps,
1200 time_trace: sess.opts.unstable_opts.llvm_time_trace,
1201 opts: Arc::new(sess.opts.clone()),
1202 prof: sess.prof.clone(),
1203 exported_symbols,
1204 remark: sess.opts.cg.remark.clone(),
1205 remark_dir,
1206 incr_comp_session_dir: sess.incr_comp_session_dir_opt().map(|r| r.clone()),
1207 coordinator_send,
1208 expanded_args: tcx.sess.expanded_args.clone(),
1209 diag_emitter: shared_emitter.clone(),
1210 output_filenames: Arc::clone(tcx.output_filenames(())),
1211 regular_module_config: regular_config,
1212 metadata_module_config: metadata_config,
1213 allocator_module_config: allocator_config,
1214 tm_factory: backend.target_machine_factory(tcx.sess, ol, backend_features),
1215 msvc_imps_needed: msvc_imps_needed(tcx),
1216 is_pe_coff: tcx.sess.target.is_like_windows,
1217 target_can_use_split_dwarf: tcx.sess.target_can_use_split_dwarf(),
1218 target_arch: tcx.sess.target.arch.to_string(),
1219 target_is_like_osx: tcx.sess.target.is_like_osx,
1220 target_is_like_aix: tcx.sess.target.is_like_aix,
1221 split_debuginfo: tcx.sess.split_debuginfo(),
1222 split_dwarf_kind: tcx.sess.opts.unstable_opts.split_dwarf_kind,
1223 parallel: backend.supports_parallel() && !sess.opts.unstable_opts.no_parallel_backend,
1224 pointer_size: tcx.data_layout.pointer_size,
1225 };
1226
1227 return B::spawn_named_thread(cgcx.time_trace, "coordinator".to_string(), move || {
1363 let mut worker_id_counter = 0;
1364 let mut free_worker_ids = Vec::new();
1365 let mut get_worker_id = |free_worker_ids: &mut Vec<usize>| {
1366 if let Some(id) = free_worker_ids.pop() {
1367 id
1368 } else {
1369 let id = worker_id_counter;
1370 worker_id_counter += 1;
1371 id
1372 }
1373 };
1374
1375 let mut autodiff_items = Vec::new();
1378 let mut compiled_modules = vec![];
1379 let mut compiled_allocator_module = None;
1380 let mut needs_link = Vec::new();
1381 let mut needs_fat_lto = Vec::new();
1382 let mut needs_thin_lto = Vec::new();
1383 let mut lto_import_only_modules = Vec::new();
1384 let mut started_lto = false;
1385
1386 #[derive(Debug, PartialEq)]
1391 enum CodegenState {
1392 Ongoing,
1393 Completed,
1394 Aborted,
1395 }
1396 use CodegenState::*;
1397 let mut codegen_state = Ongoing;
1398
1399 let mut work_items = Vec::<(WorkItem<B>, u64)>::new();
1401
1402 let mut tokens = Vec::new();
1405
1406 let mut main_thread_state = MainThreadState::Idle;
1407
1408 let mut running_with_own_token = 0;
1411
1412 let running_with_any_token = |main_thread_state, running_with_own_token| {
1415 running_with_own_token
1416 + if main_thread_state == MainThreadState::Lending { 1 } else { 0 }
1417 };
1418
1419 let mut llvm_start_time: Option<VerboseTimingGuard<'_>> = None;
1420
1421 loop {
1427 if codegen_state == Ongoing {
1431 if main_thread_state == MainThreadState::Idle {
1432 let extra_tokens = tokens.len().checked_sub(running_with_own_token).unwrap();
1440 let additional_running = std::cmp::min(extra_tokens, work_items.len());
1441 let anticipated_running = running_with_own_token + additional_running + 1;
1442
1443 if !queue_full_enough(work_items.len(), anticipated_running) {
1444 if codegen_worker_send.send(CguMessage).is_err() {
1446 panic!("Could not send CguMessage to main thread")
1447 }
1448 main_thread_state = MainThreadState::Codegenning;
1449 } else {
1450 let (item, _) =
1454 work_items.pop().expect("queue empty - queue_full_enough() broken?");
1455 main_thread_state = MainThreadState::Lending;
1456 spawn_work(
1457 &cgcx,
1458 &mut llvm_start_time,
1459 get_worker_id(&mut free_worker_ids),
1460 item,
1461 );
1462 }
1463 }
1464 } else if codegen_state == Completed {
1465 if running_with_any_token(main_thread_state, running_with_own_token) == 0
1466 && work_items.is_empty()
1467 {
1468 if needs_fat_lto.is_empty()
1470 && needs_thin_lto.is_empty()
1471 && lto_import_only_modules.is_empty()
1472 {
1473 break;
1475 }
1476
1477 assert!(!started_lto);
1483 started_lto = true;
1484
1485 let needs_fat_lto = mem::take(&mut needs_fat_lto);
1486 let needs_thin_lto = mem::take(&mut needs_thin_lto);
1487 let import_only_modules = mem::take(&mut lto_import_only_modules);
1488
1489 for (work, cost) in generate_lto_work(
1490 &cgcx,
1491 autodiff_items.clone(),
1492 needs_fat_lto,
1493 needs_thin_lto,
1494 import_only_modules,
1495 ) {
1496 let insertion_index = work_items
1497 .binary_search_by_key(&cost, |&(_, cost)| cost)
1498 .unwrap_or_else(|e| e);
1499 work_items.insert(insertion_index, (work, cost));
1500 if cgcx.parallel {
1501 helper.request_token();
1502 }
1503 }
1504 }
1505
1506 match main_thread_state {
1510 MainThreadState::Idle => {
1511 if let Some((item, _)) = work_items.pop() {
1512 main_thread_state = MainThreadState::Lending;
1513 spawn_work(
1514 &cgcx,
1515 &mut llvm_start_time,
1516 get_worker_id(&mut free_worker_ids),
1517 item,
1518 );
1519 } else {
1520 assert!(running_with_own_token > 0);
1527 running_with_own_token -= 1;
1528 main_thread_state = MainThreadState::Lending;
1529 }
1530 }
1531 MainThreadState::Codegenning => bug!(
1532 "codegen worker should not be codegenning after \
1533 codegen was already completed"
1534 ),
1535 MainThreadState::Lending => {
1536 }
1538 }
1539 } else {
1540 assert!(codegen_state == Aborted);
1543 if running_with_any_token(main_thread_state, running_with_own_token) == 0 {
1544 break;
1545 }
1546 }
1547
1548 if codegen_state != Aborted {
1551 while running_with_own_token < tokens.len()
1552 && let Some((item, _)) = work_items.pop()
1553 {
1554 spawn_work(
1555 &cgcx,
1556 &mut llvm_start_time,
1557 get_worker_id(&mut free_worker_ids),
1558 item,
1559 );
1560 running_with_own_token += 1;
1561 }
1562 }
1563
1564 tokens.truncate(running_with_own_token);
1566
1567 let mut free_worker = |worker_id| {
1573 if main_thread_state == MainThreadState::Lending {
1574 main_thread_state = MainThreadState::Idle;
1575 } else {
1576 running_with_own_token -= 1;
1577 }
1578
1579 free_worker_ids.push(worker_id);
1580 };
1581
1582 let msg = coordinator_receive.recv().unwrap();
1583 match *msg.downcast::<Message<B>>().ok().unwrap() {
1584 Message::Token(token) => {
1588 match token {
1589 Ok(token) => {
1590 tokens.push(token);
1591
1592 if main_thread_state == MainThreadState::Lending {
1593 main_thread_state = MainThreadState::Idle;
1598 running_with_own_token += 1;
1599 }
1600 }
1601 Err(e) => {
1602 let msg = &format!("failed to acquire jobserver token: {e}");
1603 shared_emitter.fatal(msg);
1604 codegen_state = Aborted;
1605 }
1606 }
1607 }
1608
1609 Message::CodegenDone { llvm_work_item, cost } => {
1610 let insertion_index = work_items.binary_search_by_key(&cost, |&(_, cost)| cost);
1619 let insertion_index = match insertion_index {
1620 Ok(idx) | Err(idx) => idx,
1621 };
1622 work_items.insert(insertion_index, (llvm_work_item, cost));
1623
1624 if cgcx.parallel {
1625 helper.request_token();
1626 }
1627 assert_eq!(main_thread_state, MainThreadState::Codegenning);
1628 main_thread_state = MainThreadState::Idle;
1629 }
1630
1631 Message::AddAutoDiffItems(mut items) => {
1632 autodiff_items.append(&mut items);
1633 }
1634
1635 Message::CodegenComplete => {
1636 if codegen_state != Aborted {
1637 codegen_state = Completed;
1638 }
1639 assert_eq!(main_thread_state, MainThreadState::Codegenning);
1640 main_thread_state = MainThreadState::Idle;
1641 }
1642
1643 Message::CodegenAborted => {
1651 codegen_state = Aborted;
1652 }
1653
1654 Message::WorkItem { result, worker_id } => {
1655 free_worker(worker_id);
1656
1657 match result {
1658 Ok(WorkItemResult::Finished(compiled_module)) => {
1659 match compiled_module.kind {
1660 ModuleKind::Regular => {
1661 assert!(needs_link.is_empty());
1662 compiled_modules.push(compiled_module);
1663 }
1664 ModuleKind::Allocator => {
1665 assert!(compiled_allocator_module.is_none());
1666 compiled_allocator_module = Some(compiled_module);
1667 }
1668 ModuleKind::Metadata => bug!("Should be handled separately"),
1669 }
1670 }
1671 Ok(WorkItemResult::NeedsLink(module)) => {
1672 assert!(compiled_modules.is_empty());
1673 needs_link.push(module);
1674 }
1675 Ok(WorkItemResult::NeedsFatLto(fat_lto_input)) => {
1676 assert!(!started_lto);
1677 assert!(needs_thin_lto.is_empty());
1678 needs_fat_lto.push(fat_lto_input);
1679 }
1680 Ok(WorkItemResult::NeedsThinLto(name, thin_buffer)) => {
1681 assert!(!started_lto);
1682 assert!(needs_fat_lto.is_empty());
1683 needs_thin_lto.push((name, thin_buffer));
1684 }
1685 Err(Some(WorkerFatalError)) => {
1686 codegen_state = Aborted;
1688 }
1689 Err(None) => {
1690 bug!("worker thread panicked");
1693 }
1694 }
1695 }
1696
1697 Message::AddImportOnlyModule { module_data, work_product } => {
1698 assert!(!started_lto);
1699 assert_eq!(codegen_state, Ongoing);
1700 assert_eq!(main_thread_state, MainThreadState::Codegenning);
1701 lto_import_only_modules.push((module_data, work_product));
1702 main_thread_state = MainThreadState::Idle;
1703 }
1704 }
1705 }
1706
1707 if codegen_state == Aborted {
1708 return Err(());
1709 }
1710
1711 let needs_link = mem::take(&mut needs_link);
1712 if !needs_link.is_empty() {
1713 assert!(compiled_modules.is_empty());
1714 let dcx = cgcx.create_dcx();
1715 let dcx = dcx.handle();
1716 let module = B::run_link(&cgcx, dcx, needs_link).map_err(|_| ())?;
1717 let module = unsafe {
1718 B::codegen(&cgcx, dcx, module, cgcx.config(ModuleKind::Regular)).map_err(|_| ())?
1719 };
1720 compiled_modules.push(module);
1721 }
1722
1723 drop(llvm_start_time);
1725
1726 compiled_modules.sort_by(|a, b| a.name.cmp(&b.name));
1730
1731 Ok(CompiledModules {
1732 modules: compiled_modules,
1733 allocator_module: compiled_allocator_module,
1734 })
1735 })
1736 .expect("failed to spawn coordinator thread");
1737
1738 fn queue_full_enough(items_in_queue: usize, workers_running: usize) -> bool {
1741 let quarter_of_workers = workers_running - 3 * workers_running / 4;
1792 items_in_queue > 0 && items_in_queue >= quarter_of_workers
1793 }
1794}
1795
1796#[must_use]
1798pub(crate) struct WorkerFatalError;
1799
1800fn spawn_work<'a, B: ExtraBackendMethods>(
1801 cgcx: &'a CodegenContext<B>,
1802 llvm_start_time: &mut Option<VerboseTimingGuard<'a>>,
1803 worker_id: usize,
1804 work: WorkItem<B>,
1805) {
1806 if cgcx.config(work.module_kind()).time_module && llvm_start_time.is_none() {
1807 *llvm_start_time = Some(cgcx.prof.verbose_generic_activity("LLVM_passes"));
1808 }
1809
1810 let cgcx = cgcx.clone();
1811
1812 B::spawn_named_thread(cgcx.time_trace, work.short_description(), move || {
1813 struct Bomb<B: ExtraBackendMethods> {
1816 coordinator_send: Sender<Box<dyn Any + Send>>,
1817 result: Option<Result<WorkItemResult<B>, FatalError>>,
1818 worker_id: usize,
1819 }
1820 impl<B: ExtraBackendMethods> Drop for Bomb<B> {
1821 fn drop(&mut self) {
1822 let worker_id = self.worker_id;
1823 let msg = match self.result.take() {
1824 Some(Ok(result)) => Message::WorkItem::<B> { result: Ok(result), worker_id },
1825 Some(Err(FatalError)) => {
1826 Message::WorkItem::<B> { result: Err(Some(WorkerFatalError)), worker_id }
1827 }
1828 None => Message::WorkItem::<B> { result: Err(None), worker_id },
1829 };
1830 drop(self.coordinator_send.send(Box::new(msg)));
1831 }
1832 }
1833
1834 let mut bomb =
1835 Bomb::<B> { coordinator_send: cgcx.coordinator_send.clone(), result: None, worker_id };
1836
1837 bomb.result = {
1844 let module_config = cgcx.config(work.module_kind());
1845
1846 Some(match work {
1847 WorkItem::Optimize(m) => {
1848 let _timer =
1849 cgcx.prof.generic_activity_with_arg("codegen_module_optimize", &*m.name);
1850 execute_optimize_work_item(&cgcx, m, module_config)
1851 }
1852 WorkItem::CopyPostLtoArtifacts(m) => {
1853 let _timer = cgcx.prof.generic_activity_with_arg(
1854 "codegen_copy_artifacts_from_incr_cache",
1855 &*m.name,
1856 );
1857 Ok(execute_copy_from_cache_work_item(&cgcx, m, module_config))
1858 }
1859 WorkItem::LTO(m) => {
1860 let _timer =
1861 cgcx.prof.generic_activity_with_arg("codegen_module_perform_lto", m.name());
1862 execute_lto_work_item(&cgcx, m, module_config)
1863 }
1864 })
1865 };
1866 })
1867 .expect("failed to spawn work thread");
1868}
1869
1870enum SharedEmitterMessage {
1871 Diagnostic(Diagnostic),
1872 InlineAsmError(SpanData, String, Level, Option<(String, Vec<InnerSpan>)>),
1873 Fatal(String),
1874}
1875
1876#[derive(Clone)]
1877pub struct SharedEmitter {
1878 sender: Sender<SharedEmitterMessage>,
1879}
1880
1881pub struct SharedEmitterMain {
1882 receiver: Receiver<SharedEmitterMessage>,
1883}
1884
1885impl SharedEmitter {
1886 fn new() -> (SharedEmitter, SharedEmitterMain) {
1887 let (sender, receiver) = channel();
1888
1889 (SharedEmitter { sender }, SharedEmitterMain { receiver })
1890 }
1891
1892 pub fn inline_asm_error(
1893 &self,
1894 span: SpanData,
1895 msg: String,
1896 level: Level,
1897 source: Option<(String, Vec<InnerSpan>)>,
1898 ) {
1899 drop(self.sender.send(SharedEmitterMessage::InlineAsmError(span, msg, level, source)));
1900 }
1901
1902 fn fatal(&self, msg: &str) {
1903 drop(self.sender.send(SharedEmitterMessage::Fatal(msg.to_string())));
1904 }
1905}
1906
1907impl Translate for SharedEmitter {
1908 fn fluent_bundle(&self) -> Option<&FluentBundle> {
1909 None
1910 }
1911
1912 fn fallback_fluent_bundle(&self) -> &FluentBundle {
1913 panic!("shared emitter attempted to translate a diagnostic");
1914 }
1915}
1916
1917impl Emitter for SharedEmitter {
1918 fn emit_diagnostic(
1919 &mut self,
1920 mut diag: rustc_errors::DiagInner,
1921 _registry: &rustc_errors::registry::Registry,
1922 ) {
1923 assert_eq!(diag.span, MultiSpan::new());
1926 assert_eq!(diag.suggestions, Suggestions::Enabled(vec![]));
1927 assert_eq!(diag.sort_span, rustc_span::DUMMY_SP);
1928 assert_eq!(diag.is_lint, None);
1929 let args = mem::replace(&mut diag.args, DiagArgMap::default());
1932 drop(
1933 self.sender.send(SharedEmitterMessage::Diagnostic(Diagnostic {
1934 level: diag.level(),
1935 messages: diag.messages,
1936 code: diag.code,
1937 children: diag
1938 .children
1939 .into_iter()
1940 .map(|child| Subdiagnostic { level: child.level, messages: child.messages })
1941 .collect(),
1942 args,
1943 })),
1944 );
1945 }
1946
1947 fn source_map(&self) -> Option<&SourceMap> {
1948 None
1949 }
1950}
1951
1952impl SharedEmitterMain {
1953 fn check(&self, sess: &Session, blocking: bool) {
1954 loop {
1955 let message = if blocking {
1956 match self.receiver.recv() {
1957 Ok(message) => Ok(message),
1958 Err(_) => Err(()),
1959 }
1960 } else {
1961 match self.receiver.try_recv() {
1962 Ok(message) => Ok(message),
1963 Err(_) => Err(()),
1964 }
1965 };
1966
1967 match message {
1968 Ok(SharedEmitterMessage::Diagnostic(diag)) => {
1969 let dcx = sess.dcx();
1972 let mut d =
1973 rustc_errors::DiagInner::new_with_messages(diag.level, diag.messages);
1974 d.code = diag.code; d.children = diag
1976 .children
1977 .into_iter()
1978 .map(|sub| rustc_errors::Subdiag {
1979 level: sub.level,
1980 messages: sub.messages,
1981 span: MultiSpan::new(),
1982 })
1983 .collect();
1984 d.args = diag.args;
1985 dcx.emit_diagnostic(d);
1986 sess.dcx().abort_if_errors();
1987 }
1988 Ok(SharedEmitterMessage::InlineAsmError(span, msg, level, source)) => {
1989 assert_matches!(level, Level::Error | Level::Warning | Level::Note);
1990 let mut err = Diag::<()>::new(sess.dcx(), level, msg);
1991 if !span.is_dummy() {
1992 err.span(span.span());
1993 }
1994
1995 if let Some((buffer, spans)) = source {
1997 let source = sess
1998 .source_map()
1999 .new_source_file(FileName::inline_asm_source_code(&buffer), buffer);
2000 let spans: Vec<_> = spans
2001 .iter()
2002 .map(|sp| {
2003 Span::with_root_ctxt(
2004 source.normalized_byte_pos(sp.start as u32),
2005 source.normalized_byte_pos(sp.end as u32),
2006 )
2007 })
2008 .collect();
2009 err.span_note(spans, "instantiated into assembly here");
2010 }
2011
2012 err.emit();
2013 }
2014 Ok(SharedEmitterMessage::Fatal(msg)) => {
2015 sess.dcx().fatal(msg);
2016 }
2017 Err(_) => {
2018 break;
2019 }
2020 }
2021 }
2022 }
2023}
2024
2025pub struct Coordinator<B: ExtraBackendMethods> {
2026 pub sender: Sender<Box<dyn Any + Send>>,
2027 future: Option<thread::JoinHandle<Result<CompiledModules, ()>>>,
2028 phantom: PhantomData<B>,
2030}
2031
2032impl<B: ExtraBackendMethods> Coordinator<B> {
2033 fn join(mut self) -> std::thread::Result<Result<CompiledModules, ()>> {
2034 self.future.take().unwrap().join()
2035 }
2036}
2037
2038impl<B: ExtraBackendMethods> Drop for Coordinator<B> {
2039 fn drop(&mut self) {
2040 if let Some(future) = self.future.take() {
2041 drop(self.sender.send(Box::new(Message::CodegenAborted::<B>)));
2044 drop(future.join());
2045 }
2046 }
2047}
2048
2049pub struct OngoingCodegen<B: ExtraBackendMethods> {
2050 pub backend: B,
2051 pub metadata: EncodedMetadata,
2052 pub metadata_module: Option<CompiledModule>,
2053 pub crate_info: CrateInfo,
2054 pub codegen_worker_receive: Receiver<CguMessage>,
2055 pub shared_emitter_main: SharedEmitterMain,
2056 pub output_filenames: Arc<OutputFilenames>,
2057 pub coordinator: Coordinator<B>,
2058}
2059
2060impl<B: ExtraBackendMethods> OngoingCodegen<B> {
2061 pub fn join(self, sess: &Session) -> (CodegenResults, FxIndexMap<WorkProductId, WorkProduct>) {
2062 self.shared_emitter_main.check(sess, true);
2063 let compiled_modules = sess.time("join_worker_thread", || match self.coordinator.join() {
2064 Ok(Ok(compiled_modules)) => compiled_modules,
2065 Ok(Err(())) => {
2066 sess.dcx().abort_if_errors();
2067 panic!("expected abort due to worker thread errors")
2068 }
2069 Err(_) => {
2070 bug!("panic during codegen/LLVM phase");
2071 }
2072 });
2073
2074 sess.dcx().abort_if_errors();
2075
2076 let work_products =
2077 copy_all_cgu_workproducts_to_incr_comp_cache_dir(sess, &compiled_modules);
2078 produce_final_output_artifacts(sess, &compiled_modules, &self.output_filenames);
2079
2080 if sess.codegen_units().as_usize() == 1 && sess.opts.unstable_opts.time_llvm_passes {
2083 self.backend.print_pass_timings()
2084 }
2085
2086 if sess.print_llvm_stats() {
2087 self.backend.print_statistics()
2088 }
2089
2090 (
2091 CodegenResults {
2092 metadata: self.metadata,
2093 crate_info: self.crate_info,
2094
2095 modules: compiled_modules.modules,
2096 allocator_module: compiled_modules.allocator_module,
2097 metadata_module: self.metadata_module,
2098 },
2099 work_products,
2100 )
2101 }
2102
2103 pub(crate) fn codegen_finished(&self, tcx: TyCtxt<'_>) {
2104 self.wait_for_signal_to_codegen_item();
2105 self.check_for_errors(tcx.sess);
2106 drop(self.coordinator.sender.send(Box::new(Message::CodegenComplete::<B>)));
2107 }
2108
2109 pub(crate) fn submit_autodiff_items(&self, items: Vec<AutoDiffItem>) {
2110 drop(self.coordinator.sender.send(Box::new(Message::<B>::AddAutoDiffItems(items))));
2111 }
2112
2113 pub(crate) fn check_for_errors(&self, sess: &Session) {
2114 self.shared_emitter_main.check(sess, false);
2115 }
2116
2117 pub(crate) fn wait_for_signal_to_codegen_item(&self) {
2118 match self.codegen_worker_receive.recv() {
2119 Ok(CguMessage) => {
2120 }
2122 Err(_) => {
2123 }
2126 }
2127 }
2128}
2129
2130pub(crate) fn submit_codegened_module_to_llvm<B: ExtraBackendMethods>(
2131 _backend: &B,
2132 tx_to_llvm_workers: &Sender<Box<dyn Any + Send>>,
2133 module: ModuleCodegen<B::Module>,
2134 cost: u64,
2135) {
2136 let llvm_work_item = WorkItem::Optimize(module);
2137 drop(tx_to_llvm_workers.send(Box::new(Message::CodegenDone::<B> { llvm_work_item, cost })));
2138}
2139
2140pub(crate) fn submit_post_lto_module_to_llvm<B: ExtraBackendMethods>(
2141 _backend: &B,
2142 tx_to_llvm_workers: &Sender<Box<dyn Any + Send>>,
2143 module: CachedModuleCodegen,
2144) {
2145 let llvm_work_item = WorkItem::CopyPostLtoArtifacts(module);
2146 drop(tx_to_llvm_workers.send(Box::new(Message::CodegenDone::<B> { llvm_work_item, cost: 0 })));
2147}
2148
2149pub(crate) fn submit_pre_lto_module_to_llvm<B: ExtraBackendMethods>(
2150 _backend: &B,
2151 tcx: TyCtxt<'_>,
2152 tx_to_llvm_workers: &Sender<Box<dyn Any + Send>>,
2153 module: CachedModuleCodegen,
2154) {
2155 let filename = pre_lto_bitcode_filename(&module.name);
2156 let bc_path = in_incr_comp_dir_sess(tcx.sess, &filename);
2157 let file = fs::File::open(&bc_path)
2158 .unwrap_or_else(|e| panic!("failed to open bitcode file `{}`: {}", bc_path.display(), e));
2159
2160 let mmap = unsafe {
2161 Mmap::map(file).unwrap_or_else(|e| {
2162 panic!("failed to mmap bitcode file `{}`: {}", bc_path.display(), e)
2163 })
2164 };
2165 drop(tx_to_llvm_workers.send(Box::new(Message::AddImportOnlyModule::<B> {
2167 module_data: SerializedModule::FromUncompressedFile(mmap),
2168 work_product: module.source,
2169 })));
2170}
2171
2172fn pre_lto_bitcode_filename(module_name: &str) -> String {
2173 format!("{module_name}.{PRE_LTO_BC_EXT}")
2174}
2175
2176fn msvc_imps_needed(tcx: TyCtxt<'_>) -> bool {
2177 assert!(
2180 !(tcx.sess.opts.cg.linker_plugin_lto.enabled()
2181 && tcx.sess.target.is_like_windows
2182 && tcx.sess.opts.cg.prefer_dynamic)
2183 );
2184
2185 let can_have_static_objects =
2189 tcx.sess.lto() == Lto::Thin || tcx.crate_types().iter().any(|ct| *ct == CrateType::Rlib);
2190
2191 tcx.sess.target.is_like_windows &&
2192 can_have_static_objects &&
2193 !tcx.sess.opts.cg.linker_plugin_lto.enabled()
2197}