rustc_codegen_ssa/back/
write.rs

1use std::any::Any;
2use std::assert_matches::assert_matches;
3use std::marker::PhantomData;
4use std::path::{Path, PathBuf};
5use std::sync::Arc;
6use std::sync::mpsc::{Receiver, Sender, channel};
7use std::{fs, io, mem, str, thread};
8
9use rustc_abi::Size;
10use rustc_ast::attr;
11use rustc_ast::expand::autodiff_attrs::AutoDiffItem;
12use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
13use rustc_data_structures::jobserver::{self, Acquired};
14use rustc_data_structures::memmap::Mmap;
15use rustc_data_structures::profiling::{SelfProfilerRef, VerboseTimingGuard};
16use rustc_errors::emitter::Emitter;
17use rustc_errors::translation::Translator;
18use rustc_errors::{
19    Diag, DiagArgMap, DiagCtxt, DiagMessage, ErrCode, FatalError, Level, MultiSpan, Style,
20    Suggestions,
21};
22use rustc_fs_util::link_or_copy;
23use rustc_hir::def_id::{CrateNum, LOCAL_CRATE};
24use rustc_incremental::{
25    copy_cgu_workproduct_to_incr_comp_cache_dir, in_incr_comp_dir, in_incr_comp_dir_sess,
26};
27use rustc_metadata::fs::copy_to_stdout;
28use rustc_middle::bug;
29use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
30use rustc_middle::middle::exported_symbols::SymbolExportInfo;
31use rustc_middle::ty::TyCtxt;
32use rustc_session::Session;
33use rustc_session::config::{
34    self, CrateType, Lto, OutFileName, OutputFilenames, OutputType, Passes, SwitchWithOptPath,
35};
36use rustc_span::source_map::SourceMap;
37use rustc_span::{FileName, InnerSpan, Span, SpanData, sym};
38use rustc_target::spec::{MergeFunctions, SanitizerSet};
39use tracing::debug;
40
41use super::link::{self, ensure_removed};
42use super::lto::{self, SerializedModule};
43use super::symbol_export::symbol_name_for_instance_in_crate;
44use crate::errors::{AutodiffWithoutLto, ErrorCreatingRemarkDir};
45use crate::traits::*;
46use crate::{
47    CachedModuleCodegen, CodegenResults, CompiledModule, CrateInfo, ModuleCodegen, ModuleKind,
48    errors,
49};
50
51const PRE_LTO_BC_EXT: &str = "pre-lto.bc";
52
53/// What kind of object file to emit.
54#[derive(Clone, Copy, PartialEq)]
55pub enum EmitObj {
56    // No object file.
57    None,
58
59    // Just uncompressed llvm bitcode. Provides easy compatibility with
60    // emscripten's ecc compiler, when used as the linker.
61    Bitcode,
62
63    // Object code, possibly augmented with a bitcode section.
64    ObjectCode(BitcodeSection),
65}
66
67/// What kind of llvm bitcode section to embed in an object file.
68#[derive(Clone, Copy, PartialEq)]
69pub enum BitcodeSection {
70    // No bitcode section.
71    None,
72
73    // A full, uncompressed bitcode section.
74    Full,
75}
76
77/// Module-specific configuration for `optimize_and_codegen`.
78pub struct ModuleConfig {
79    /// Names of additional optimization passes to run.
80    pub passes: Vec<String>,
81    /// Some(level) to optimize at a certain level, or None to run
82    /// absolutely no optimizations (used for the metadata module).
83    pub opt_level: Option<config::OptLevel>,
84
85    /// Some(level) to optimize binary size, or None to not affect program size.
86    pub opt_size: Option<config::OptLevel>,
87
88    pub pgo_gen: SwitchWithOptPath,
89    pub pgo_use: Option<PathBuf>,
90    pub pgo_sample_use: Option<PathBuf>,
91    pub debug_info_for_profiling: bool,
92    pub instrument_coverage: bool,
93
94    pub sanitizer: SanitizerSet,
95    pub sanitizer_recover: SanitizerSet,
96    pub sanitizer_dataflow_abilist: Vec<String>,
97    pub sanitizer_memory_track_origins: usize,
98
99    // Flags indicating which outputs to produce.
100    pub emit_pre_lto_bc: bool,
101    pub emit_no_opt_bc: bool,
102    pub emit_bc: bool,
103    pub emit_ir: bool,
104    pub emit_asm: bool,
105    pub emit_obj: EmitObj,
106    pub emit_thin_lto: bool,
107    pub emit_thin_lto_summary: bool,
108    pub bc_cmdline: String,
109
110    // Miscellaneous flags. These are mostly copied from command-line
111    // options.
112    pub verify_llvm_ir: bool,
113    pub lint_llvm_ir: bool,
114    pub no_prepopulate_passes: bool,
115    pub no_builtins: bool,
116    pub time_module: bool,
117    pub vectorize_loop: bool,
118    pub vectorize_slp: bool,
119    pub merge_functions: bool,
120    pub emit_lifetime_markers: bool,
121    pub llvm_plugins: Vec<String>,
122    pub autodiff: Vec<config::AutoDiff>,
123}
124
125impl ModuleConfig {
126    fn new(kind: ModuleKind, tcx: TyCtxt<'_>, no_builtins: bool) -> ModuleConfig {
127        // If it's a regular module, use `$regular`, otherwise use `$other`.
128        // `$regular` and `$other` are evaluated lazily.
129        macro_rules! if_regular {
130            ($regular: expr, $other: expr) => {
131                if let ModuleKind::Regular = kind { $regular } else { $other }
132            };
133        }
134
135        let sess = tcx.sess;
136        let opt_level_and_size = if_regular!(Some(sess.opts.optimize), None);
137
138        let save_temps = sess.opts.cg.save_temps;
139
140        let should_emit_obj = sess.opts.output_types.contains_key(&OutputType::Exe)
141            || match kind {
142                ModuleKind::Regular => sess.opts.output_types.contains_key(&OutputType::Object),
143                ModuleKind::Allocator => false,
144            };
145
146        let emit_obj = if !should_emit_obj {
147            EmitObj::None
148        } else if sess.target.obj_is_bitcode
149            || (sess.opts.cg.linker_plugin_lto.enabled() && !no_builtins)
150        {
151            // This case is selected if the target uses objects as bitcode, or
152            // if linker plugin LTO is enabled. In the linker plugin LTO case
153            // the assumption is that the final link-step will read the bitcode
154            // and convert it to object code. This may be done by either the
155            // native linker or rustc itself.
156            //
157            // Note, however, that the linker-plugin-lto requested here is
158            // explicitly ignored for `#![no_builtins]` crates. These crates are
159            // specifically ignored by rustc's LTO passes and wouldn't work if
160            // loaded into the linker. These crates define symbols that LLVM
161            // lowers intrinsics to, and these symbol dependencies aren't known
162            // until after codegen. As a result any crate marked
163            // `#![no_builtins]` is assumed to not participate in LTO and
164            // instead goes on to generate object code.
165            EmitObj::Bitcode
166        } else if need_bitcode_in_object(tcx) {
167            EmitObj::ObjectCode(BitcodeSection::Full)
168        } else {
169            EmitObj::ObjectCode(BitcodeSection::None)
170        };
171
172        ModuleConfig {
173            passes: if_regular!(sess.opts.cg.passes.clone(), vec![]),
174
175            opt_level: opt_level_and_size,
176            opt_size: opt_level_and_size,
177
178            pgo_gen: if_regular!(
179                sess.opts.cg.profile_generate.clone(),
180                SwitchWithOptPath::Disabled
181            ),
182            pgo_use: if_regular!(sess.opts.cg.profile_use.clone(), None),
183            pgo_sample_use: if_regular!(sess.opts.unstable_opts.profile_sample_use.clone(), None),
184            debug_info_for_profiling: sess.opts.unstable_opts.debug_info_for_profiling,
185            instrument_coverage: if_regular!(sess.instrument_coverage(), false),
186
187            sanitizer: if_regular!(sess.opts.unstable_opts.sanitizer, SanitizerSet::empty()),
188            sanitizer_dataflow_abilist: if_regular!(
189                sess.opts.unstable_opts.sanitizer_dataflow_abilist.clone(),
190                Vec::new()
191            ),
192            sanitizer_recover: if_regular!(
193                sess.opts.unstable_opts.sanitizer_recover,
194                SanitizerSet::empty()
195            ),
196            sanitizer_memory_track_origins: if_regular!(
197                sess.opts.unstable_opts.sanitizer_memory_track_origins,
198                0
199            ),
200
201            emit_pre_lto_bc: if_regular!(
202                save_temps || need_pre_lto_bitcode_for_incr_comp(sess),
203                false
204            ),
205            emit_no_opt_bc: if_regular!(save_temps, false),
206            emit_bc: if_regular!(
207                save_temps || sess.opts.output_types.contains_key(&OutputType::Bitcode),
208                save_temps
209            ),
210            emit_ir: if_regular!(
211                sess.opts.output_types.contains_key(&OutputType::LlvmAssembly),
212                false
213            ),
214            emit_asm: if_regular!(
215                sess.opts.output_types.contains_key(&OutputType::Assembly),
216                false
217            ),
218            emit_obj,
219            emit_thin_lto: sess.opts.unstable_opts.emit_thin_lto,
220            emit_thin_lto_summary: if_regular!(
221                sess.opts.output_types.contains_key(&OutputType::ThinLinkBitcode),
222                false
223            ),
224            bc_cmdline: sess.target.bitcode_llvm_cmdline.to_string(),
225
226            verify_llvm_ir: sess.verify_llvm_ir(),
227            lint_llvm_ir: sess.opts.unstable_opts.lint_llvm_ir,
228            no_prepopulate_passes: sess.opts.cg.no_prepopulate_passes,
229            no_builtins: no_builtins || sess.target.no_builtins,
230
231            // Exclude metadata and allocator modules from time_passes output,
232            // since they throw off the "LLVM passes" measurement.
233            time_module: if_regular!(true, false),
234
235            // Copy what clang does by turning on loop vectorization at O2 and
236            // slp vectorization at O3.
237            vectorize_loop: !sess.opts.cg.no_vectorize_loops
238                && (sess.opts.optimize == config::OptLevel::More
239                    || sess.opts.optimize == config::OptLevel::Aggressive),
240            vectorize_slp: !sess.opts.cg.no_vectorize_slp
241                && sess.opts.optimize == config::OptLevel::Aggressive,
242
243            // Some targets (namely, NVPTX) interact badly with the
244            // MergeFunctions pass. This is because MergeFunctions can generate
245            // new function calls which may interfere with the target calling
246            // convention; e.g. for the NVPTX target, PTX kernels should not
247            // call other PTX kernels. MergeFunctions can also be configured to
248            // generate aliases instead, but aliases are not supported by some
249            // backends (again, NVPTX). Therefore, allow targets to opt out of
250            // the MergeFunctions pass, but otherwise keep the pass enabled (at
251            // O2 and O3) since it can be useful for reducing code size.
252            merge_functions: match sess
253                .opts
254                .unstable_opts
255                .merge_functions
256                .unwrap_or(sess.target.merge_functions)
257            {
258                MergeFunctions::Disabled => false,
259                MergeFunctions::Trampolines | MergeFunctions::Aliases => {
260                    use config::OptLevel::*;
261                    match sess.opts.optimize {
262                        Aggressive | More | SizeMin | Size => true,
263                        Less | No => false,
264                    }
265                }
266            },
267
268            emit_lifetime_markers: sess.emit_lifetime_markers(),
269            llvm_plugins: if_regular!(sess.opts.unstable_opts.llvm_plugins.clone(), vec![]),
270            autodiff: if_regular!(sess.opts.unstable_opts.autodiff.clone(), vec![]),
271        }
272    }
273
274    pub fn bitcode_needed(&self) -> bool {
275        self.emit_bc
276            || self.emit_thin_lto_summary
277            || self.emit_obj == EmitObj::Bitcode
278            || self.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full)
279    }
280
281    pub fn embed_bitcode(&self) -> bool {
282        self.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full)
283    }
284}
285
286/// Configuration passed to the function returned by the `target_machine_factory`.
287pub struct TargetMachineFactoryConfig {
288    /// Split DWARF is enabled in LLVM by checking that `TM.MCOptions.SplitDwarfFile` isn't empty,
289    /// so the path to the dwarf object has to be provided when we create the target machine.
290    /// This can be ignored by backends which do not need it for their Split DWARF support.
291    pub split_dwarf_file: Option<PathBuf>,
292
293    /// The name of the output object file. Used for setting OutputFilenames in target options
294    /// so that LLVM can emit the CodeView S_OBJNAME record in pdb files
295    pub output_obj_file: Option<PathBuf>,
296}
297
298impl TargetMachineFactoryConfig {
299    pub fn new(
300        cgcx: &CodegenContext<impl WriteBackendMethods>,
301        module_name: &str,
302    ) -> TargetMachineFactoryConfig {
303        let split_dwarf_file = if cgcx.target_can_use_split_dwarf {
304            cgcx.output_filenames.split_dwarf_path(
305                cgcx.split_debuginfo,
306                cgcx.split_dwarf_kind,
307                module_name,
308                cgcx.invocation_temp.as_deref(),
309            )
310        } else {
311            None
312        };
313
314        let output_obj_file = Some(cgcx.output_filenames.temp_path_for_cgu(
315            OutputType::Object,
316            module_name,
317            cgcx.invocation_temp.as_deref(),
318        ));
319        TargetMachineFactoryConfig { split_dwarf_file, output_obj_file }
320    }
321}
322
323pub type TargetMachineFactoryFn<B> = Arc<
324    dyn Fn(
325            TargetMachineFactoryConfig,
326        ) -> Result<
327            <B as WriteBackendMethods>::TargetMachine,
328            <B as WriteBackendMethods>::TargetMachineError,
329        > + Send
330        + Sync,
331>;
332
333type ExportedSymbols = FxHashMap<CrateNum, Arc<Vec<(String, SymbolExportInfo)>>>;
334
335/// Additional resources used by optimize_and_codegen (not module specific)
336#[derive(Clone)]
337pub struct CodegenContext<B: WriteBackendMethods> {
338    // Resources needed when running LTO
339    pub prof: SelfProfilerRef,
340    pub lto: Lto,
341    pub save_temps: bool,
342    pub fewer_names: bool,
343    pub time_trace: bool,
344    pub exported_symbols: Option<Arc<ExportedSymbols>>,
345    pub opts: Arc<config::Options>,
346    pub crate_types: Vec<CrateType>,
347    pub each_linked_rlib_for_lto: Vec<(CrateNum, PathBuf)>,
348    pub output_filenames: Arc<OutputFilenames>,
349    pub invocation_temp: Option<String>,
350    pub regular_module_config: Arc<ModuleConfig>,
351    pub allocator_module_config: Arc<ModuleConfig>,
352    pub tm_factory: TargetMachineFactoryFn<B>,
353    pub msvc_imps_needed: bool,
354    pub is_pe_coff: bool,
355    pub target_can_use_split_dwarf: bool,
356    pub target_arch: String,
357    pub target_is_like_darwin: bool,
358    pub target_is_like_aix: bool,
359    pub split_debuginfo: rustc_target::spec::SplitDebuginfo,
360    pub split_dwarf_kind: rustc_session::config::SplitDwarfKind,
361    pub pointer_size: Size,
362
363    /// All commandline args used to invoke the compiler, with @file args fully expanded.
364    /// This will only be used within debug info, e.g. in the pdb file on windows
365    /// This is mainly useful for other tools that reads that debuginfo to figure out
366    /// how to call the compiler with the same arguments.
367    pub expanded_args: Vec<String>,
368
369    /// Emitter to use for diagnostics produced during codegen.
370    pub diag_emitter: SharedEmitter,
371    /// LLVM optimizations for which we want to print remarks.
372    pub remark: Passes,
373    /// Directory into which should the LLVM optimization remarks be written.
374    /// If `None`, they will be written to stderr.
375    pub remark_dir: Option<PathBuf>,
376    /// The incremental compilation session directory, or None if we are not
377    /// compiling incrementally
378    pub incr_comp_session_dir: Option<PathBuf>,
379    /// Channel back to the main control thread to send messages to
380    pub coordinator_send: Sender<Box<dyn Any + Send>>,
381    /// `true` if the codegen should be run in parallel.
382    ///
383    /// Depends on [`ExtraBackendMethods::supports_parallel()`] and `-Zno_parallel_backend`.
384    pub parallel: bool,
385}
386
387impl<B: WriteBackendMethods> CodegenContext<B> {
388    pub fn create_dcx(&self) -> DiagCtxt {
389        DiagCtxt::new(Box::new(self.diag_emitter.clone()))
390    }
391
392    pub fn config(&self, kind: ModuleKind) -> &ModuleConfig {
393        match kind {
394            ModuleKind::Regular => &self.regular_module_config,
395            ModuleKind::Allocator => &self.allocator_module_config,
396        }
397    }
398}
399
400fn generate_lto_work<B: ExtraBackendMethods>(
401    cgcx: &CodegenContext<B>,
402    autodiff: Vec<AutoDiffItem>,
403    needs_fat_lto: Vec<FatLtoInput<B>>,
404    needs_thin_lto: Vec<(String, B::ThinBuffer)>,
405    import_only_modules: Vec<(SerializedModule<B::ModuleBuffer>, WorkProduct)>,
406) -> Vec<(WorkItem<B>, u64)> {
407    let _prof_timer = cgcx.prof.generic_activity("codegen_generate_lto_work");
408
409    if !needs_fat_lto.is_empty() {
410        assert!(needs_thin_lto.is_empty());
411        let mut module =
412            B::run_fat_lto(cgcx, needs_fat_lto, import_only_modules).unwrap_or_else(|e| e.raise());
413        if cgcx.lto == Lto::Fat && !autodiff.is_empty() {
414            let config = cgcx.config(ModuleKind::Regular);
415            module = module.autodiff(cgcx, autodiff, config).unwrap_or_else(|e| e.raise());
416        }
417        // We are adding a single work item, so the cost doesn't matter.
418        vec![(WorkItem::LTO(module), 0)]
419    } else {
420        if !autodiff.is_empty() {
421            let dcx = cgcx.create_dcx();
422            dcx.handle().emit_fatal(AutodiffWithoutLto {});
423        }
424        assert!(needs_fat_lto.is_empty());
425        let (lto_modules, copy_jobs) = B::run_thin_lto(cgcx, needs_thin_lto, import_only_modules)
426            .unwrap_or_else(|e| e.raise());
427        lto_modules
428            .into_iter()
429            .map(|module| {
430                let cost = module.cost();
431                (WorkItem::LTO(module), cost)
432            })
433            .chain(copy_jobs.into_iter().map(|wp| {
434                (
435                    WorkItem::CopyPostLtoArtifacts(CachedModuleCodegen {
436                        name: wp.cgu_name.clone(),
437                        source: wp,
438                    }),
439                    0, // copying is very cheap
440                )
441            }))
442            .collect()
443    }
444}
445
446struct CompiledModules {
447    modules: Vec<CompiledModule>,
448    allocator_module: Option<CompiledModule>,
449}
450
451fn need_bitcode_in_object(tcx: TyCtxt<'_>) -> bool {
452    let sess = tcx.sess;
453    sess.opts.cg.embed_bitcode
454        && tcx.crate_types().contains(&CrateType::Rlib)
455        && sess.opts.output_types.contains_key(&OutputType::Exe)
456}
457
458fn need_pre_lto_bitcode_for_incr_comp(sess: &Session) -> bool {
459    if sess.opts.incremental.is_none() {
460        return false;
461    }
462
463    match sess.lto() {
464        Lto::No => false,
465        Lto::Fat | Lto::Thin | Lto::ThinLocal => true,
466    }
467}
468
469pub(crate) fn start_async_codegen<B: ExtraBackendMethods>(
470    backend: B,
471    tcx: TyCtxt<'_>,
472    target_cpu: String,
473) -> OngoingCodegen<B> {
474    let (coordinator_send, coordinator_receive) = channel();
475
476    let crate_attrs = tcx.hir_attrs(rustc_hir::CRATE_HIR_ID);
477    let no_builtins = attr::contains_name(crate_attrs, sym::no_builtins);
478
479    let crate_info = CrateInfo::new(tcx, target_cpu);
480
481    let regular_config = ModuleConfig::new(ModuleKind::Regular, tcx, no_builtins);
482    let allocator_config = ModuleConfig::new(ModuleKind::Allocator, tcx, no_builtins);
483
484    let (shared_emitter, shared_emitter_main) = SharedEmitter::new();
485    let (codegen_worker_send, codegen_worker_receive) = channel();
486
487    let coordinator_thread = start_executing_work(
488        backend.clone(),
489        tcx,
490        &crate_info,
491        shared_emitter,
492        codegen_worker_send,
493        coordinator_receive,
494        Arc::new(regular_config),
495        Arc::new(allocator_config),
496        coordinator_send.clone(),
497    );
498
499    OngoingCodegen {
500        backend,
501        crate_info,
502
503        codegen_worker_receive,
504        shared_emitter_main,
505        coordinator: Coordinator {
506            sender: coordinator_send,
507            future: Some(coordinator_thread),
508            phantom: PhantomData,
509        },
510        output_filenames: Arc::clone(tcx.output_filenames(())),
511    }
512}
513
514fn copy_all_cgu_workproducts_to_incr_comp_cache_dir(
515    sess: &Session,
516    compiled_modules: &CompiledModules,
517) -> FxIndexMap<WorkProductId, WorkProduct> {
518    let mut work_products = FxIndexMap::default();
519
520    if sess.opts.incremental.is_none() {
521        return work_products;
522    }
523
524    let _timer = sess.timer("copy_all_cgu_workproducts_to_incr_comp_cache_dir");
525
526    for module in compiled_modules.modules.iter().filter(|m| m.kind == ModuleKind::Regular) {
527        let mut files = Vec::new();
528        if let Some(object_file_path) = &module.object {
529            files.push((OutputType::Object.extension(), object_file_path.as_path()));
530        }
531        if let Some(dwarf_object_file_path) = &module.dwarf_object {
532            files.push(("dwo", dwarf_object_file_path.as_path()));
533        }
534        if let Some(path) = &module.assembly {
535            files.push((OutputType::Assembly.extension(), path.as_path()));
536        }
537        if let Some(path) = &module.llvm_ir {
538            files.push((OutputType::LlvmAssembly.extension(), path.as_path()));
539        }
540        if let Some(path) = &module.bytecode {
541            files.push((OutputType::Bitcode.extension(), path.as_path()));
542        }
543        if let Some((id, product)) = copy_cgu_workproduct_to_incr_comp_cache_dir(
544            sess,
545            &module.name,
546            files.as_slice(),
547            &module.links_from_incr_cache,
548        ) {
549            work_products.insert(id, product);
550        }
551    }
552
553    work_products
554}
555
556fn produce_final_output_artifacts(
557    sess: &Session,
558    compiled_modules: &CompiledModules,
559    crate_output: &OutputFilenames,
560) {
561    let mut user_wants_bitcode = false;
562    let mut user_wants_objects = false;
563
564    // Produce final compile outputs.
565    let copy_gracefully = |from: &Path, to: &OutFileName| match to {
566        OutFileName::Stdout if let Err(e) = copy_to_stdout(from) => {
567            sess.dcx().emit_err(errors::CopyPath::new(from, to.as_path(), e));
568        }
569        OutFileName::Real(path) if let Err(e) = fs::copy(from, path) => {
570            sess.dcx().emit_err(errors::CopyPath::new(from, path, e));
571        }
572        _ => {}
573    };
574
575    let copy_if_one_unit = |output_type: OutputType, keep_numbered: bool| {
576        if let [module] = &compiled_modules.modules[..] {
577            // 1) Only one codegen unit. In this case it's no difficulty
578            //    to copy `foo.0.x` to `foo.x`.
579            let path = crate_output.temp_path_for_cgu(
580                output_type,
581                &module.name,
582                sess.invocation_temp.as_deref(),
583            );
584            let output = crate_output.path(output_type);
585            if !output_type.is_text_output() && output.is_tty() {
586                sess.dcx()
587                    .emit_err(errors::BinaryOutputToTty { shorthand: output_type.shorthand() });
588            } else {
589                copy_gracefully(&path, &output);
590            }
591            if !sess.opts.cg.save_temps && !keep_numbered {
592                // The user just wants `foo.x`, not `foo.#module-name#.x`.
593                ensure_removed(sess.dcx(), &path);
594            }
595        } else {
596            if crate_output.outputs.contains_explicit_name(&output_type) {
597                // 2) Multiple codegen units, with `--emit foo=some_name`. We have
598                //    no good solution for this case, so warn the user.
599                sess.dcx()
600                    .emit_warn(errors::IgnoringEmitPath { extension: output_type.extension() });
601            } else if crate_output.single_output_file.is_some() {
602                // 3) Multiple codegen units, with `-o some_name`. We have
603                //    no good solution for this case, so warn the user.
604                sess.dcx().emit_warn(errors::IgnoringOutput { extension: output_type.extension() });
605            } else {
606                // 4) Multiple codegen units, but no explicit name. We
607                //    just leave the `foo.0.x` files in place.
608                // (We don't have to do any work in this case.)
609            }
610        }
611    };
612
613    // Flag to indicate whether the user explicitly requested bitcode.
614    // Otherwise, we produced it only as a temporary output, and will need
615    // to get rid of it.
616    for output_type in crate_output.outputs.keys() {
617        match *output_type {
618            OutputType::Bitcode => {
619                user_wants_bitcode = true;
620                // Copy to .bc, but always keep the .0.bc. There is a later
621                // check to figure out if we should delete .0.bc files, or keep
622                // them for making an rlib.
623                copy_if_one_unit(OutputType::Bitcode, true);
624            }
625            OutputType::ThinLinkBitcode => {
626                copy_if_one_unit(OutputType::ThinLinkBitcode, false);
627            }
628            OutputType::LlvmAssembly => {
629                copy_if_one_unit(OutputType::LlvmAssembly, false);
630            }
631            OutputType::Assembly => {
632                copy_if_one_unit(OutputType::Assembly, false);
633            }
634            OutputType::Object => {
635                user_wants_objects = true;
636                copy_if_one_unit(OutputType::Object, true);
637            }
638            OutputType::Mir | OutputType::Metadata | OutputType::Exe | OutputType::DepInfo => {}
639        }
640    }
641
642    // Clean up unwanted temporary files.
643
644    // We create the following files by default:
645    //  - #crate#.#module-name#.bc
646    //  - #crate#.#module-name#.o
647    //  - #crate#.crate.metadata.bc
648    //  - #crate#.crate.metadata.o
649    //  - #crate#.o (linked from crate.##.o)
650    //  - #crate#.bc (copied from crate.##.bc)
651    // We may create additional files if requested by the user (through
652    // `-C save-temps` or `--emit=` flags).
653
654    if !sess.opts.cg.save_temps {
655        // Remove the temporary .#module-name#.o objects. If the user didn't
656        // explicitly request bitcode (with --emit=bc), and the bitcode is not
657        // needed for building an rlib, then we must remove .#module-name#.bc as
658        // well.
659
660        // Specific rules for keeping .#module-name#.bc:
661        //  - If the user requested bitcode (`user_wants_bitcode`), and
662        //    codegen_units > 1, then keep it.
663        //  - If the user requested bitcode but codegen_units == 1, then we
664        //    can toss .#module-name#.bc because we copied it to .bc earlier.
665        //  - If we're not building an rlib and the user didn't request
666        //    bitcode, then delete .#module-name#.bc.
667        // If you change how this works, also update back::link::link_rlib,
668        // where .#module-name#.bc files are (maybe) deleted after making an
669        // rlib.
670        let needs_crate_object = crate_output.outputs.contains_key(&OutputType::Exe);
671
672        let keep_numbered_bitcode = user_wants_bitcode && sess.codegen_units().as_usize() > 1;
673
674        let keep_numbered_objects =
675            needs_crate_object || (user_wants_objects && sess.codegen_units().as_usize() > 1);
676
677        for module in compiled_modules.modules.iter() {
678            if !keep_numbered_objects {
679                if let Some(ref path) = module.object {
680                    ensure_removed(sess.dcx(), path);
681                }
682
683                if let Some(ref path) = module.dwarf_object {
684                    ensure_removed(sess.dcx(), path);
685                }
686            }
687
688            if let Some(ref path) = module.bytecode {
689                if !keep_numbered_bitcode {
690                    ensure_removed(sess.dcx(), path);
691                }
692            }
693        }
694
695        if !user_wants_bitcode
696            && let Some(ref allocator_module) = compiled_modules.allocator_module
697            && let Some(ref path) = allocator_module.bytecode
698        {
699            ensure_removed(sess.dcx(), path);
700        }
701    }
702
703    if sess.opts.json_artifact_notifications {
704        if let [module] = &compiled_modules.modules[..] {
705            module.for_each_output(|_path, ty| {
706                if sess.opts.output_types.contains_key(&ty) {
707                    let descr = ty.shorthand();
708                    // for single cgu file is renamed to drop cgu specific suffix
709                    // so we regenerate it the same way
710                    let path = crate_output.path(ty);
711                    sess.dcx().emit_artifact_notification(path.as_path(), descr);
712                }
713            });
714        } else {
715            for module in &compiled_modules.modules {
716                module.for_each_output(|path, ty| {
717                    if sess.opts.output_types.contains_key(&ty) {
718                        let descr = ty.shorthand();
719                        sess.dcx().emit_artifact_notification(&path, descr);
720                    }
721                });
722            }
723        }
724    }
725
726    // We leave the following files around by default:
727    //  - #crate#.o
728    //  - #crate#.crate.metadata.o
729    //  - #crate#.bc
730    // These are used in linking steps and will be cleaned up afterward.
731}
732
733pub(crate) enum WorkItem<B: WriteBackendMethods> {
734    /// Optimize a newly codegened, totally unoptimized module.
735    Optimize(ModuleCodegen<B::Module>),
736    /// Copy the post-LTO artifacts from the incremental cache to the output
737    /// directory.
738    CopyPostLtoArtifacts(CachedModuleCodegen),
739    /// Performs (Thin)LTO on the given module.
740    LTO(lto::LtoModuleCodegen<B>),
741}
742
743impl<B: WriteBackendMethods> WorkItem<B> {
744    fn module_kind(&self) -> ModuleKind {
745        match *self {
746            WorkItem::Optimize(ref m) => m.kind,
747            WorkItem::CopyPostLtoArtifacts(_) | WorkItem::LTO(_) => ModuleKind::Regular,
748        }
749    }
750
751    /// Generate a short description of this work item suitable for use as a thread name.
752    fn short_description(&self) -> String {
753        // `pthread_setname()` on *nix ignores anything beyond the first 15
754        // bytes. Use short descriptions to maximize the space available for
755        // the module name.
756        #[cfg(not(windows))]
757        fn desc(short: &str, _long: &str, name: &str) -> String {
758            // The short label is three bytes, and is followed by a space. That
759            // leaves 11 bytes for the CGU name. How we obtain those 11 bytes
760            // depends on the CGU name form.
761            //
762            // - Non-incremental, e.g. `regex.f10ba03eb5ec7975-cgu.0`: the part
763            //   before the `-cgu.0` is the same for every CGU, so use the
764            //   `cgu.0` part. The number suffix will be different for each
765            //   CGU.
766            //
767            // - Incremental (normal), e.g. `2i52vvl2hco29us0`: use the whole
768            //   name because each CGU will have a unique ASCII hash, and the
769            //   first 11 bytes will be enough to identify it.
770            //
771            // - Incremental (with `-Zhuman-readable-cgu-names`), e.g.
772            //   `regex.f10ba03eb5ec7975-re_builder.volatile`: use the whole
773            //   name. The first 11 bytes won't be enough to uniquely identify
774            //   it, but no obvious substring will, and this is a rarely used
775            //   option so it doesn't matter much.
776            //
777            assert_eq!(short.len(), 3);
778            let name = if let Some(index) = name.find("-cgu.") {
779                &name[index + 1..] // +1 skips the leading '-'.
780            } else {
781                name
782            };
783            format!("{short} {name}")
784        }
785
786        // Windows has no thread name length limit, so use more descriptive names.
787        #[cfg(windows)]
788        fn desc(_short: &str, long: &str, name: &str) -> String {
789            format!("{long} {name}")
790        }
791
792        match self {
793            WorkItem::Optimize(m) => desc("opt", "optimize module", &m.name),
794            WorkItem::CopyPostLtoArtifacts(m) => desc("cpy", "copy LTO artifacts for", &m.name),
795            WorkItem::LTO(m) => desc("lto", "LTO module", m.name()),
796        }
797    }
798}
799
800/// A result produced by the backend.
801pub(crate) enum WorkItemResult<B: WriteBackendMethods> {
802    /// The backend has finished compiling a CGU, nothing more required.
803    Finished(CompiledModule),
804
805    /// The backend has finished compiling a CGU, which now needs linking
806    /// because `-Zcombine-cgu` was specified.
807    NeedsLink(ModuleCodegen<B::Module>),
808
809    /// The backend has finished compiling a CGU, which now needs to go through
810    /// fat LTO.
811    NeedsFatLto(FatLtoInput<B>),
812
813    /// The backend has finished compiling a CGU, which now needs to go through
814    /// thin LTO.
815    NeedsThinLto(String, B::ThinBuffer),
816}
817
818pub enum FatLtoInput<B: WriteBackendMethods> {
819    Serialized { name: String, buffer: B::ModuleBuffer },
820    InMemory(ModuleCodegen<B::Module>),
821}
822
823/// Actual LTO type we end up choosing based on multiple factors.
824pub(crate) enum ComputedLtoType {
825    No,
826    Thin,
827    Fat,
828}
829
830pub(crate) fn compute_per_cgu_lto_type(
831    sess_lto: &Lto,
832    opts: &config::Options,
833    sess_crate_types: &[CrateType],
834    module_kind: ModuleKind,
835) -> ComputedLtoType {
836    // If the linker does LTO, we don't have to do it. Note that we
837    // keep doing full LTO, if it is requested, as not to break the
838    // assumption that the output will be a single module.
839    let linker_does_lto = opts.cg.linker_plugin_lto.enabled();
840
841    // When we're automatically doing ThinLTO for multi-codegen-unit
842    // builds we don't actually want to LTO the allocator modules if
843    // it shows up. This is due to various linker shenanigans that
844    // we'll encounter later.
845    let is_allocator = module_kind == ModuleKind::Allocator;
846
847    // We ignore a request for full crate graph LTO if the crate type
848    // is only an rlib, as there is no full crate graph to process,
849    // that'll happen later.
850    //
851    // This use case currently comes up primarily for targets that
852    // require LTO so the request for LTO is always unconditionally
853    // passed down to the backend, but we don't actually want to do
854    // anything about it yet until we've got a final product.
855    let is_rlib = matches!(sess_crate_types, [CrateType::Rlib]);
856
857    match sess_lto {
858        Lto::ThinLocal if !linker_does_lto && !is_allocator => ComputedLtoType::Thin,
859        Lto::Thin if !linker_does_lto && !is_rlib => ComputedLtoType::Thin,
860        Lto::Fat if !is_rlib => ComputedLtoType::Fat,
861        _ => ComputedLtoType::No,
862    }
863}
864
865fn execute_optimize_work_item<B: ExtraBackendMethods>(
866    cgcx: &CodegenContext<B>,
867    mut module: ModuleCodegen<B::Module>,
868    module_config: &ModuleConfig,
869) -> Result<WorkItemResult<B>, FatalError> {
870    let dcx = cgcx.create_dcx();
871    let dcx = dcx.handle();
872
873    B::optimize(cgcx, dcx, &mut module, module_config)?;
874
875    // After we've done the initial round of optimizations we need to
876    // decide whether to synchronously codegen this module or ship it
877    // back to the coordinator thread for further LTO processing (which
878    // has to wait for all the initial modules to be optimized).
879
880    let lto_type = compute_per_cgu_lto_type(&cgcx.lto, &cgcx.opts, &cgcx.crate_types, module.kind);
881
882    // If we're doing some form of incremental LTO then we need to be sure to
883    // save our module to disk first.
884    let bitcode = if cgcx.config(module.kind).emit_pre_lto_bc {
885        let filename = pre_lto_bitcode_filename(&module.name);
886        cgcx.incr_comp_session_dir.as_ref().map(|path| path.join(&filename))
887    } else {
888        None
889    };
890
891    match lto_type {
892        ComputedLtoType::No => finish_intra_module_work(cgcx, module, module_config),
893        ComputedLtoType::Thin => {
894            let (name, thin_buffer) = B::prepare_thin(module, false);
895            if let Some(path) = bitcode {
896                fs::write(&path, thin_buffer.data()).unwrap_or_else(|e| {
897                    panic!("Error writing pre-lto-bitcode file `{}`: {}", path.display(), e);
898                });
899            }
900            Ok(WorkItemResult::NeedsThinLto(name, thin_buffer))
901        }
902        ComputedLtoType::Fat => match bitcode {
903            Some(path) => {
904                let (name, buffer) = B::serialize_module(module);
905                fs::write(&path, buffer.data()).unwrap_or_else(|e| {
906                    panic!("Error writing pre-lto-bitcode file `{}`: {}", path.display(), e);
907                });
908                Ok(WorkItemResult::NeedsFatLto(FatLtoInput::Serialized { name, buffer }))
909            }
910            None => Ok(WorkItemResult::NeedsFatLto(FatLtoInput::InMemory(module))),
911        },
912    }
913}
914
915fn execute_copy_from_cache_work_item<B: ExtraBackendMethods>(
916    cgcx: &CodegenContext<B>,
917    module: CachedModuleCodegen,
918    module_config: &ModuleConfig,
919) -> WorkItemResult<B> {
920    let incr_comp_session_dir = cgcx.incr_comp_session_dir.as_ref().unwrap();
921
922    let mut links_from_incr_cache = Vec::new();
923
924    let mut load_from_incr_comp_dir = |output_path: PathBuf, saved_path: &str| {
925        let source_file = in_incr_comp_dir(incr_comp_session_dir, saved_path);
926        debug!(
927            "copying preexisting module `{}` from {:?} to {}",
928            module.name,
929            source_file,
930            output_path.display()
931        );
932        match link_or_copy(&source_file, &output_path) {
933            Ok(_) => {
934                links_from_incr_cache.push(source_file);
935                Some(output_path)
936            }
937            Err(error) => {
938                cgcx.create_dcx().handle().emit_err(errors::CopyPathBuf {
939                    source_file,
940                    output_path,
941                    error,
942                });
943                None
944            }
945        }
946    };
947
948    let dwarf_object =
949        module.source.saved_files.get("dwo").as_ref().and_then(|saved_dwarf_object_file| {
950            let dwarf_obj_out = cgcx
951                .output_filenames
952                .split_dwarf_path(
953                    cgcx.split_debuginfo,
954                    cgcx.split_dwarf_kind,
955                    &module.name,
956                    cgcx.invocation_temp.as_deref(),
957                )
958                .expect(
959                    "saved dwarf object in work product but `split_dwarf_path` returned `None`",
960                );
961            load_from_incr_comp_dir(dwarf_obj_out, saved_dwarf_object_file)
962        });
963
964    let mut load_from_incr_cache = |perform, output_type: OutputType| {
965        if perform {
966            let saved_file = module.source.saved_files.get(output_type.extension())?;
967            let output_path = cgcx.output_filenames.temp_path_for_cgu(
968                output_type,
969                &module.name,
970                cgcx.invocation_temp.as_deref(),
971            );
972            load_from_incr_comp_dir(output_path, &saved_file)
973        } else {
974            None
975        }
976    };
977
978    let should_emit_obj = module_config.emit_obj != EmitObj::None;
979    let assembly = load_from_incr_cache(module_config.emit_asm, OutputType::Assembly);
980    let llvm_ir = load_from_incr_cache(module_config.emit_ir, OutputType::LlvmAssembly);
981    let bytecode = load_from_incr_cache(module_config.emit_bc, OutputType::Bitcode);
982    let object = load_from_incr_cache(should_emit_obj, OutputType::Object);
983    if should_emit_obj && object.is_none() {
984        cgcx.create_dcx().handle().emit_fatal(errors::NoSavedObjectFile { cgu_name: &module.name })
985    }
986
987    WorkItemResult::Finished(CompiledModule {
988        links_from_incr_cache,
989        name: module.name,
990        kind: ModuleKind::Regular,
991        object,
992        dwarf_object,
993        bytecode,
994        assembly,
995        llvm_ir,
996    })
997}
998
999fn execute_lto_work_item<B: ExtraBackendMethods>(
1000    cgcx: &CodegenContext<B>,
1001    module: lto::LtoModuleCodegen<B>,
1002    module_config: &ModuleConfig,
1003) -> Result<WorkItemResult<B>, FatalError> {
1004    let module = module.optimize(cgcx)?;
1005    finish_intra_module_work(cgcx, module, module_config)
1006}
1007
1008fn finish_intra_module_work<B: ExtraBackendMethods>(
1009    cgcx: &CodegenContext<B>,
1010    module: ModuleCodegen<B::Module>,
1011    module_config: &ModuleConfig,
1012) -> Result<WorkItemResult<B>, FatalError> {
1013    let dcx = cgcx.create_dcx();
1014    let dcx = dcx.handle();
1015
1016    if !cgcx.opts.unstable_opts.combine_cgu || module.kind == ModuleKind::Allocator {
1017        let module = B::codegen(cgcx, dcx, module, module_config)?;
1018        Ok(WorkItemResult::Finished(module))
1019    } else {
1020        Ok(WorkItemResult::NeedsLink(module))
1021    }
1022}
1023
1024/// Messages sent to the coordinator.
1025pub(crate) enum Message<B: WriteBackendMethods> {
1026    /// A jobserver token has become available. Sent from the jobserver helper
1027    /// thread.
1028    Token(io::Result<Acquired>),
1029
1030    /// The backend has finished processing a work item for a codegen unit.
1031    /// Sent from a backend worker thread.
1032    WorkItem { result: Result<WorkItemResult<B>, Option<WorkerFatalError>>, worker_id: usize },
1033
1034    /// A vector containing all the AutoDiff tasks that we have to pass to Enzyme.
1035    AddAutoDiffItems(Vec<AutoDiffItem>),
1036
1037    /// The frontend has finished generating something (backend IR or a
1038    /// post-LTO artifact) for a codegen unit, and it should be passed to the
1039    /// backend. Sent from the main thread.
1040    CodegenDone { llvm_work_item: WorkItem<B>, cost: u64 },
1041
1042    /// Similar to `CodegenDone`, but for reusing a pre-LTO artifact
1043    /// Sent from the main thread.
1044    AddImportOnlyModule {
1045        module_data: SerializedModule<B::ModuleBuffer>,
1046        work_product: WorkProduct,
1047    },
1048
1049    /// The frontend has finished generating everything for all codegen units.
1050    /// Sent from the main thread.
1051    CodegenComplete,
1052
1053    /// Some normal-ish compiler error occurred, and codegen should be wound
1054    /// down. Sent from the main thread.
1055    CodegenAborted,
1056}
1057
1058/// A message sent from the coordinator thread to the main thread telling it to
1059/// process another codegen unit.
1060pub struct CguMessage;
1061
1062// A cut-down version of `rustc_errors::DiagInner` that impls `Send`, which
1063// can be used to send diagnostics from codegen threads to the main thread.
1064// It's missing the following fields from `rustc_errors::DiagInner`.
1065// - `span`: it doesn't impl `Send`.
1066// - `suggestions`: it doesn't impl `Send`, and isn't used for codegen
1067//   diagnostics.
1068// - `sort_span`: it doesn't impl `Send`.
1069// - `is_lint`: lints aren't relevant during codegen.
1070// - `emitted_at`: not used for codegen diagnostics.
1071struct Diagnostic {
1072    level: Level,
1073    messages: Vec<(DiagMessage, Style)>,
1074    code: Option<ErrCode>,
1075    children: Vec<Subdiagnostic>,
1076    args: DiagArgMap,
1077}
1078
1079// A cut-down version of `rustc_errors::Subdiag` that impls `Send`. It's
1080// missing the following fields from `rustc_errors::Subdiag`.
1081// - `span`: it doesn't impl `Send`.
1082pub(crate) struct Subdiagnostic {
1083    level: Level,
1084    messages: Vec<(DiagMessage, Style)>,
1085}
1086
1087#[derive(PartialEq, Clone, Copy, Debug)]
1088enum MainThreadState {
1089    /// Doing nothing.
1090    Idle,
1091
1092    /// Doing codegen, i.e. MIR-to-LLVM-IR conversion.
1093    Codegenning,
1094
1095    /// Idle, but lending the compiler process's Token to an LLVM thread so it can do useful work.
1096    Lending,
1097}
1098
1099fn start_executing_work<B: ExtraBackendMethods>(
1100    backend: B,
1101    tcx: TyCtxt<'_>,
1102    crate_info: &CrateInfo,
1103    shared_emitter: SharedEmitter,
1104    codegen_worker_send: Sender<CguMessage>,
1105    coordinator_receive: Receiver<Box<dyn Any + Send>>,
1106    regular_config: Arc<ModuleConfig>,
1107    allocator_config: Arc<ModuleConfig>,
1108    tx_to_llvm_workers: Sender<Box<dyn Any + Send>>,
1109) -> thread::JoinHandle<Result<CompiledModules, ()>> {
1110    let coordinator_send = tx_to_llvm_workers;
1111    let sess = tcx.sess;
1112
1113    let mut each_linked_rlib_for_lto = Vec::new();
1114    drop(link::each_linked_rlib(crate_info, None, &mut |cnum, path| {
1115        if link::ignored_for_lto(sess, crate_info, cnum) {
1116            return;
1117        }
1118        each_linked_rlib_for_lto.push((cnum, path.to_path_buf()));
1119    }));
1120
1121    // Compute the set of symbols we need to retain when doing LTO (if we need to)
1122    let exported_symbols = {
1123        let mut exported_symbols = FxHashMap::default();
1124
1125        let copy_symbols = |cnum| {
1126            let symbols = tcx
1127                .exported_symbols(cnum)
1128                .iter()
1129                .map(|&(s, lvl)| (symbol_name_for_instance_in_crate(tcx, s, cnum), lvl))
1130                .collect();
1131            Arc::new(symbols)
1132        };
1133
1134        match sess.lto() {
1135            Lto::No => None,
1136            Lto::ThinLocal => {
1137                exported_symbols.insert(LOCAL_CRATE, copy_symbols(LOCAL_CRATE));
1138                Some(Arc::new(exported_symbols))
1139            }
1140            Lto::Fat | Lto::Thin => {
1141                exported_symbols.insert(LOCAL_CRATE, copy_symbols(LOCAL_CRATE));
1142                for &(cnum, ref _path) in &each_linked_rlib_for_lto {
1143                    exported_symbols.insert(cnum, copy_symbols(cnum));
1144                }
1145                Some(Arc::new(exported_symbols))
1146            }
1147        }
1148    };
1149
1150    // First up, convert our jobserver into a helper thread so we can use normal
1151    // mpsc channels to manage our messages and such.
1152    // After we've requested tokens then we'll, when we can,
1153    // get tokens on `coordinator_receive` which will
1154    // get managed in the main loop below.
1155    let coordinator_send2 = coordinator_send.clone();
1156    let helper = jobserver::client()
1157        .into_helper_thread(move |token| {
1158            drop(coordinator_send2.send(Box::new(Message::Token::<B>(token))));
1159        })
1160        .expect("failed to spawn helper thread");
1161
1162    let ol =
1163        if tcx.sess.opts.unstable_opts.no_codegen || !tcx.sess.opts.output_types.should_codegen() {
1164            // If we know that we won’t be doing codegen, create target machines without optimisation.
1165            config::OptLevel::No
1166        } else {
1167            tcx.backend_optimization_level(())
1168        };
1169    let backend_features = tcx.global_backend_features(());
1170
1171    let remark_dir = if let Some(ref dir) = sess.opts.unstable_opts.remark_dir {
1172        let result = fs::create_dir_all(dir).and_then(|_| dir.canonicalize());
1173        match result {
1174            Ok(dir) => Some(dir),
1175            Err(error) => sess.dcx().emit_fatal(ErrorCreatingRemarkDir { error }),
1176        }
1177    } else {
1178        None
1179    };
1180
1181    let cgcx = CodegenContext::<B> {
1182        crate_types: tcx.crate_types().to_vec(),
1183        each_linked_rlib_for_lto,
1184        lto: sess.lto(),
1185        fewer_names: sess.fewer_names(),
1186        save_temps: sess.opts.cg.save_temps,
1187        time_trace: sess.opts.unstable_opts.llvm_time_trace,
1188        opts: Arc::new(sess.opts.clone()),
1189        prof: sess.prof.clone(),
1190        exported_symbols,
1191        remark: sess.opts.cg.remark.clone(),
1192        remark_dir,
1193        incr_comp_session_dir: sess.incr_comp_session_dir_opt().map(|r| r.clone()),
1194        coordinator_send,
1195        expanded_args: tcx.sess.expanded_args.clone(),
1196        diag_emitter: shared_emitter.clone(),
1197        output_filenames: Arc::clone(tcx.output_filenames(())),
1198        regular_module_config: regular_config,
1199        allocator_module_config: allocator_config,
1200        tm_factory: backend.target_machine_factory(tcx.sess, ol, backend_features),
1201        msvc_imps_needed: msvc_imps_needed(tcx),
1202        is_pe_coff: tcx.sess.target.is_like_windows,
1203        target_can_use_split_dwarf: tcx.sess.target_can_use_split_dwarf(),
1204        target_arch: tcx.sess.target.arch.to_string(),
1205        target_is_like_darwin: tcx.sess.target.is_like_darwin,
1206        target_is_like_aix: tcx.sess.target.is_like_aix,
1207        split_debuginfo: tcx.sess.split_debuginfo(),
1208        split_dwarf_kind: tcx.sess.opts.unstable_opts.split_dwarf_kind,
1209        parallel: backend.supports_parallel() && !sess.opts.unstable_opts.no_parallel_backend,
1210        pointer_size: tcx.data_layout.pointer_size,
1211        invocation_temp: sess.invocation_temp.clone(),
1212    };
1213
1214    // This is the "main loop" of parallel work happening for parallel codegen.
1215    // It's here that we manage parallelism, schedule work, and work with
1216    // messages coming from clients.
1217    //
1218    // There are a few environmental pre-conditions that shape how the system
1219    // is set up:
1220    //
1221    // - Error reporting can only happen on the main thread because that's the
1222    //   only place where we have access to the compiler `Session`.
1223    // - LLVM work can be done on any thread.
1224    // - Codegen can only happen on the main thread.
1225    // - Each thread doing substantial work must be in possession of a `Token`
1226    //   from the `Jobserver`.
1227    // - The compiler process always holds one `Token`. Any additional `Tokens`
1228    //   have to be requested from the `Jobserver`.
1229    //
1230    // Error Reporting
1231    // ===============
1232    // The error reporting restriction is handled separately from the rest: We
1233    // set up a `SharedEmitter` that holds an open channel to the main thread.
1234    // When an error occurs on any thread, the shared emitter will send the
1235    // error message to the receiver main thread (`SharedEmitterMain`). The
1236    // main thread will periodically query this error message queue and emit
1237    // any error messages it has received. It might even abort compilation if
1238    // it has received a fatal error. In this case we rely on all other threads
1239    // being torn down automatically with the main thread.
1240    // Since the main thread will often be busy doing codegen work, error
1241    // reporting will be somewhat delayed, since the message queue can only be
1242    // checked in between two work packages.
1243    //
1244    // Work Processing Infrastructure
1245    // ==============================
1246    // The work processing infrastructure knows three major actors:
1247    //
1248    // - the coordinator thread,
1249    // - the main thread, and
1250    // - LLVM worker threads
1251    //
1252    // The coordinator thread is running a message loop. It instructs the main
1253    // thread about what work to do when, and it will spawn off LLVM worker
1254    // threads as open LLVM WorkItems become available.
1255    //
1256    // The job of the main thread is to codegen CGUs into LLVM work packages
1257    // (since the main thread is the only thread that can do this). The main
1258    // thread will block until it receives a message from the coordinator, upon
1259    // which it will codegen one CGU, send it to the coordinator and block
1260    // again. This way the coordinator can control what the main thread is
1261    // doing.
1262    //
1263    // The coordinator keeps a queue of LLVM WorkItems, and when a `Token` is
1264    // available, it will spawn off a new LLVM worker thread and let it process
1265    // a WorkItem. When a LLVM worker thread is done with its WorkItem,
1266    // it will just shut down, which also frees all resources associated with
1267    // the given LLVM module, and sends a message to the coordinator that the
1268    // WorkItem has been completed.
1269    //
1270    // Work Scheduling
1271    // ===============
1272    // The scheduler's goal is to minimize the time it takes to complete all
1273    // work there is, however, we also want to keep memory consumption low
1274    // if possible. These two goals are at odds with each other: If memory
1275    // consumption were not an issue, we could just let the main thread produce
1276    // LLVM WorkItems at full speed, assuring maximal utilization of
1277    // Tokens/LLVM worker threads. However, since codegen is usually faster
1278    // than LLVM processing, the queue of LLVM WorkItems would fill up and each
1279    // WorkItem potentially holds on to a substantial amount of memory.
1280    //
1281    // So the actual goal is to always produce just enough LLVM WorkItems as
1282    // not to starve our LLVM worker threads. That means, once we have enough
1283    // WorkItems in our queue, we can block the main thread, so it does not
1284    // produce more until we need them.
1285    //
1286    // Doing LLVM Work on the Main Thread
1287    // ----------------------------------
1288    // Since the main thread owns the compiler process's implicit `Token`, it is
1289    // wasteful to keep it blocked without doing any work. Therefore, what we do
1290    // in this case is: We spawn off an additional LLVM worker thread that helps
1291    // reduce the queue. The work it is doing corresponds to the implicit
1292    // `Token`. The coordinator will mark the main thread as being busy with
1293    // LLVM work. (The actual work happens on another OS thread but we just care
1294    // about `Tokens`, not actual threads).
1295    //
1296    // When any LLVM worker thread finishes while the main thread is marked as
1297    // "busy with LLVM work", we can do a little switcheroo: We give the Token
1298    // of the just finished thread to the LLVM worker thread that is working on
1299    // behalf of the main thread's implicit Token, thus freeing up the main
1300    // thread again. The coordinator can then again decide what the main thread
1301    // should do. This allows the coordinator to make decisions at more points
1302    // in time.
1303    //
1304    // Striking a Balance between Throughput and Memory Consumption
1305    // ------------------------------------------------------------
1306    // Since our two goals, (1) use as many Tokens as possible and (2) keep
1307    // memory consumption as low as possible, are in conflict with each other,
1308    // we have to find a trade off between them. Right now, the goal is to keep
1309    // all workers busy, which means that no worker should find the queue empty
1310    // when it is ready to start.
1311    // How do we do achieve this? Good question :) We actually never know how
1312    // many `Tokens` are potentially available so it's hard to say how much to
1313    // fill up the queue before switching the main thread to LLVM work. Also we
1314    // currently don't have a means to estimate how long a running LLVM worker
1315    // will still be busy with it's current WorkItem. However, we know the
1316    // maximal count of available Tokens that makes sense (=the number of CPU
1317    // cores), so we can take a conservative guess. The heuristic we use here
1318    // is implemented in the `queue_full_enough()` function.
1319    //
1320    // Some Background on Jobservers
1321    // -----------------------------
1322    // It's worth also touching on the management of parallelism here. We don't
1323    // want to just spawn a thread per work item because while that's optimal
1324    // parallelism it may overload a system with too many threads or violate our
1325    // configuration for the maximum amount of cpu to use for this process. To
1326    // manage this we use the `jobserver` crate.
1327    //
1328    // Job servers are an artifact of GNU make and are used to manage
1329    // parallelism between processes. A jobserver is a glorified IPC semaphore
1330    // basically. Whenever we want to run some work we acquire the semaphore,
1331    // and whenever we're done with that work we release the semaphore. In this
1332    // manner we can ensure that the maximum number of parallel workers is
1333    // capped at any one point in time.
1334    //
1335    // LTO and the coordinator thread
1336    // ------------------------------
1337    //
1338    // The final job the coordinator thread is responsible for is managing LTO
1339    // and how that works. When LTO is requested what we'll do is collect all
1340    // optimized LLVM modules into a local vector on the coordinator. Once all
1341    // modules have been codegened and optimized we hand this to the `lto`
1342    // module for further optimization. The `lto` module will return back a list
1343    // of more modules to work on, which the coordinator will continue to spawn
1344    // work for.
1345    //
1346    // Each LLVM module is automatically sent back to the coordinator for LTO if
1347    // necessary. There's already optimizations in place to avoid sending work
1348    // back to the coordinator if LTO isn't requested.
1349    return B::spawn_named_thread(cgcx.time_trace, "coordinator".to_string(), move || {
1350        let mut worker_id_counter = 0;
1351        let mut free_worker_ids = Vec::new();
1352        let mut get_worker_id = |free_worker_ids: &mut Vec<usize>| {
1353            if let Some(id) = free_worker_ids.pop() {
1354                id
1355            } else {
1356                let id = worker_id_counter;
1357                worker_id_counter += 1;
1358                id
1359            }
1360        };
1361
1362        // This is where we collect codegen units that have gone all the way
1363        // through codegen and LLVM.
1364        let mut autodiff_items = Vec::new();
1365        let mut compiled_modules = vec![];
1366        let mut compiled_allocator_module = None;
1367        let mut needs_link = Vec::new();
1368        let mut needs_fat_lto = Vec::new();
1369        let mut needs_thin_lto = Vec::new();
1370        let mut lto_import_only_modules = Vec::new();
1371        let mut started_lto = false;
1372
1373        /// Possible state transitions:
1374        /// - Ongoing -> Completed
1375        /// - Ongoing -> Aborted
1376        /// - Completed -> Aborted
1377        #[derive(Debug, PartialEq)]
1378        enum CodegenState {
1379            Ongoing,
1380            Completed,
1381            Aborted,
1382        }
1383        use CodegenState::*;
1384        let mut codegen_state = Ongoing;
1385
1386        // This is the queue of LLVM work items that still need processing.
1387        let mut work_items = Vec::<(WorkItem<B>, u64)>::new();
1388
1389        // This are the Jobserver Tokens we currently hold. Does not include
1390        // the implicit Token the compiler process owns no matter what.
1391        let mut tokens = Vec::new();
1392
1393        let mut main_thread_state = MainThreadState::Idle;
1394
1395        // How many LLVM worker threads are running while holding a Token. This
1396        // *excludes* any that the main thread is lending a Token to.
1397        let mut running_with_own_token = 0;
1398
1399        // How many LLVM worker threads are running in total. This *includes*
1400        // any that the main thread is lending a Token to.
1401        let running_with_any_token = |main_thread_state, running_with_own_token| {
1402            running_with_own_token
1403                + if main_thread_state == MainThreadState::Lending { 1 } else { 0 }
1404        };
1405
1406        let mut llvm_start_time: Option<VerboseTimingGuard<'_>> = None;
1407
1408        // Run the message loop while there's still anything that needs message
1409        // processing. Note that as soon as codegen is aborted we simply want to
1410        // wait for all existing work to finish, so many of the conditions here
1411        // only apply if codegen hasn't been aborted as they represent pending
1412        // work to be done.
1413        loop {
1414            // While there are still CGUs to be codegened, the coordinator has
1415            // to decide how to utilize the compiler processes implicit Token:
1416            // For codegenning more CGU or for running them through LLVM.
1417            if codegen_state == Ongoing {
1418                if main_thread_state == MainThreadState::Idle {
1419                    // Compute the number of workers that will be running once we've taken as many
1420                    // items from the work queue as we can, plus one for the main thread. It's not
1421                    // critically important that we use this instead of just
1422                    // `running_with_own_token`, but it prevents the `queue_full_enough` heuristic
1423                    // from fluctuating just because a worker finished up and we decreased the
1424                    // `running_with_own_token` count, even though we're just going to increase it
1425                    // right after this when we put a new worker to work.
1426                    let extra_tokens = tokens.len().checked_sub(running_with_own_token).unwrap();
1427                    let additional_running = std::cmp::min(extra_tokens, work_items.len());
1428                    let anticipated_running = running_with_own_token + additional_running + 1;
1429
1430                    if !queue_full_enough(work_items.len(), anticipated_running) {
1431                        // The queue is not full enough, process more codegen units:
1432                        if codegen_worker_send.send(CguMessage).is_err() {
1433                            panic!("Could not send CguMessage to main thread")
1434                        }
1435                        main_thread_state = MainThreadState::Codegenning;
1436                    } else {
1437                        // The queue is full enough to not let the worker
1438                        // threads starve. Use the implicit Token to do some
1439                        // LLVM work too.
1440                        let (item, _) =
1441                            work_items.pop().expect("queue empty - queue_full_enough() broken?");
1442                        main_thread_state = MainThreadState::Lending;
1443                        spawn_work(
1444                            &cgcx,
1445                            &mut llvm_start_time,
1446                            get_worker_id(&mut free_worker_ids),
1447                            item,
1448                        );
1449                    }
1450                }
1451            } else if codegen_state == Completed {
1452                if running_with_any_token(main_thread_state, running_with_own_token) == 0
1453                    && work_items.is_empty()
1454                {
1455                    // All codegen work is done. Do we have LTO work to do?
1456                    if needs_fat_lto.is_empty()
1457                        && needs_thin_lto.is_empty()
1458                        && lto_import_only_modules.is_empty()
1459                    {
1460                        // Nothing more to do!
1461                        break;
1462                    }
1463
1464                    // We have LTO work to do. Perform the serial work here of
1465                    // figuring out what we're going to LTO and then push a
1466                    // bunch of work items onto our queue to do LTO. This all
1467                    // happens on the coordinator thread but it's very quick so
1468                    // we don't worry about tokens.
1469                    assert!(!started_lto);
1470                    started_lto = true;
1471
1472                    let needs_fat_lto = mem::take(&mut needs_fat_lto);
1473                    let needs_thin_lto = mem::take(&mut needs_thin_lto);
1474                    let import_only_modules = mem::take(&mut lto_import_only_modules);
1475
1476                    for (work, cost) in generate_lto_work(
1477                        &cgcx,
1478                        autodiff_items.clone(),
1479                        needs_fat_lto,
1480                        needs_thin_lto,
1481                        import_only_modules,
1482                    ) {
1483                        let insertion_index = work_items
1484                            .binary_search_by_key(&cost, |&(_, cost)| cost)
1485                            .unwrap_or_else(|e| e);
1486                        work_items.insert(insertion_index, (work, cost));
1487                        if cgcx.parallel {
1488                            helper.request_token();
1489                        }
1490                    }
1491                }
1492
1493                // In this branch, we know that everything has been codegened,
1494                // so it's just a matter of determining whether the implicit
1495                // Token is free to use for LLVM work.
1496                match main_thread_state {
1497                    MainThreadState::Idle => {
1498                        if let Some((item, _)) = work_items.pop() {
1499                            main_thread_state = MainThreadState::Lending;
1500                            spawn_work(
1501                                &cgcx,
1502                                &mut llvm_start_time,
1503                                get_worker_id(&mut free_worker_ids),
1504                                item,
1505                            );
1506                        } else {
1507                            // There is no unstarted work, so let the main thread
1508                            // take over for a running worker. Otherwise the
1509                            // implicit token would just go to waste.
1510                            // We reduce the `running` counter by one. The
1511                            // `tokens.truncate()` below will take care of
1512                            // giving the Token back.
1513                            assert!(running_with_own_token > 0);
1514                            running_with_own_token -= 1;
1515                            main_thread_state = MainThreadState::Lending;
1516                        }
1517                    }
1518                    MainThreadState::Codegenning => bug!(
1519                        "codegen worker should not be codegenning after \
1520                              codegen was already completed"
1521                    ),
1522                    MainThreadState::Lending => {
1523                        // Already making good use of that token
1524                    }
1525                }
1526            } else {
1527                // Don't queue up any more work if codegen was aborted, we're
1528                // just waiting for our existing children to finish.
1529                assert!(codegen_state == Aborted);
1530                if running_with_any_token(main_thread_state, running_with_own_token) == 0 {
1531                    break;
1532                }
1533            }
1534
1535            // Spin up what work we can, only doing this while we've got available
1536            // parallelism slots and work left to spawn.
1537            if codegen_state != Aborted {
1538                while running_with_own_token < tokens.len()
1539                    && let Some((item, _)) = work_items.pop()
1540                {
1541                    spawn_work(
1542                        &cgcx,
1543                        &mut llvm_start_time,
1544                        get_worker_id(&mut free_worker_ids),
1545                        item,
1546                    );
1547                    running_with_own_token += 1;
1548                }
1549            }
1550
1551            // Relinquish accidentally acquired extra tokens.
1552            tokens.truncate(running_with_own_token);
1553
1554            // If a thread exits successfully then we drop a token associated
1555            // with that worker and update our `running_with_own_token` count.
1556            // We may later re-acquire a token to continue running more work.
1557            // We may also not actually drop a token here if the worker was
1558            // running with an "ephemeral token".
1559            let mut free_worker = |worker_id| {
1560                if main_thread_state == MainThreadState::Lending {
1561                    main_thread_state = MainThreadState::Idle;
1562                } else {
1563                    running_with_own_token -= 1;
1564                }
1565
1566                free_worker_ids.push(worker_id);
1567            };
1568
1569            let msg = coordinator_receive.recv().unwrap();
1570            match *msg.downcast::<Message<B>>().ok().unwrap() {
1571                // Save the token locally and the next turn of the loop will use
1572                // this to spawn a new unit of work, or it may get dropped
1573                // immediately if we have no more work to spawn.
1574                Message::Token(token) => {
1575                    match token {
1576                        Ok(token) => {
1577                            tokens.push(token);
1578
1579                            if main_thread_state == MainThreadState::Lending {
1580                                // If the main thread token is used for LLVM work
1581                                // at the moment, we turn that thread into a regular
1582                                // LLVM worker thread, so the main thread is free
1583                                // to react to codegen demand.
1584                                main_thread_state = MainThreadState::Idle;
1585                                running_with_own_token += 1;
1586                            }
1587                        }
1588                        Err(e) => {
1589                            let msg = &format!("failed to acquire jobserver token: {e}");
1590                            shared_emitter.fatal(msg);
1591                            codegen_state = Aborted;
1592                        }
1593                    }
1594                }
1595
1596                Message::CodegenDone { llvm_work_item, cost } => {
1597                    // We keep the queue sorted by estimated processing cost,
1598                    // so that more expensive items are processed earlier. This
1599                    // is good for throughput as it gives the main thread more
1600                    // time to fill up the queue and it avoids scheduling
1601                    // expensive items to the end.
1602                    // Note, however, that this is not ideal for memory
1603                    // consumption, as LLVM module sizes are not evenly
1604                    // distributed.
1605                    let insertion_index = work_items.binary_search_by_key(&cost, |&(_, cost)| cost);
1606                    let insertion_index = match insertion_index {
1607                        Ok(idx) | Err(idx) => idx,
1608                    };
1609                    work_items.insert(insertion_index, (llvm_work_item, cost));
1610
1611                    if cgcx.parallel {
1612                        helper.request_token();
1613                    }
1614                    assert_eq!(main_thread_state, MainThreadState::Codegenning);
1615                    main_thread_state = MainThreadState::Idle;
1616                }
1617
1618                Message::AddAutoDiffItems(mut items) => {
1619                    autodiff_items.append(&mut items);
1620                }
1621
1622                Message::CodegenComplete => {
1623                    if codegen_state != Aborted {
1624                        codegen_state = Completed;
1625                    }
1626                    assert_eq!(main_thread_state, MainThreadState::Codegenning);
1627                    main_thread_state = MainThreadState::Idle;
1628                }
1629
1630                // If codegen is aborted that means translation was aborted due
1631                // to some normal-ish compiler error. In this situation we want
1632                // to exit as soon as possible, but we want to make sure all
1633                // existing work has finished. Flag codegen as being done, and
1634                // then conditions above will ensure no more work is spawned but
1635                // we'll keep executing this loop until `running_with_own_token`
1636                // hits 0.
1637                Message::CodegenAborted => {
1638                    codegen_state = Aborted;
1639                }
1640
1641                Message::WorkItem { result, worker_id } => {
1642                    free_worker(worker_id);
1643
1644                    match result {
1645                        Ok(WorkItemResult::Finished(compiled_module)) => {
1646                            match compiled_module.kind {
1647                                ModuleKind::Regular => {
1648                                    assert!(needs_link.is_empty());
1649                                    compiled_modules.push(compiled_module);
1650                                }
1651                                ModuleKind::Allocator => {
1652                                    assert!(compiled_allocator_module.is_none());
1653                                    compiled_allocator_module = Some(compiled_module);
1654                                }
1655                            }
1656                        }
1657                        Ok(WorkItemResult::NeedsLink(module)) => {
1658                            assert!(compiled_modules.is_empty());
1659                            needs_link.push(module);
1660                        }
1661                        Ok(WorkItemResult::NeedsFatLto(fat_lto_input)) => {
1662                            assert!(!started_lto);
1663                            assert!(needs_thin_lto.is_empty());
1664                            needs_fat_lto.push(fat_lto_input);
1665                        }
1666                        Ok(WorkItemResult::NeedsThinLto(name, thin_buffer)) => {
1667                            assert!(!started_lto);
1668                            assert!(needs_fat_lto.is_empty());
1669                            needs_thin_lto.push((name, thin_buffer));
1670                        }
1671                        Err(Some(WorkerFatalError)) => {
1672                            // Like `CodegenAborted`, wait for remaining work to finish.
1673                            codegen_state = Aborted;
1674                        }
1675                        Err(None) => {
1676                            // If the thread failed that means it panicked, so
1677                            // we abort immediately.
1678                            bug!("worker thread panicked");
1679                        }
1680                    }
1681                }
1682
1683                Message::AddImportOnlyModule { module_data, work_product } => {
1684                    assert!(!started_lto);
1685                    assert_eq!(codegen_state, Ongoing);
1686                    assert_eq!(main_thread_state, MainThreadState::Codegenning);
1687                    lto_import_only_modules.push((module_data, work_product));
1688                    main_thread_state = MainThreadState::Idle;
1689                }
1690            }
1691        }
1692
1693        if codegen_state == Aborted {
1694            return Err(());
1695        }
1696
1697        let needs_link = mem::take(&mut needs_link);
1698        if !needs_link.is_empty() {
1699            assert!(compiled_modules.is_empty());
1700            let dcx = cgcx.create_dcx();
1701            let dcx = dcx.handle();
1702            let module = B::run_link(&cgcx, dcx, needs_link).map_err(|_| ())?;
1703            let module =
1704                B::codegen(&cgcx, dcx, module, cgcx.config(ModuleKind::Regular)).map_err(|_| ())?;
1705            compiled_modules.push(module);
1706        }
1707
1708        // Drop to print timings
1709        drop(llvm_start_time);
1710
1711        // Regardless of what order these modules completed in, report them to
1712        // the backend in the same order every time to ensure that we're handing
1713        // out deterministic results.
1714        compiled_modules.sort_by(|a, b| a.name.cmp(&b.name));
1715
1716        Ok(CompiledModules {
1717            modules: compiled_modules,
1718            allocator_module: compiled_allocator_module,
1719        })
1720    })
1721    .expect("failed to spawn coordinator thread");
1722
1723    // A heuristic that determines if we have enough LLVM WorkItems in the
1724    // queue so that the main thread can do LLVM work instead of codegen
1725    fn queue_full_enough(items_in_queue: usize, workers_running: usize) -> bool {
1726        // This heuristic scales ahead-of-time codegen according to available
1727        // concurrency, as measured by `workers_running`. The idea is that the
1728        // more concurrency we have available, the more demand there will be for
1729        // work items, and the fuller the queue should be kept to meet demand.
1730        // An important property of this approach is that we codegen ahead of
1731        // time only as much as necessary, so as to keep fewer LLVM modules in
1732        // memory at once, thereby reducing memory consumption.
1733        //
1734        // When the number of workers running is less than the max concurrency
1735        // available to us, this heuristic can cause us to instruct the main
1736        // thread to work on an LLVM item (that is, tell it to "LLVM") instead
1737        // of codegen, even though it seems like it *should* be codegenning so
1738        // that we can create more work items and spawn more LLVM workers.
1739        //
1740        // But this is not a problem. When the main thread is told to LLVM,
1741        // according to this heuristic and how work is scheduled, there is
1742        // always at least one item in the queue, and therefore at least one
1743        // pending jobserver token request. If there *is* more concurrency
1744        // available, we will immediately receive a token, which will upgrade
1745        // the main thread's LLVM worker to a real one (conceptually), and free
1746        // up the main thread to codegen if necessary. On the other hand, if
1747        // there isn't more concurrency, then the main thread working on an LLVM
1748        // item is appropriate, as long as the queue is full enough for demand.
1749        //
1750        // Speaking of which, how full should we keep the queue? Probably less
1751        // full than you'd think. A lot has to go wrong for the queue not to be
1752        // full enough and for that to have a negative effect on compile times.
1753        //
1754        // Workers are unlikely to finish at exactly the same time, so when one
1755        // finishes and takes another work item off the queue, we often have
1756        // ample time to codegen at that point before the next worker finishes.
1757        // But suppose that codegen takes so long that the workers exhaust the
1758        // queue, and we have one or more workers that have nothing to work on.
1759        // Well, it might not be so bad. Of all the LLVM modules we create and
1760        // optimize, one has to finish last. It's not necessarily the case that
1761        // by losing some concurrency for a moment, we delay the point at which
1762        // that last LLVM module is finished and the rest of compilation can
1763        // proceed. Also, when we can't take advantage of some concurrency, we
1764        // give tokens back to the job server. That enables some other rustc to
1765        // potentially make use of the available concurrency. That could even
1766        // *decrease* overall compile time if we're lucky. But yes, if no other
1767        // rustc can make use of the concurrency, then we've squandered it.
1768        //
1769        // However, keeping the queue full is also beneficial when we have a
1770        // surge in available concurrency. Then items can be taken from the
1771        // queue immediately, without having to wait for codegen.
1772        //
1773        // So, the heuristic below tries to keep one item in the queue for every
1774        // four running workers. Based on limited benchmarking, this appears to
1775        // be more than sufficient to avoid increasing compilation times.
1776        let quarter_of_workers = workers_running - 3 * workers_running / 4;
1777        items_in_queue > 0 && items_in_queue >= quarter_of_workers
1778    }
1779}
1780
1781/// `FatalError` is explicitly not `Send`.
1782#[must_use]
1783pub(crate) struct WorkerFatalError;
1784
1785fn spawn_work<'a, B: ExtraBackendMethods>(
1786    cgcx: &'a CodegenContext<B>,
1787    llvm_start_time: &mut Option<VerboseTimingGuard<'a>>,
1788    worker_id: usize,
1789    work: WorkItem<B>,
1790) {
1791    if cgcx.config(work.module_kind()).time_module && llvm_start_time.is_none() {
1792        *llvm_start_time = Some(cgcx.prof.verbose_generic_activity("LLVM_passes"));
1793    }
1794
1795    let cgcx = cgcx.clone();
1796
1797    B::spawn_named_thread(cgcx.time_trace, work.short_description(), move || {
1798        // Set up a destructor which will fire off a message that we're done as
1799        // we exit.
1800        struct Bomb<B: ExtraBackendMethods> {
1801            coordinator_send: Sender<Box<dyn Any + Send>>,
1802            result: Option<Result<WorkItemResult<B>, FatalError>>,
1803            worker_id: usize,
1804        }
1805        impl<B: ExtraBackendMethods> Drop for Bomb<B> {
1806            fn drop(&mut self) {
1807                let worker_id = self.worker_id;
1808                let msg = match self.result.take() {
1809                    Some(Ok(result)) => Message::WorkItem::<B> { result: Ok(result), worker_id },
1810                    Some(Err(FatalError)) => {
1811                        Message::WorkItem::<B> { result: Err(Some(WorkerFatalError)), worker_id }
1812                    }
1813                    None => Message::WorkItem::<B> { result: Err(None), worker_id },
1814                };
1815                drop(self.coordinator_send.send(Box::new(msg)));
1816            }
1817        }
1818
1819        let mut bomb =
1820            Bomb::<B> { coordinator_send: cgcx.coordinator_send.clone(), result: None, worker_id };
1821
1822        // Execute the work itself, and if it finishes successfully then flag
1823        // ourselves as a success as well.
1824        //
1825        // Note that we ignore any `FatalError` coming out of `execute_work_item`,
1826        // as a diagnostic was already sent off to the main thread - just
1827        // surface that there was an error in this worker.
1828        bomb.result = {
1829            let module_config = cgcx.config(work.module_kind());
1830
1831            Some(match work {
1832                WorkItem::Optimize(m) => {
1833                    let _timer =
1834                        cgcx.prof.generic_activity_with_arg("codegen_module_optimize", &*m.name);
1835                    execute_optimize_work_item(&cgcx, m, module_config)
1836                }
1837                WorkItem::CopyPostLtoArtifacts(m) => {
1838                    let _timer = cgcx.prof.generic_activity_with_arg(
1839                        "codegen_copy_artifacts_from_incr_cache",
1840                        &*m.name,
1841                    );
1842                    Ok(execute_copy_from_cache_work_item(&cgcx, m, module_config))
1843                }
1844                WorkItem::LTO(m) => {
1845                    let _timer =
1846                        cgcx.prof.generic_activity_with_arg("codegen_module_perform_lto", m.name());
1847                    execute_lto_work_item(&cgcx, m, module_config)
1848                }
1849            })
1850        };
1851    })
1852    .expect("failed to spawn work thread");
1853}
1854
1855enum SharedEmitterMessage {
1856    Diagnostic(Diagnostic),
1857    InlineAsmError(SpanData, String, Level, Option<(String, Vec<InnerSpan>)>),
1858    Fatal(String),
1859}
1860
1861#[derive(Clone)]
1862pub struct SharedEmitter {
1863    sender: Sender<SharedEmitterMessage>,
1864}
1865
1866pub struct SharedEmitterMain {
1867    receiver: Receiver<SharedEmitterMessage>,
1868}
1869
1870impl SharedEmitter {
1871    fn new() -> (SharedEmitter, SharedEmitterMain) {
1872        let (sender, receiver) = channel();
1873
1874        (SharedEmitter { sender }, SharedEmitterMain { receiver })
1875    }
1876
1877    pub fn inline_asm_error(
1878        &self,
1879        span: SpanData,
1880        msg: String,
1881        level: Level,
1882        source: Option<(String, Vec<InnerSpan>)>,
1883    ) {
1884        drop(self.sender.send(SharedEmitterMessage::InlineAsmError(span, msg, level, source)));
1885    }
1886
1887    fn fatal(&self, msg: &str) {
1888        drop(self.sender.send(SharedEmitterMessage::Fatal(msg.to_string())));
1889    }
1890}
1891
1892impl Emitter for SharedEmitter {
1893    fn emit_diagnostic(
1894        &mut self,
1895        mut diag: rustc_errors::DiagInner,
1896        _registry: &rustc_errors::registry::Registry,
1897    ) {
1898        // Check that we aren't missing anything interesting when converting to
1899        // the cut-down local `DiagInner`.
1900        assert_eq!(diag.span, MultiSpan::new());
1901        assert_eq!(diag.suggestions, Suggestions::Enabled(vec![]));
1902        assert_eq!(diag.sort_span, rustc_span::DUMMY_SP);
1903        assert_eq!(diag.is_lint, None);
1904        // No sensible check for `diag.emitted_at`.
1905
1906        let args = mem::replace(&mut diag.args, DiagArgMap::default());
1907        drop(
1908            self.sender.send(SharedEmitterMessage::Diagnostic(Diagnostic {
1909                level: diag.level(),
1910                messages: diag.messages,
1911                code: diag.code,
1912                children: diag
1913                    .children
1914                    .into_iter()
1915                    .map(|child| Subdiagnostic { level: child.level, messages: child.messages })
1916                    .collect(),
1917                args,
1918            })),
1919        );
1920    }
1921
1922    fn source_map(&self) -> Option<&SourceMap> {
1923        None
1924    }
1925
1926    fn translator(&self) -> &Translator {
1927        panic!("shared emitter attempted to translate a diagnostic");
1928    }
1929}
1930
1931impl SharedEmitterMain {
1932    fn check(&self, sess: &Session, blocking: bool) {
1933        loop {
1934            let message = if blocking {
1935                match self.receiver.recv() {
1936                    Ok(message) => Ok(message),
1937                    Err(_) => Err(()),
1938                }
1939            } else {
1940                match self.receiver.try_recv() {
1941                    Ok(message) => Ok(message),
1942                    Err(_) => Err(()),
1943                }
1944            };
1945
1946            match message {
1947                Ok(SharedEmitterMessage::Diagnostic(diag)) => {
1948                    // The diagnostic has been received on the main thread.
1949                    // Convert it back to a full `Diagnostic` and emit.
1950                    let dcx = sess.dcx();
1951                    let mut d =
1952                        rustc_errors::DiagInner::new_with_messages(diag.level, diag.messages);
1953                    d.code = diag.code; // may be `None`, that's ok
1954                    d.children = diag
1955                        .children
1956                        .into_iter()
1957                        .map(|sub| rustc_errors::Subdiag {
1958                            level: sub.level,
1959                            messages: sub.messages,
1960                            span: MultiSpan::new(),
1961                        })
1962                        .collect();
1963                    d.args = diag.args;
1964                    dcx.emit_diagnostic(d);
1965                    sess.dcx().abort_if_errors();
1966                }
1967                Ok(SharedEmitterMessage::InlineAsmError(span, msg, level, source)) => {
1968                    assert_matches!(level, Level::Error | Level::Warning | Level::Note);
1969                    let mut err = Diag::<()>::new(sess.dcx(), level, msg);
1970                    if !span.is_dummy() {
1971                        err.span(span.span());
1972                    }
1973
1974                    // Point to the generated assembly if it is available.
1975                    if let Some((buffer, spans)) = source {
1976                        let source = sess
1977                            .source_map()
1978                            .new_source_file(FileName::inline_asm_source_code(&buffer), buffer);
1979                        let spans: Vec<_> = spans
1980                            .iter()
1981                            .map(|sp| {
1982                                Span::with_root_ctxt(
1983                                    source.normalized_byte_pos(sp.start as u32),
1984                                    source.normalized_byte_pos(sp.end as u32),
1985                                )
1986                            })
1987                            .collect();
1988                        err.span_note(spans, "instantiated into assembly here");
1989                    }
1990
1991                    err.emit();
1992                }
1993                Ok(SharedEmitterMessage::Fatal(msg)) => {
1994                    sess.dcx().fatal(msg);
1995                }
1996                Err(_) => {
1997                    break;
1998                }
1999            }
2000        }
2001    }
2002}
2003
2004pub struct Coordinator<B: ExtraBackendMethods> {
2005    pub sender: Sender<Box<dyn Any + Send>>,
2006    future: Option<thread::JoinHandle<Result<CompiledModules, ()>>>,
2007    // Only used for the Message type.
2008    phantom: PhantomData<B>,
2009}
2010
2011impl<B: ExtraBackendMethods> Coordinator<B> {
2012    fn join(mut self) -> std::thread::Result<Result<CompiledModules, ()>> {
2013        self.future.take().unwrap().join()
2014    }
2015}
2016
2017impl<B: ExtraBackendMethods> Drop for Coordinator<B> {
2018    fn drop(&mut self) {
2019        if let Some(future) = self.future.take() {
2020            // If we haven't joined yet, signal to the coordinator that it should spawn no more
2021            // work, and wait for worker threads to finish.
2022            drop(self.sender.send(Box::new(Message::CodegenAborted::<B>)));
2023            drop(future.join());
2024        }
2025    }
2026}
2027
2028pub struct OngoingCodegen<B: ExtraBackendMethods> {
2029    pub backend: B,
2030    pub crate_info: CrateInfo,
2031    pub codegen_worker_receive: Receiver<CguMessage>,
2032    pub shared_emitter_main: SharedEmitterMain,
2033    pub output_filenames: Arc<OutputFilenames>,
2034    pub coordinator: Coordinator<B>,
2035}
2036
2037impl<B: ExtraBackendMethods> OngoingCodegen<B> {
2038    pub fn join(self, sess: &Session) -> (CodegenResults, FxIndexMap<WorkProductId, WorkProduct>) {
2039        self.shared_emitter_main.check(sess, true);
2040        let compiled_modules = sess.time("join_worker_thread", || match self.coordinator.join() {
2041            Ok(Ok(compiled_modules)) => compiled_modules,
2042            Ok(Err(())) => {
2043                sess.dcx().abort_if_errors();
2044                panic!("expected abort due to worker thread errors")
2045            }
2046            Err(_) => {
2047                bug!("panic during codegen/LLVM phase");
2048            }
2049        });
2050
2051        sess.dcx().abort_if_errors();
2052
2053        let work_products =
2054            copy_all_cgu_workproducts_to_incr_comp_cache_dir(sess, &compiled_modules);
2055        produce_final_output_artifacts(sess, &compiled_modules, &self.output_filenames);
2056
2057        // FIXME: time_llvm_passes support - does this use a global context or
2058        // something?
2059        if sess.codegen_units().as_usize() == 1 && sess.opts.unstable_opts.time_llvm_passes {
2060            self.backend.print_pass_timings()
2061        }
2062
2063        if sess.print_llvm_stats() {
2064            self.backend.print_statistics()
2065        }
2066
2067        (
2068            CodegenResults {
2069                crate_info: self.crate_info,
2070
2071                modules: compiled_modules.modules,
2072                allocator_module: compiled_modules.allocator_module,
2073            },
2074            work_products,
2075        )
2076    }
2077
2078    pub(crate) fn codegen_finished(&self, tcx: TyCtxt<'_>) {
2079        self.wait_for_signal_to_codegen_item();
2080        self.check_for_errors(tcx.sess);
2081        drop(self.coordinator.sender.send(Box::new(Message::CodegenComplete::<B>)));
2082    }
2083
2084    pub(crate) fn submit_autodiff_items(&self, items: Vec<AutoDiffItem>) {
2085        drop(self.coordinator.sender.send(Box::new(Message::<B>::AddAutoDiffItems(items))));
2086    }
2087
2088    pub(crate) fn check_for_errors(&self, sess: &Session) {
2089        self.shared_emitter_main.check(sess, false);
2090    }
2091
2092    pub(crate) fn wait_for_signal_to_codegen_item(&self) {
2093        match self.codegen_worker_receive.recv() {
2094            Ok(CguMessage) => {
2095                // Ok to proceed.
2096            }
2097            Err(_) => {
2098                // One of the LLVM threads must have panicked, fall through so
2099                // error handling can be reached.
2100            }
2101        }
2102    }
2103}
2104
2105pub(crate) fn submit_codegened_module_to_llvm<B: ExtraBackendMethods>(
2106    _backend: &B,
2107    tx_to_llvm_workers: &Sender<Box<dyn Any + Send>>,
2108    module: ModuleCodegen<B::Module>,
2109    cost: u64,
2110) {
2111    let llvm_work_item = WorkItem::Optimize(module);
2112    drop(tx_to_llvm_workers.send(Box::new(Message::CodegenDone::<B> { llvm_work_item, cost })));
2113}
2114
2115pub(crate) fn submit_post_lto_module_to_llvm<B: ExtraBackendMethods>(
2116    _backend: &B,
2117    tx_to_llvm_workers: &Sender<Box<dyn Any + Send>>,
2118    module: CachedModuleCodegen,
2119) {
2120    let llvm_work_item = WorkItem::CopyPostLtoArtifacts(module);
2121    drop(tx_to_llvm_workers.send(Box::new(Message::CodegenDone::<B> { llvm_work_item, cost: 0 })));
2122}
2123
2124pub(crate) fn submit_pre_lto_module_to_llvm<B: ExtraBackendMethods>(
2125    _backend: &B,
2126    tcx: TyCtxt<'_>,
2127    tx_to_llvm_workers: &Sender<Box<dyn Any + Send>>,
2128    module: CachedModuleCodegen,
2129) {
2130    let filename = pre_lto_bitcode_filename(&module.name);
2131    let bc_path = in_incr_comp_dir_sess(tcx.sess, &filename);
2132    let file = fs::File::open(&bc_path)
2133        .unwrap_or_else(|e| panic!("failed to open bitcode file `{}`: {}", bc_path.display(), e));
2134
2135    let mmap = unsafe {
2136        Mmap::map(file).unwrap_or_else(|e| {
2137            panic!("failed to mmap bitcode file `{}`: {}", bc_path.display(), e)
2138        })
2139    };
2140    // Schedule the module to be loaded
2141    drop(tx_to_llvm_workers.send(Box::new(Message::AddImportOnlyModule::<B> {
2142        module_data: SerializedModule::FromUncompressedFile(mmap),
2143        work_product: module.source,
2144    })));
2145}
2146
2147fn pre_lto_bitcode_filename(module_name: &str) -> String {
2148    format!("{module_name}.{PRE_LTO_BC_EXT}")
2149}
2150
2151fn msvc_imps_needed(tcx: TyCtxt<'_>) -> bool {
2152    // This should never be true (because it's not supported). If it is true,
2153    // something is wrong with commandline arg validation.
2154    assert!(
2155        !(tcx.sess.opts.cg.linker_plugin_lto.enabled()
2156            && tcx.sess.target.is_like_windows
2157            && tcx.sess.opts.cg.prefer_dynamic)
2158    );
2159
2160    // We need to generate _imp__ symbol if we are generating an rlib or we include one
2161    // indirectly from ThinLTO. In theory these are not needed as ThinLTO could resolve
2162    // these, but it currently does not do so.
2163    let can_have_static_objects =
2164        tcx.sess.lto() == Lto::Thin || tcx.crate_types().contains(&CrateType::Rlib);
2165
2166    tcx.sess.target.is_like_windows &&
2167    can_have_static_objects   &&
2168    // ThinLTO can't handle this workaround in all cases, so we don't
2169    // emit the `__imp_` symbols. Instead we make them unnecessary by disallowing
2170    // dynamic linking when linker plugin LTO is enabled.
2171    !tcx.sess.opts.cg.linker_plugin_lto.enabled()
2172}