rustc_codegen_ssa/back/
write.rs

1use std::any::Any;
2use std::assert_matches::assert_matches;
3use std::marker::PhantomData;
4use std::path::{Path, PathBuf};
5use std::sync::Arc;
6use std::sync::mpsc::{Receiver, Sender, channel};
7use std::{fs, io, mem, str, thread};
8
9use rustc_abi::Size;
10use rustc_ast::attr;
11use rustc_ast::expand::autodiff_attrs::AutoDiffItem;
12use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
13use rustc_data_structures::jobserver::{self, Acquired};
14use rustc_data_structures::memmap::Mmap;
15use rustc_data_structures::profiling::{SelfProfilerRef, VerboseTimingGuard};
16use rustc_errors::emitter::Emitter;
17use rustc_errors::translation::Translator;
18use rustc_errors::{
19    Diag, DiagArgMap, DiagCtxt, DiagMessage, ErrCode, FatalError, Level, MultiSpan, Style,
20    Suggestions,
21};
22use rustc_fs_util::link_or_copy;
23use rustc_hir::def_id::{CrateNum, LOCAL_CRATE};
24use rustc_incremental::{
25    copy_cgu_workproduct_to_incr_comp_cache_dir, in_incr_comp_dir, in_incr_comp_dir_sess,
26};
27use rustc_metadata::fs::copy_to_stdout;
28use rustc_middle::bug;
29use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
30use rustc_middle::middle::exported_symbols::SymbolExportInfo;
31use rustc_middle::ty::TyCtxt;
32use rustc_session::Session;
33use rustc_session::config::{
34    self, CrateType, Lto, OutFileName, OutputFilenames, OutputType, Passes, SwitchWithOptPath,
35};
36use rustc_span::source_map::SourceMap;
37use rustc_span::{FileName, InnerSpan, Span, SpanData, sym};
38use rustc_target::spec::{MergeFunctions, SanitizerSet};
39use tracing::debug;
40
41use super::link::{self, ensure_removed};
42use super::lto::{self, SerializedModule};
43use super::symbol_export::symbol_name_for_instance_in_crate;
44use crate::errors::{AutodiffWithoutLto, ErrorCreatingRemarkDir};
45use crate::traits::*;
46use crate::{
47    CachedModuleCodegen, CodegenResults, CompiledModule, CrateInfo, ModuleCodegen, ModuleKind,
48    errors,
49};
50
51const PRE_LTO_BC_EXT: &str = "pre-lto.bc";
52
53/// What kind of object file to emit.
54#[derive(Clone, Copy, PartialEq)]
55pub enum EmitObj {
56    // No object file.
57    None,
58
59    // Just uncompressed llvm bitcode. Provides easy compatibility with
60    // emscripten's ecc compiler, when used as the linker.
61    Bitcode,
62
63    // Object code, possibly augmented with a bitcode section.
64    ObjectCode(BitcodeSection),
65}
66
67/// What kind of llvm bitcode section to embed in an object file.
68#[derive(Clone, Copy, PartialEq)]
69pub enum BitcodeSection {
70    // No bitcode section.
71    None,
72
73    // A full, uncompressed bitcode section.
74    Full,
75}
76
77/// Module-specific configuration for `optimize_and_codegen`.
78pub struct ModuleConfig {
79    /// Names of additional optimization passes to run.
80    pub passes: Vec<String>,
81    /// Some(level) to optimize at a certain level, or None to run
82    /// absolutely no optimizations (used for the metadata module).
83    pub opt_level: Option<config::OptLevel>,
84
85    /// Some(level) to optimize binary size, or None to not affect program size.
86    pub opt_size: Option<config::OptLevel>,
87
88    pub pgo_gen: SwitchWithOptPath,
89    pub pgo_use: Option<PathBuf>,
90    pub pgo_sample_use: Option<PathBuf>,
91    pub debug_info_for_profiling: bool,
92    pub instrument_coverage: bool,
93
94    pub sanitizer: SanitizerSet,
95    pub sanitizer_recover: SanitizerSet,
96    pub sanitizer_dataflow_abilist: Vec<String>,
97    pub sanitizer_memory_track_origins: usize,
98
99    // Flags indicating which outputs to produce.
100    pub emit_pre_lto_bc: bool,
101    pub emit_no_opt_bc: bool,
102    pub emit_bc: bool,
103    pub emit_ir: bool,
104    pub emit_asm: bool,
105    pub emit_obj: EmitObj,
106    pub emit_thin_lto: bool,
107    pub emit_thin_lto_summary: bool,
108    pub bc_cmdline: String,
109
110    // Miscellaneous flags. These are mostly copied from command-line
111    // options.
112    pub verify_llvm_ir: bool,
113    pub lint_llvm_ir: bool,
114    pub no_prepopulate_passes: bool,
115    pub no_builtins: bool,
116    pub time_module: bool,
117    pub vectorize_loop: bool,
118    pub vectorize_slp: bool,
119    pub merge_functions: bool,
120    pub emit_lifetime_markers: bool,
121    pub llvm_plugins: Vec<String>,
122    pub autodiff: Vec<config::AutoDiff>,
123}
124
125impl ModuleConfig {
126    fn new(kind: ModuleKind, tcx: TyCtxt<'_>, no_builtins: bool) -> ModuleConfig {
127        // If it's a regular module, use `$regular`, otherwise use `$other`.
128        // `$regular` and `$other` are evaluated lazily.
129        macro_rules! if_regular {
130            ($regular: expr, $other: expr) => {
131                if let ModuleKind::Regular = kind { $regular } else { $other }
132            };
133        }
134
135        let sess = tcx.sess;
136        let opt_level_and_size = if_regular!(Some(sess.opts.optimize), None);
137
138        let save_temps = sess.opts.cg.save_temps;
139
140        let should_emit_obj = sess.opts.output_types.contains_key(&OutputType::Exe)
141            || match kind {
142                ModuleKind::Regular => sess.opts.output_types.contains_key(&OutputType::Object),
143                ModuleKind::Allocator => false,
144            };
145
146        let emit_obj = if !should_emit_obj {
147            EmitObj::None
148        } else if sess.target.obj_is_bitcode
149            || (sess.opts.cg.linker_plugin_lto.enabled() && !no_builtins)
150        {
151            // This case is selected if the target uses objects as bitcode, or
152            // if linker plugin LTO is enabled. In the linker plugin LTO case
153            // the assumption is that the final link-step will read the bitcode
154            // and convert it to object code. This may be done by either the
155            // native linker or rustc itself.
156            //
157            // Note, however, that the linker-plugin-lto requested here is
158            // explicitly ignored for `#![no_builtins]` crates. These crates are
159            // specifically ignored by rustc's LTO passes and wouldn't work if
160            // loaded into the linker. These crates define symbols that LLVM
161            // lowers intrinsics to, and these symbol dependencies aren't known
162            // until after codegen. As a result any crate marked
163            // `#![no_builtins]` is assumed to not participate in LTO and
164            // instead goes on to generate object code.
165            EmitObj::Bitcode
166        } else if need_bitcode_in_object(tcx) {
167            EmitObj::ObjectCode(BitcodeSection::Full)
168        } else {
169            EmitObj::ObjectCode(BitcodeSection::None)
170        };
171
172        ModuleConfig {
173            passes: if_regular!(sess.opts.cg.passes.clone(), vec![]),
174
175            opt_level: opt_level_and_size,
176            opt_size: opt_level_and_size,
177
178            pgo_gen: if_regular!(
179                sess.opts.cg.profile_generate.clone(),
180                SwitchWithOptPath::Disabled
181            ),
182            pgo_use: if_regular!(sess.opts.cg.profile_use.clone(), None),
183            pgo_sample_use: if_regular!(sess.opts.unstable_opts.profile_sample_use.clone(), None),
184            debug_info_for_profiling: sess.opts.unstable_opts.debug_info_for_profiling,
185            instrument_coverage: if_regular!(sess.instrument_coverage(), false),
186
187            sanitizer: if_regular!(sess.opts.unstable_opts.sanitizer, SanitizerSet::empty()),
188            sanitizer_dataflow_abilist: if_regular!(
189                sess.opts.unstable_opts.sanitizer_dataflow_abilist.clone(),
190                Vec::new()
191            ),
192            sanitizer_recover: if_regular!(
193                sess.opts.unstable_opts.sanitizer_recover,
194                SanitizerSet::empty()
195            ),
196            sanitizer_memory_track_origins: if_regular!(
197                sess.opts.unstable_opts.sanitizer_memory_track_origins,
198                0
199            ),
200
201            emit_pre_lto_bc: if_regular!(
202                save_temps || need_pre_lto_bitcode_for_incr_comp(sess),
203                false
204            ),
205            emit_no_opt_bc: if_regular!(save_temps, false),
206            emit_bc: if_regular!(
207                save_temps || sess.opts.output_types.contains_key(&OutputType::Bitcode),
208                save_temps
209            ),
210            emit_ir: if_regular!(
211                sess.opts.output_types.contains_key(&OutputType::LlvmAssembly),
212                false
213            ),
214            emit_asm: if_regular!(
215                sess.opts.output_types.contains_key(&OutputType::Assembly),
216                false
217            ),
218            emit_obj,
219            emit_thin_lto: sess.opts.unstable_opts.emit_thin_lto,
220            emit_thin_lto_summary: if_regular!(
221                sess.opts.output_types.contains_key(&OutputType::ThinLinkBitcode),
222                false
223            ),
224            bc_cmdline: sess.target.bitcode_llvm_cmdline.to_string(),
225
226            verify_llvm_ir: sess.verify_llvm_ir(),
227            lint_llvm_ir: sess.opts.unstable_opts.lint_llvm_ir,
228            no_prepopulate_passes: sess.opts.cg.no_prepopulate_passes,
229            no_builtins: no_builtins || sess.target.no_builtins,
230
231            // Exclude metadata and allocator modules from time_passes output,
232            // since they throw off the "LLVM passes" measurement.
233            time_module: if_regular!(true, false),
234
235            // Copy what clang does by turning on loop vectorization at O2 and
236            // slp vectorization at O3.
237            vectorize_loop: !sess.opts.cg.no_vectorize_loops
238                && (sess.opts.optimize == config::OptLevel::More
239                    || sess.opts.optimize == config::OptLevel::Aggressive),
240            vectorize_slp: !sess.opts.cg.no_vectorize_slp
241                && sess.opts.optimize == config::OptLevel::Aggressive,
242
243            // Some targets (namely, NVPTX) interact badly with the
244            // MergeFunctions pass. This is because MergeFunctions can generate
245            // new function calls which may interfere with the target calling
246            // convention; e.g. for the NVPTX target, PTX kernels should not
247            // call other PTX kernels. MergeFunctions can also be configured to
248            // generate aliases instead, but aliases are not supported by some
249            // backends (again, NVPTX). Therefore, allow targets to opt out of
250            // the MergeFunctions pass, but otherwise keep the pass enabled (at
251            // O2 and O3) since it can be useful for reducing code size.
252            merge_functions: match sess
253                .opts
254                .unstable_opts
255                .merge_functions
256                .unwrap_or(sess.target.merge_functions)
257            {
258                MergeFunctions::Disabled => false,
259                MergeFunctions::Trampolines | MergeFunctions::Aliases => {
260                    use config::OptLevel::*;
261                    match sess.opts.optimize {
262                        Aggressive | More | SizeMin | Size => true,
263                        Less | No => false,
264                    }
265                }
266            },
267
268            emit_lifetime_markers: sess.emit_lifetime_markers(),
269            llvm_plugins: if_regular!(sess.opts.unstable_opts.llvm_plugins.clone(), vec![]),
270            autodiff: if_regular!(sess.opts.unstable_opts.autodiff.clone(), vec![]),
271        }
272    }
273
274    pub fn bitcode_needed(&self) -> bool {
275        self.emit_bc
276            || self.emit_thin_lto_summary
277            || self.emit_obj == EmitObj::Bitcode
278            || self.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full)
279    }
280
281    pub fn embed_bitcode(&self) -> bool {
282        self.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full)
283    }
284}
285
286/// Configuration passed to the function returned by the `target_machine_factory`.
287pub struct TargetMachineFactoryConfig {
288    /// Split DWARF is enabled in LLVM by checking that `TM.MCOptions.SplitDwarfFile` isn't empty,
289    /// so the path to the dwarf object has to be provided when we create the target machine.
290    /// This can be ignored by backends which do not need it for their Split DWARF support.
291    pub split_dwarf_file: Option<PathBuf>,
292
293    /// The name of the output object file. Used for setting OutputFilenames in target options
294    /// so that LLVM can emit the CodeView S_OBJNAME record in pdb files
295    pub output_obj_file: Option<PathBuf>,
296}
297
298impl TargetMachineFactoryConfig {
299    pub fn new(
300        cgcx: &CodegenContext<impl WriteBackendMethods>,
301        module_name: &str,
302    ) -> TargetMachineFactoryConfig {
303        let split_dwarf_file = if cgcx.target_can_use_split_dwarf {
304            cgcx.output_filenames.split_dwarf_path(
305                cgcx.split_debuginfo,
306                cgcx.split_dwarf_kind,
307                module_name,
308                cgcx.invocation_temp.as_deref(),
309            )
310        } else {
311            None
312        };
313
314        let output_obj_file = Some(cgcx.output_filenames.temp_path_for_cgu(
315            OutputType::Object,
316            module_name,
317            cgcx.invocation_temp.as_deref(),
318        ));
319        TargetMachineFactoryConfig { split_dwarf_file, output_obj_file }
320    }
321}
322
323pub type TargetMachineFactoryFn<B> = Arc<
324    dyn Fn(
325            TargetMachineFactoryConfig,
326        ) -> Result<
327            <B as WriteBackendMethods>::TargetMachine,
328            <B as WriteBackendMethods>::TargetMachineError,
329        > + Send
330        + Sync,
331>;
332
333type ExportedSymbols = FxHashMap<CrateNum, Arc<Vec<(String, SymbolExportInfo)>>>;
334
335/// Additional resources used by optimize_and_codegen (not module specific)
336#[derive(Clone)]
337pub struct CodegenContext<B: WriteBackendMethods> {
338    // Resources needed when running LTO
339    pub prof: SelfProfilerRef,
340    pub lto: Lto,
341    pub save_temps: bool,
342    pub fewer_names: bool,
343    pub time_trace: bool,
344    pub exported_symbols: Option<Arc<ExportedSymbols>>,
345    pub opts: Arc<config::Options>,
346    pub crate_types: Vec<CrateType>,
347    pub each_linked_rlib_for_lto: Vec<(CrateNum, PathBuf)>,
348    pub output_filenames: Arc<OutputFilenames>,
349    pub invocation_temp: Option<String>,
350    pub regular_module_config: Arc<ModuleConfig>,
351    pub allocator_module_config: Arc<ModuleConfig>,
352    pub tm_factory: TargetMachineFactoryFn<B>,
353    pub msvc_imps_needed: bool,
354    pub is_pe_coff: bool,
355    pub target_can_use_split_dwarf: bool,
356    pub target_arch: String,
357    pub target_is_like_darwin: bool,
358    pub target_is_like_aix: bool,
359    pub split_debuginfo: rustc_target::spec::SplitDebuginfo,
360    pub split_dwarf_kind: rustc_session::config::SplitDwarfKind,
361    pub pointer_size: Size,
362
363    /// All commandline args used to invoke the compiler, with @file args fully expanded.
364    /// This will only be used within debug info, e.g. in the pdb file on windows
365    /// This is mainly useful for other tools that reads that debuginfo to figure out
366    /// how to call the compiler with the same arguments.
367    pub expanded_args: Vec<String>,
368
369    /// Emitter to use for diagnostics produced during codegen.
370    pub diag_emitter: SharedEmitter,
371    /// LLVM optimizations for which we want to print remarks.
372    pub remark: Passes,
373    /// Directory into which should the LLVM optimization remarks be written.
374    /// If `None`, they will be written to stderr.
375    pub remark_dir: Option<PathBuf>,
376    /// The incremental compilation session directory, or None if we are not
377    /// compiling incrementally
378    pub incr_comp_session_dir: Option<PathBuf>,
379    /// Channel back to the main control thread to send messages to
380    pub coordinator_send: Sender<Box<dyn Any + Send>>,
381    /// `true` if the codegen should be run in parallel.
382    ///
383    /// Depends on [`ExtraBackendMethods::supports_parallel()`] and `-Zno_parallel_backend`.
384    pub parallel: bool,
385}
386
387impl<B: WriteBackendMethods> CodegenContext<B> {
388    pub fn create_dcx(&self) -> DiagCtxt {
389        DiagCtxt::new(Box::new(self.diag_emitter.clone()))
390    }
391
392    pub fn config(&self, kind: ModuleKind) -> &ModuleConfig {
393        match kind {
394            ModuleKind::Regular => &self.regular_module_config,
395            ModuleKind::Allocator => &self.allocator_module_config,
396        }
397    }
398}
399
400fn generate_lto_work<B: ExtraBackendMethods>(
401    cgcx: &CodegenContext<B>,
402    autodiff: Vec<AutoDiffItem>,
403    needs_fat_lto: Vec<FatLtoInput<B>>,
404    needs_thin_lto: Vec<(String, B::ThinBuffer)>,
405    import_only_modules: Vec<(SerializedModule<B::ModuleBuffer>, WorkProduct)>,
406) -> Vec<(WorkItem<B>, u64)> {
407    let _prof_timer = cgcx.prof.generic_activity("codegen_generate_lto_work");
408
409    if !needs_fat_lto.is_empty() {
410        assert!(needs_thin_lto.is_empty());
411        let mut module =
412            B::run_fat_lto(cgcx, needs_fat_lto, import_only_modules).unwrap_or_else(|e| e.raise());
413        if cgcx.lto == Lto::Fat && !autodiff.is_empty() {
414            let config = cgcx.config(ModuleKind::Regular);
415            module = module.autodiff(cgcx, autodiff, config).unwrap_or_else(|e| e.raise());
416        }
417        // We are adding a single work item, so the cost doesn't matter.
418        vec![(WorkItem::LTO(module), 0)]
419    } else {
420        if !autodiff.is_empty() {
421            let dcx = cgcx.create_dcx();
422            dcx.handle().emit_fatal(AutodiffWithoutLto {});
423        }
424        assert!(needs_fat_lto.is_empty());
425        let (lto_modules, copy_jobs) = B::run_thin_lto(cgcx, needs_thin_lto, import_only_modules)
426            .unwrap_or_else(|e| e.raise());
427        lto_modules
428            .into_iter()
429            .map(|module| {
430                let cost = module.cost();
431                (WorkItem::LTO(module), cost)
432            })
433            .chain(copy_jobs.into_iter().map(|wp| {
434                (
435                    WorkItem::CopyPostLtoArtifacts(CachedModuleCodegen {
436                        name: wp.cgu_name.clone(),
437                        source: wp,
438                    }),
439                    0, // copying is very cheap
440                )
441            }))
442            .collect()
443    }
444}
445
446struct CompiledModules {
447    modules: Vec<CompiledModule>,
448    allocator_module: Option<CompiledModule>,
449}
450
451fn need_bitcode_in_object(tcx: TyCtxt<'_>) -> bool {
452    let sess = tcx.sess;
453    sess.opts.cg.embed_bitcode
454        && tcx.crate_types().contains(&CrateType::Rlib)
455        && sess.opts.output_types.contains_key(&OutputType::Exe)
456}
457
458fn need_pre_lto_bitcode_for_incr_comp(sess: &Session) -> bool {
459    if sess.opts.incremental.is_none() {
460        return false;
461    }
462
463    match sess.lto() {
464        Lto::No => false,
465        Lto::Fat | Lto::Thin | Lto::ThinLocal => true,
466    }
467}
468
469pub(crate) fn start_async_codegen<B: ExtraBackendMethods>(
470    backend: B,
471    tcx: TyCtxt<'_>,
472    target_cpu: String,
473) -> OngoingCodegen<B> {
474    let (coordinator_send, coordinator_receive) = channel();
475
476    let crate_attrs = tcx.hir_attrs(rustc_hir::CRATE_HIR_ID);
477    let no_builtins = attr::contains_name(crate_attrs, sym::no_builtins);
478
479    let crate_info = CrateInfo::new(tcx, target_cpu);
480
481    let regular_config = ModuleConfig::new(ModuleKind::Regular, tcx, no_builtins);
482    let allocator_config = ModuleConfig::new(ModuleKind::Allocator, tcx, no_builtins);
483
484    let (shared_emitter, shared_emitter_main) = SharedEmitter::new();
485    let (codegen_worker_send, codegen_worker_receive) = channel();
486
487    let coordinator_thread = start_executing_work(
488        backend.clone(),
489        tcx,
490        &crate_info,
491        shared_emitter,
492        codegen_worker_send,
493        coordinator_receive,
494        Arc::new(regular_config),
495        Arc::new(allocator_config),
496        coordinator_send.clone(),
497    );
498
499    OngoingCodegen {
500        backend,
501        crate_info,
502
503        codegen_worker_receive,
504        shared_emitter_main,
505        coordinator: Coordinator {
506            sender: coordinator_send,
507            future: Some(coordinator_thread),
508            phantom: PhantomData,
509        },
510        output_filenames: Arc::clone(tcx.output_filenames(())),
511    }
512}
513
514fn copy_all_cgu_workproducts_to_incr_comp_cache_dir(
515    sess: &Session,
516    compiled_modules: &CompiledModules,
517) -> FxIndexMap<WorkProductId, WorkProduct> {
518    let mut work_products = FxIndexMap::default();
519
520    if sess.opts.incremental.is_none() {
521        return work_products;
522    }
523
524    let _timer = sess.timer("copy_all_cgu_workproducts_to_incr_comp_cache_dir");
525
526    for module in compiled_modules.modules.iter().filter(|m| m.kind == ModuleKind::Regular) {
527        let mut files = Vec::new();
528        if let Some(object_file_path) = &module.object {
529            files.push((OutputType::Object.extension(), object_file_path.as_path()));
530        }
531        if let Some(dwarf_object_file_path) = &module.dwarf_object {
532            files.push(("dwo", dwarf_object_file_path.as_path()));
533        }
534        if let Some(path) = &module.assembly {
535            files.push((OutputType::Assembly.extension(), path.as_path()));
536        }
537        if let Some(path) = &module.llvm_ir {
538            files.push((OutputType::LlvmAssembly.extension(), path.as_path()));
539        }
540        if let Some(path) = &module.bytecode {
541            files.push((OutputType::Bitcode.extension(), path.as_path()));
542        }
543        if let Some((id, product)) = copy_cgu_workproduct_to_incr_comp_cache_dir(
544            sess,
545            &module.name,
546            files.as_slice(),
547            &module.links_from_incr_cache,
548        ) {
549            work_products.insert(id, product);
550        }
551    }
552
553    work_products
554}
555
556fn produce_final_output_artifacts(
557    sess: &Session,
558    compiled_modules: &CompiledModules,
559    crate_output: &OutputFilenames,
560) {
561    let mut user_wants_bitcode = false;
562    let mut user_wants_objects = false;
563
564    // Produce final compile outputs.
565    let copy_gracefully = |from: &Path, to: &OutFileName| match to {
566        OutFileName::Stdout if let Err(e) = copy_to_stdout(from) => {
567            sess.dcx().emit_err(errors::CopyPath::new(from, to.as_path(), e));
568        }
569        OutFileName::Real(path) if let Err(e) = fs::copy(from, path) => {
570            sess.dcx().emit_err(errors::CopyPath::new(from, path, e));
571        }
572        _ => {}
573    };
574
575    let copy_if_one_unit = |output_type: OutputType, keep_numbered: bool| {
576        if let [module] = &compiled_modules.modules[..] {
577            // 1) Only one codegen unit. In this case it's no difficulty
578            //    to copy `foo.0.x` to `foo.x`.
579            let path = crate_output.temp_path_for_cgu(
580                output_type,
581                &module.name,
582                sess.invocation_temp.as_deref(),
583            );
584            let output = crate_output.path(output_type);
585            if !output_type.is_text_output() && output.is_tty() {
586                sess.dcx()
587                    .emit_err(errors::BinaryOutputToTty { shorthand: output_type.shorthand() });
588            } else {
589                copy_gracefully(&path, &output);
590            }
591            if !sess.opts.cg.save_temps && !keep_numbered {
592                // The user just wants `foo.x`, not `foo.#module-name#.x`.
593                ensure_removed(sess.dcx(), &path);
594            }
595        } else {
596            if crate_output.outputs.contains_explicit_name(&output_type) {
597                // 2) Multiple codegen units, with `--emit foo=some_name`. We have
598                //    no good solution for this case, so warn the user.
599                sess.dcx()
600                    .emit_warn(errors::IgnoringEmitPath { extension: output_type.extension() });
601            } else if crate_output.single_output_file.is_some() {
602                // 3) Multiple codegen units, with `-o some_name`. We have
603                //    no good solution for this case, so warn the user.
604                sess.dcx().emit_warn(errors::IgnoringOutput { extension: output_type.extension() });
605            } else {
606                // 4) Multiple codegen units, but no explicit name. We
607                //    just leave the `foo.0.x` files in place.
608                // (We don't have to do any work in this case.)
609            }
610        }
611    };
612
613    // Flag to indicate whether the user explicitly requested bitcode.
614    // Otherwise, we produced it only as a temporary output, and will need
615    // to get rid of it.
616    for output_type in crate_output.outputs.keys() {
617        match *output_type {
618            OutputType::Bitcode => {
619                user_wants_bitcode = true;
620                // Copy to .bc, but always keep the .0.bc. There is a later
621                // check to figure out if we should delete .0.bc files, or keep
622                // them for making an rlib.
623                copy_if_one_unit(OutputType::Bitcode, true);
624            }
625            OutputType::ThinLinkBitcode => {
626                copy_if_one_unit(OutputType::ThinLinkBitcode, false);
627            }
628            OutputType::LlvmAssembly => {
629                copy_if_one_unit(OutputType::LlvmAssembly, false);
630            }
631            OutputType::Assembly => {
632                copy_if_one_unit(OutputType::Assembly, false);
633            }
634            OutputType::Object => {
635                user_wants_objects = true;
636                copy_if_one_unit(OutputType::Object, true);
637            }
638            OutputType::Mir | OutputType::Metadata | OutputType::Exe | OutputType::DepInfo => {}
639        }
640    }
641
642    // Clean up unwanted temporary files.
643
644    // We create the following files by default:
645    //  - #crate#.#module-name#.bc
646    //  - #crate#.#module-name#.o
647    //  - #crate#.crate.metadata.bc
648    //  - #crate#.crate.metadata.o
649    //  - #crate#.o (linked from crate.##.o)
650    //  - #crate#.bc (copied from crate.##.bc)
651    // We may create additional files if requested by the user (through
652    // `-C save-temps` or `--emit=` flags).
653
654    if !sess.opts.cg.save_temps {
655        // Remove the temporary .#module-name#.o objects. If the user didn't
656        // explicitly request bitcode (with --emit=bc), and the bitcode is not
657        // needed for building an rlib, then we must remove .#module-name#.bc as
658        // well.
659
660        // Specific rules for keeping .#module-name#.bc:
661        //  - If the user requested bitcode (`user_wants_bitcode`), and
662        //    codegen_units > 1, then keep it.
663        //  - If the user requested bitcode but codegen_units == 1, then we
664        //    can toss .#module-name#.bc because we copied it to .bc earlier.
665        //  - If we're not building an rlib and the user didn't request
666        //    bitcode, then delete .#module-name#.bc.
667        // If you change how this works, also update back::link::link_rlib,
668        // where .#module-name#.bc files are (maybe) deleted after making an
669        // rlib.
670        let needs_crate_object = crate_output.outputs.contains_key(&OutputType::Exe);
671
672        let keep_numbered_bitcode = user_wants_bitcode && sess.codegen_units().as_usize() > 1;
673
674        let keep_numbered_objects =
675            needs_crate_object || (user_wants_objects && sess.codegen_units().as_usize() > 1);
676
677        for module in compiled_modules.modules.iter() {
678            if !keep_numbered_objects {
679                if let Some(ref path) = module.object {
680                    ensure_removed(sess.dcx(), path);
681                }
682
683                if let Some(ref path) = module.dwarf_object {
684                    ensure_removed(sess.dcx(), path);
685                }
686            }
687
688            if let Some(ref path) = module.bytecode {
689                if !keep_numbered_bitcode {
690                    ensure_removed(sess.dcx(), path);
691                }
692            }
693        }
694
695        if !user_wants_bitcode
696            && let Some(ref allocator_module) = compiled_modules.allocator_module
697            && let Some(ref path) = allocator_module.bytecode
698        {
699            ensure_removed(sess.dcx(), path);
700        }
701    }
702
703    if sess.opts.json_artifact_notifications {
704        if let [module] = &compiled_modules.modules[..] {
705            module.for_each_output(|_path, ty| {
706                if sess.opts.output_types.contains_key(&ty) {
707                    let descr = ty.shorthand();
708                    // for single cgu file is renamed to drop cgu specific suffix
709                    // so we regenerate it the same way
710                    let path = crate_output.path(ty);
711                    sess.dcx().emit_artifact_notification(path.as_path(), descr);
712                }
713            });
714        } else {
715            for module in &compiled_modules.modules {
716                module.for_each_output(|path, ty| {
717                    if sess.opts.output_types.contains_key(&ty) {
718                        let descr = ty.shorthand();
719                        sess.dcx().emit_artifact_notification(&path, descr);
720                    }
721                });
722            }
723        }
724    }
725
726    // We leave the following files around by default:
727    //  - #crate#.o
728    //  - #crate#.crate.metadata.o
729    //  - #crate#.bc
730    // These are used in linking steps and will be cleaned up afterward.
731}
732
733pub(crate) enum WorkItem<B: WriteBackendMethods> {
734    /// Optimize a newly codegened, totally unoptimized module.
735    Optimize(ModuleCodegen<B::Module>),
736    /// Copy the post-LTO artifacts from the incremental cache to the output
737    /// directory.
738    CopyPostLtoArtifacts(CachedModuleCodegen),
739    /// Performs (Thin)LTO on the given module.
740    LTO(lto::LtoModuleCodegen<B>),
741}
742
743impl<B: WriteBackendMethods> WorkItem<B> {
744    fn module_kind(&self) -> ModuleKind {
745        match *self {
746            WorkItem::Optimize(ref m) => m.kind,
747            WorkItem::CopyPostLtoArtifacts(_) | WorkItem::LTO(_) => ModuleKind::Regular,
748        }
749    }
750
751    /// Generate a short description of this work item suitable for use as a thread name.
752    fn short_description(&self) -> String {
753        // `pthread_setname()` on *nix ignores anything beyond the first 15
754        // bytes. Use short descriptions to maximize the space available for
755        // the module name.
756        #[cfg(not(windows))]
757        fn desc(short: &str, _long: &str, name: &str) -> String {
758            // The short label is three bytes, and is followed by a space. That
759            // leaves 11 bytes for the CGU name. How we obtain those 11 bytes
760            // depends on the CGU name form.
761            //
762            // - Non-incremental, e.g. `regex.f10ba03eb5ec7975-cgu.0`: the part
763            //   before the `-cgu.0` is the same for every CGU, so use the
764            //   `cgu.0` part. The number suffix will be different for each
765            //   CGU.
766            //
767            // - Incremental (normal), e.g. `2i52vvl2hco29us0`: use the whole
768            //   name because each CGU will have a unique ASCII hash, and the
769            //   first 11 bytes will be enough to identify it.
770            //
771            // - Incremental (with `-Zhuman-readable-cgu-names`), e.g.
772            //   `regex.f10ba03eb5ec7975-re_builder.volatile`: use the whole
773            //   name. The first 11 bytes won't be enough to uniquely identify
774            //   it, but no obvious substring will, and this is a rarely used
775            //   option so it doesn't matter much.
776            //
777            assert_eq!(short.len(), 3);
778            let name = if let Some(index) = name.find("-cgu.") {
779                &name[index + 1..] // +1 skips the leading '-'.
780            } else {
781                name
782            };
783            format!("{short} {name}")
784        }
785
786        // Windows has no thread name length limit, so use more descriptive names.
787        #[cfg(windows)]
788        fn desc(_short: &str, long: &str, name: &str) -> String {
789            format!("{long} {name}")
790        }
791
792        match self {
793            WorkItem::Optimize(m) => desc("opt", "optimize module", &m.name),
794            WorkItem::CopyPostLtoArtifacts(m) => desc("cpy", "copy LTO artifacts for", &m.name),
795            WorkItem::LTO(m) => desc("lto", "LTO module", m.name()),
796        }
797    }
798}
799
800/// A result produced by the backend.
801pub(crate) enum WorkItemResult<B: WriteBackendMethods> {
802    /// The backend has finished compiling a CGU, nothing more required.
803    Finished(CompiledModule),
804
805    /// The backend has finished compiling a CGU, which now needs linking
806    /// because `-Zcombine-cgu` was specified.
807    NeedsLink(ModuleCodegen<B::Module>),
808
809    /// The backend has finished compiling a CGU, which now needs to go through
810    /// fat LTO.
811    NeedsFatLto(FatLtoInput<B>),
812
813    /// The backend has finished compiling a CGU, which now needs to go through
814    /// thin LTO.
815    NeedsThinLto(String, B::ThinBuffer),
816}
817
818pub enum FatLtoInput<B: WriteBackendMethods> {
819    Serialized { name: String, buffer: B::ModuleBuffer },
820    InMemory(ModuleCodegen<B::Module>),
821}
822
823/// Actual LTO type we end up choosing based on multiple factors.
824pub(crate) enum ComputedLtoType {
825    No,
826    Thin,
827    Fat,
828}
829
830pub(crate) fn compute_per_cgu_lto_type(
831    sess_lto: &Lto,
832    opts: &config::Options,
833    sess_crate_types: &[CrateType],
834    module_kind: ModuleKind,
835) -> ComputedLtoType {
836    // If the linker does LTO, we don't have to do it. Note that we
837    // keep doing full LTO, if it is requested, as not to break the
838    // assumption that the output will be a single module.
839    let linker_does_lto = opts.cg.linker_plugin_lto.enabled();
840
841    // When we're automatically doing ThinLTO for multi-codegen-unit
842    // builds we don't actually want to LTO the allocator modules if
843    // it shows up. This is due to various linker shenanigans that
844    // we'll encounter later.
845    let is_allocator = module_kind == ModuleKind::Allocator;
846
847    // We ignore a request for full crate graph LTO if the crate type
848    // is only an rlib, as there is no full crate graph to process,
849    // that'll happen later.
850    //
851    // This use case currently comes up primarily for targets that
852    // require LTO so the request for LTO is always unconditionally
853    // passed down to the backend, but we don't actually want to do
854    // anything about it yet until we've got a final product.
855    let is_rlib = matches!(sess_crate_types, [CrateType::Rlib]);
856
857    match sess_lto {
858        Lto::ThinLocal if !linker_does_lto && !is_allocator => ComputedLtoType::Thin,
859        Lto::Thin if !linker_does_lto && !is_rlib => ComputedLtoType::Thin,
860        Lto::Fat if !is_rlib => ComputedLtoType::Fat,
861        _ => ComputedLtoType::No,
862    }
863}
864
865fn execute_optimize_work_item<B: ExtraBackendMethods>(
866    cgcx: &CodegenContext<B>,
867    mut module: ModuleCodegen<B::Module>,
868    module_config: &ModuleConfig,
869) -> Result<WorkItemResult<B>, FatalError> {
870    let dcx = cgcx.create_dcx();
871    let dcx = dcx.handle();
872
873    B::optimize(cgcx, dcx, &mut module, module_config)?;
874
875    // After we've done the initial round of optimizations we need to
876    // decide whether to synchronously codegen this module or ship it
877    // back to the coordinator thread for further LTO processing (which
878    // has to wait for all the initial modules to be optimized).
879
880    let lto_type = compute_per_cgu_lto_type(&cgcx.lto, &cgcx.opts, &cgcx.crate_types, module.kind);
881
882    // If we're doing some form of incremental LTO then we need to be sure to
883    // save our module to disk first.
884    let bitcode = if cgcx.config(module.kind).emit_pre_lto_bc {
885        let filename = pre_lto_bitcode_filename(&module.name);
886        cgcx.incr_comp_session_dir.as_ref().map(|path| path.join(&filename))
887    } else {
888        None
889    };
890
891    match lto_type {
892        ComputedLtoType::No => finish_intra_module_work(cgcx, module, module_config),
893        ComputedLtoType::Thin => {
894            let (name, thin_buffer) = B::prepare_thin(module, false);
895            if let Some(path) = bitcode {
896                fs::write(&path, thin_buffer.data()).unwrap_or_else(|e| {
897                    panic!("Error writing pre-lto-bitcode file `{}`: {}", path.display(), e);
898                });
899            }
900            Ok(WorkItemResult::NeedsThinLto(name, thin_buffer))
901        }
902        ComputedLtoType::Fat => match bitcode {
903            Some(path) => {
904                let (name, buffer) = B::serialize_module(module);
905                fs::write(&path, buffer.data()).unwrap_or_else(|e| {
906                    panic!("Error writing pre-lto-bitcode file `{}`: {}", path.display(), e);
907                });
908                Ok(WorkItemResult::NeedsFatLto(FatLtoInput::Serialized { name, buffer }))
909            }
910            None => Ok(WorkItemResult::NeedsFatLto(FatLtoInput::InMemory(module))),
911        },
912    }
913}
914
915fn execute_copy_from_cache_work_item<B: ExtraBackendMethods>(
916    cgcx: &CodegenContext<B>,
917    module: CachedModuleCodegen,
918    module_config: &ModuleConfig,
919) -> WorkItemResult<B> {
920    let incr_comp_session_dir = cgcx.incr_comp_session_dir.as_ref().unwrap();
921
922    let mut links_from_incr_cache = Vec::new();
923
924    let mut load_from_incr_comp_dir = |output_path: PathBuf, saved_path: &str| {
925        let source_file = in_incr_comp_dir(incr_comp_session_dir, saved_path);
926        debug!(
927            "copying preexisting module `{}` from {:?} to {}",
928            module.name,
929            source_file,
930            output_path.display()
931        );
932        match link_or_copy(&source_file, &output_path) {
933            Ok(_) => {
934                links_from_incr_cache.push(source_file);
935                Some(output_path)
936            }
937            Err(error) => {
938                cgcx.create_dcx().handle().emit_err(errors::CopyPathBuf {
939                    source_file,
940                    output_path,
941                    error,
942                });
943                None
944            }
945        }
946    };
947
948    let dwarf_object =
949        module.source.saved_files.get("dwo").as_ref().and_then(|saved_dwarf_object_file| {
950            let dwarf_obj_out = cgcx
951                .output_filenames
952                .split_dwarf_path(
953                    cgcx.split_debuginfo,
954                    cgcx.split_dwarf_kind,
955                    &module.name,
956                    cgcx.invocation_temp.as_deref(),
957                )
958                .expect(
959                    "saved dwarf object in work product but `split_dwarf_path` returned `None`",
960                );
961            load_from_incr_comp_dir(dwarf_obj_out, saved_dwarf_object_file)
962        });
963
964    let mut load_from_incr_cache = |perform, output_type: OutputType| {
965        if perform {
966            let saved_file = module.source.saved_files.get(output_type.extension())?;
967            let output_path = cgcx.output_filenames.temp_path_for_cgu(
968                output_type,
969                &module.name,
970                cgcx.invocation_temp.as_deref(),
971            );
972            load_from_incr_comp_dir(output_path, &saved_file)
973        } else {
974            None
975        }
976    };
977
978    let should_emit_obj = module_config.emit_obj != EmitObj::None;
979    let assembly = load_from_incr_cache(module_config.emit_asm, OutputType::Assembly);
980    let llvm_ir = load_from_incr_cache(module_config.emit_ir, OutputType::LlvmAssembly);
981    let bytecode = load_from_incr_cache(module_config.emit_bc, OutputType::Bitcode);
982    let object = load_from_incr_cache(should_emit_obj, OutputType::Object);
983    if should_emit_obj && object.is_none() {
984        cgcx.create_dcx().handle().emit_fatal(errors::NoSavedObjectFile { cgu_name: &module.name })
985    }
986
987    WorkItemResult::Finished(CompiledModule {
988        links_from_incr_cache,
989        name: module.name,
990        kind: ModuleKind::Regular,
991        object,
992        dwarf_object,
993        bytecode,
994        assembly,
995        llvm_ir,
996    })
997}
998
999fn execute_lto_work_item<B: ExtraBackendMethods>(
1000    cgcx: &CodegenContext<B>,
1001    module: lto::LtoModuleCodegen<B>,
1002    module_config: &ModuleConfig,
1003) -> Result<WorkItemResult<B>, FatalError> {
1004    let module = module.optimize(cgcx)?;
1005    finish_intra_module_work(cgcx, module, module_config)
1006}
1007
1008fn finish_intra_module_work<B: ExtraBackendMethods>(
1009    cgcx: &CodegenContext<B>,
1010    module: ModuleCodegen<B::Module>,
1011    module_config: &ModuleConfig,
1012) -> Result<WorkItemResult<B>, FatalError> {
1013    let dcx = cgcx.create_dcx();
1014    let dcx = dcx.handle();
1015
1016    if !cgcx.opts.unstable_opts.combine_cgu || module.kind == ModuleKind::Allocator {
1017        let module = B::codegen(cgcx, dcx, module, module_config)?;
1018        Ok(WorkItemResult::Finished(module))
1019    } else {
1020        Ok(WorkItemResult::NeedsLink(module))
1021    }
1022}
1023
1024/// Messages sent to the coordinator.
1025pub(crate) enum Message<B: WriteBackendMethods> {
1026    /// A jobserver token has become available. Sent from the jobserver helper
1027    /// thread.
1028    Token(io::Result<Acquired>),
1029
1030    /// The backend has finished processing a work item for a codegen unit.
1031    /// Sent from a backend worker thread.
1032    WorkItem { result: Result<WorkItemResult<B>, Option<WorkerFatalError>>, worker_id: usize },
1033
1034    /// A vector containing all the AutoDiff tasks that we have to pass to Enzyme.
1035    AddAutoDiffItems(Vec<AutoDiffItem>),
1036
1037    /// The frontend has finished generating something (backend IR or a
1038    /// post-LTO artifact) for a codegen unit, and it should be passed to the
1039    /// backend. Sent from the main thread.
1040    CodegenDone { llvm_work_item: WorkItem<B>, cost: u64 },
1041
1042    /// Similar to `CodegenDone`, but for reusing a pre-LTO artifact
1043    /// Sent from the main thread.
1044    AddImportOnlyModule {
1045        module_data: SerializedModule<B::ModuleBuffer>,
1046        work_product: WorkProduct,
1047    },
1048
1049    /// The frontend has finished generating everything for all codegen units.
1050    /// Sent from the main thread.
1051    CodegenComplete,
1052
1053    /// Some normal-ish compiler error occurred, and codegen should be wound
1054    /// down. Sent from the main thread.
1055    CodegenAborted,
1056}
1057
1058/// A message sent from the coordinator thread to the main thread telling it to
1059/// process another codegen unit.
1060pub struct CguMessage;
1061
1062// A cut-down version of `rustc_errors::DiagInner` that impls `Send`, which
1063// can be used to send diagnostics from codegen threads to the main thread.
1064// It's missing the following fields from `rustc_errors::DiagInner`.
1065// - `span`: it doesn't impl `Send`.
1066// - `suggestions`: it doesn't impl `Send`, and isn't used for codegen
1067//   diagnostics.
1068// - `sort_span`: it doesn't impl `Send`.
1069// - `is_lint`: lints aren't relevant during codegen.
1070// - `emitted_at`: not used for codegen diagnostics.
1071struct Diagnostic {
1072    level: Level,
1073    messages: Vec<(DiagMessage, Style)>,
1074    code: Option<ErrCode>,
1075    children: Vec<Subdiagnostic>,
1076    args: DiagArgMap,
1077}
1078
1079// A cut-down version of `rustc_errors::Subdiag` that impls `Send`. It's
1080// missing the following fields from `rustc_errors::Subdiag`.
1081// - `span`: it doesn't impl `Send`.
1082pub(crate) struct Subdiagnostic {
1083    level: Level,
1084    messages: Vec<(DiagMessage, Style)>,
1085}
1086
1087#[derive(PartialEq, Clone, Copy, Debug)]
1088enum MainThreadState {
1089    /// Doing nothing.
1090    Idle,
1091
1092    /// Doing codegen, i.e. MIR-to-LLVM-IR conversion.
1093    Codegenning,
1094
1095    /// Idle, but lending the compiler process's Token to an LLVM thread so it can do useful work.
1096    Lending,
1097}
1098
1099fn start_executing_work<B: ExtraBackendMethods>(
1100    backend: B,
1101    tcx: TyCtxt<'_>,
1102    crate_info: &CrateInfo,
1103    shared_emitter: SharedEmitter,
1104    codegen_worker_send: Sender<CguMessage>,
1105    coordinator_receive: Receiver<Box<dyn Any + Send>>,
1106    regular_config: Arc<ModuleConfig>,
1107    allocator_config: Arc<ModuleConfig>,
1108    tx_to_llvm_workers: Sender<Box<dyn Any + Send>>,
1109) -> thread::JoinHandle<Result<CompiledModules, ()>> {
1110    let coordinator_send = tx_to_llvm_workers;
1111    let sess = tcx.sess;
1112
1113    let mut each_linked_rlib_for_lto = Vec::new();
1114    drop(link::each_linked_rlib(crate_info, None, &mut |cnum, path| {
1115        if link::ignored_for_lto(sess, crate_info, cnum) {
1116            return;
1117        }
1118        each_linked_rlib_for_lto.push((cnum, path.to_path_buf()));
1119    }));
1120
1121    // Compute the set of symbols we need to retain when doing LTO (if we need to)
1122    let exported_symbols = {
1123        let mut exported_symbols = FxHashMap::default();
1124
1125        let copy_symbols = |cnum| {
1126            let symbols = tcx
1127                .exported_non_generic_symbols(cnum)
1128                .iter()
1129                .chain(tcx.exported_generic_symbols(cnum))
1130                .map(|&(s, lvl)| (symbol_name_for_instance_in_crate(tcx, s, cnum), lvl))
1131                .collect();
1132            Arc::new(symbols)
1133        };
1134
1135        match sess.lto() {
1136            Lto::No => None,
1137            Lto::ThinLocal => {
1138                exported_symbols.insert(LOCAL_CRATE, copy_symbols(LOCAL_CRATE));
1139                Some(Arc::new(exported_symbols))
1140            }
1141            Lto::Fat | Lto::Thin => {
1142                exported_symbols.insert(LOCAL_CRATE, copy_symbols(LOCAL_CRATE));
1143                for &(cnum, ref _path) in &each_linked_rlib_for_lto {
1144                    exported_symbols.insert(cnum, copy_symbols(cnum));
1145                }
1146                Some(Arc::new(exported_symbols))
1147            }
1148        }
1149    };
1150
1151    // First up, convert our jobserver into a helper thread so we can use normal
1152    // mpsc channels to manage our messages and such.
1153    // After we've requested tokens then we'll, when we can,
1154    // get tokens on `coordinator_receive` which will
1155    // get managed in the main loop below.
1156    let coordinator_send2 = coordinator_send.clone();
1157    let helper = jobserver::client()
1158        .into_helper_thread(move |token| {
1159            drop(coordinator_send2.send(Box::new(Message::Token::<B>(token))));
1160        })
1161        .expect("failed to spawn helper thread");
1162
1163    let ol =
1164        if tcx.sess.opts.unstable_opts.no_codegen || !tcx.sess.opts.output_types.should_codegen() {
1165            // If we know that we won’t be doing codegen, create target machines without optimisation.
1166            config::OptLevel::No
1167        } else {
1168            tcx.backend_optimization_level(())
1169        };
1170    let backend_features = tcx.global_backend_features(());
1171
1172    let remark_dir = if let Some(ref dir) = sess.opts.unstable_opts.remark_dir {
1173        let result = fs::create_dir_all(dir).and_then(|_| dir.canonicalize());
1174        match result {
1175            Ok(dir) => Some(dir),
1176            Err(error) => sess.dcx().emit_fatal(ErrorCreatingRemarkDir { error }),
1177        }
1178    } else {
1179        None
1180    };
1181
1182    let cgcx = CodegenContext::<B> {
1183        crate_types: tcx.crate_types().to_vec(),
1184        each_linked_rlib_for_lto,
1185        lto: sess.lto(),
1186        fewer_names: sess.fewer_names(),
1187        save_temps: sess.opts.cg.save_temps,
1188        time_trace: sess.opts.unstable_opts.llvm_time_trace,
1189        opts: Arc::new(sess.opts.clone()),
1190        prof: sess.prof.clone(),
1191        exported_symbols,
1192        remark: sess.opts.cg.remark.clone(),
1193        remark_dir,
1194        incr_comp_session_dir: sess.incr_comp_session_dir_opt().map(|r| r.clone()),
1195        coordinator_send,
1196        expanded_args: tcx.sess.expanded_args.clone(),
1197        diag_emitter: shared_emitter.clone(),
1198        output_filenames: Arc::clone(tcx.output_filenames(())),
1199        regular_module_config: regular_config,
1200        allocator_module_config: allocator_config,
1201        tm_factory: backend.target_machine_factory(tcx.sess, ol, backend_features),
1202        msvc_imps_needed: msvc_imps_needed(tcx),
1203        is_pe_coff: tcx.sess.target.is_like_windows,
1204        target_can_use_split_dwarf: tcx.sess.target_can_use_split_dwarf(),
1205        target_arch: tcx.sess.target.arch.to_string(),
1206        target_is_like_darwin: tcx.sess.target.is_like_darwin,
1207        target_is_like_aix: tcx.sess.target.is_like_aix,
1208        split_debuginfo: tcx.sess.split_debuginfo(),
1209        split_dwarf_kind: tcx.sess.opts.unstable_opts.split_dwarf_kind,
1210        parallel: backend.supports_parallel() && !sess.opts.unstable_opts.no_parallel_backend,
1211        pointer_size: tcx.data_layout.pointer_size(),
1212        invocation_temp: sess.invocation_temp.clone(),
1213    };
1214
1215    // This is the "main loop" of parallel work happening for parallel codegen.
1216    // It's here that we manage parallelism, schedule work, and work with
1217    // messages coming from clients.
1218    //
1219    // There are a few environmental pre-conditions that shape how the system
1220    // is set up:
1221    //
1222    // - Error reporting can only happen on the main thread because that's the
1223    //   only place where we have access to the compiler `Session`.
1224    // - LLVM work can be done on any thread.
1225    // - Codegen can only happen on the main thread.
1226    // - Each thread doing substantial work must be in possession of a `Token`
1227    //   from the `Jobserver`.
1228    // - The compiler process always holds one `Token`. Any additional `Tokens`
1229    //   have to be requested from the `Jobserver`.
1230    //
1231    // Error Reporting
1232    // ===============
1233    // The error reporting restriction is handled separately from the rest: We
1234    // set up a `SharedEmitter` that holds an open channel to the main thread.
1235    // When an error occurs on any thread, the shared emitter will send the
1236    // error message to the receiver main thread (`SharedEmitterMain`). The
1237    // main thread will periodically query this error message queue and emit
1238    // any error messages it has received. It might even abort compilation if
1239    // it has received a fatal error. In this case we rely on all other threads
1240    // being torn down automatically with the main thread.
1241    // Since the main thread will often be busy doing codegen work, error
1242    // reporting will be somewhat delayed, since the message queue can only be
1243    // checked in between two work packages.
1244    //
1245    // Work Processing Infrastructure
1246    // ==============================
1247    // The work processing infrastructure knows three major actors:
1248    //
1249    // - the coordinator thread,
1250    // - the main thread, and
1251    // - LLVM worker threads
1252    //
1253    // The coordinator thread is running a message loop. It instructs the main
1254    // thread about what work to do when, and it will spawn off LLVM worker
1255    // threads as open LLVM WorkItems become available.
1256    //
1257    // The job of the main thread is to codegen CGUs into LLVM work packages
1258    // (since the main thread is the only thread that can do this). The main
1259    // thread will block until it receives a message from the coordinator, upon
1260    // which it will codegen one CGU, send it to the coordinator and block
1261    // again. This way the coordinator can control what the main thread is
1262    // doing.
1263    //
1264    // The coordinator keeps a queue of LLVM WorkItems, and when a `Token` is
1265    // available, it will spawn off a new LLVM worker thread and let it process
1266    // a WorkItem. When a LLVM worker thread is done with its WorkItem,
1267    // it will just shut down, which also frees all resources associated with
1268    // the given LLVM module, and sends a message to the coordinator that the
1269    // WorkItem has been completed.
1270    //
1271    // Work Scheduling
1272    // ===============
1273    // The scheduler's goal is to minimize the time it takes to complete all
1274    // work there is, however, we also want to keep memory consumption low
1275    // if possible. These two goals are at odds with each other: If memory
1276    // consumption were not an issue, we could just let the main thread produce
1277    // LLVM WorkItems at full speed, assuring maximal utilization of
1278    // Tokens/LLVM worker threads. However, since codegen is usually faster
1279    // than LLVM processing, the queue of LLVM WorkItems would fill up and each
1280    // WorkItem potentially holds on to a substantial amount of memory.
1281    //
1282    // So the actual goal is to always produce just enough LLVM WorkItems as
1283    // not to starve our LLVM worker threads. That means, once we have enough
1284    // WorkItems in our queue, we can block the main thread, so it does not
1285    // produce more until we need them.
1286    //
1287    // Doing LLVM Work on the Main Thread
1288    // ----------------------------------
1289    // Since the main thread owns the compiler process's implicit `Token`, it is
1290    // wasteful to keep it blocked without doing any work. Therefore, what we do
1291    // in this case is: We spawn off an additional LLVM worker thread that helps
1292    // reduce the queue. The work it is doing corresponds to the implicit
1293    // `Token`. The coordinator will mark the main thread as being busy with
1294    // LLVM work. (The actual work happens on another OS thread but we just care
1295    // about `Tokens`, not actual threads).
1296    //
1297    // When any LLVM worker thread finishes while the main thread is marked as
1298    // "busy with LLVM work", we can do a little switcheroo: We give the Token
1299    // of the just finished thread to the LLVM worker thread that is working on
1300    // behalf of the main thread's implicit Token, thus freeing up the main
1301    // thread again. The coordinator can then again decide what the main thread
1302    // should do. This allows the coordinator to make decisions at more points
1303    // in time.
1304    //
1305    // Striking a Balance between Throughput and Memory Consumption
1306    // ------------------------------------------------------------
1307    // Since our two goals, (1) use as many Tokens as possible and (2) keep
1308    // memory consumption as low as possible, are in conflict with each other,
1309    // we have to find a trade off between them. Right now, the goal is to keep
1310    // all workers busy, which means that no worker should find the queue empty
1311    // when it is ready to start.
1312    // How do we do achieve this? Good question :) We actually never know how
1313    // many `Tokens` are potentially available so it's hard to say how much to
1314    // fill up the queue before switching the main thread to LLVM work. Also we
1315    // currently don't have a means to estimate how long a running LLVM worker
1316    // will still be busy with it's current WorkItem. However, we know the
1317    // maximal count of available Tokens that makes sense (=the number of CPU
1318    // cores), so we can take a conservative guess. The heuristic we use here
1319    // is implemented in the `queue_full_enough()` function.
1320    //
1321    // Some Background on Jobservers
1322    // -----------------------------
1323    // It's worth also touching on the management of parallelism here. We don't
1324    // want to just spawn a thread per work item because while that's optimal
1325    // parallelism it may overload a system with too many threads or violate our
1326    // configuration for the maximum amount of cpu to use for this process. To
1327    // manage this we use the `jobserver` crate.
1328    //
1329    // Job servers are an artifact of GNU make and are used to manage
1330    // parallelism between processes. A jobserver is a glorified IPC semaphore
1331    // basically. Whenever we want to run some work we acquire the semaphore,
1332    // and whenever we're done with that work we release the semaphore. In this
1333    // manner we can ensure that the maximum number of parallel workers is
1334    // capped at any one point in time.
1335    //
1336    // LTO and the coordinator thread
1337    // ------------------------------
1338    //
1339    // The final job the coordinator thread is responsible for is managing LTO
1340    // and how that works. When LTO is requested what we'll do is collect all
1341    // optimized LLVM modules into a local vector on the coordinator. Once all
1342    // modules have been codegened and optimized we hand this to the `lto`
1343    // module for further optimization. The `lto` module will return back a list
1344    // of more modules to work on, which the coordinator will continue to spawn
1345    // work for.
1346    //
1347    // Each LLVM module is automatically sent back to the coordinator for LTO if
1348    // necessary. There's already optimizations in place to avoid sending work
1349    // back to the coordinator if LTO isn't requested.
1350    return B::spawn_named_thread(cgcx.time_trace, "coordinator".to_string(), move || {
1351        let mut worker_id_counter = 0;
1352        let mut free_worker_ids = Vec::new();
1353        let mut get_worker_id = |free_worker_ids: &mut Vec<usize>| {
1354            if let Some(id) = free_worker_ids.pop() {
1355                id
1356            } else {
1357                let id = worker_id_counter;
1358                worker_id_counter += 1;
1359                id
1360            }
1361        };
1362
1363        // This is where we collect codegen units that have gone all the way
1364        // through codegen and LLVM.
1365        let mut autodiff_items = Vec::new();
1366        let mut compiled_modules = vec![];
1367        let mut compiled_allocator_module = None;
1368        let mut needs_link = Vec::new();
1369        let mut needs_fat_lto = Vec::new();
1370        let mut needs_thin_lto = Vec::new();
1371        let mut lto_import_only_modules = Vec::new();
1372        let mut started_lto = false;
1373
1374        /// Possible state transitions:
1375        /// - Ongoing -> Completed
1376        /// - Ongoing -> Aborted
1377        /// - Completed -> Aborted
1378        #[derive(Debug, PartialEq)]
1379        enum CodegenState {
1380            Ongoing,
1381            Completed,
1382            Aborted,
1383        }
1384        use CodegenState::*;
1385        let mut codegen_state = Ongoing;
1386
1387        // This is the queue of LLVM work items that still need processing.
1388        let mut work_items = Vec::<(WorkItem<B>, u64)>::new();
1389
1390        // This are the Jobserver Tokens we currently hold. Does not include
1391        // the implicit Token the compiler process owns no matter what.
1392        let mut tokens = Vec::new();
1393
1394        let mut main_thread_state = MainThreadState::Idle;
1395
1396        // How many LLVM worker threads are running while holding a Token. This
1397        // *excludes* any that the main thread is lending a Token to.
1398        let mut running_with_own_token = 0;
1399
1400        // How many LLVM worker threads are running in total. This *includes*
1401        // any that the main thread is lending a Token to.
1402        let running_with_any_token = |main_thread_state, running_with_own_token| {
1403            running_with_own_token
1404                + if main_thread_state == MainThreadState::Lending { 1 } else { 0 }
1405        };
1406
1407        let mut llvm_start_time: Option<VerboseTimingGuard<'_>> = None;
1408
1409        // Run the message loop while there's still anything that needs message
1410        // processing. Note that as soon as codegen is aborted we simply want to
1411        // wait for all existing work to finish, so many of the conditions here
1412        // only apply if codegen hasn't been aborted as they represent pending
1413        // work to be done.
1414        loop {
1415            // While there are still CGUs to be codegened, the coordinator has
1416            // to decide how to utilize the compiler processes implicit Token:
1417            // For codegenning more CGU or for running them through LLVM.
1418            if codegen_state == Ongoing {
1419                if main_thread_state == MainThreadState::Idle {
1420                    // Compute the number of workers that will be running once we've taken as many
1421                    // items from the work queue as we can, plus one for the main thread. It's not
1422                    // critically important that we use this instead of just
1423                    // `running_with_own_token`, but it prevents the `queue_full_enough` heuristic
1424                    // from fluctuating just because a worker finished up and we decreased the
1425                    // `running_with_own_token` count, even though we're just going to increase it
1426                    // right after this when we put a new worker to work.
1427                    let extra_tokens = tokens.len().checked_sub(running_with_own_token).unwrap();
1428                    let additional_running = std::cmp::min(extra_tokens, work_items.len());
1429                    let anticipated_running = running_with_own_token + additional_running + 1;
1430
1431                    if !queue_full_enough(work_items.len(), anticipated_running) {
1432                        // The queue is not full enough, process more codegen units:
1433                        if codegen_worker_send.send(CguMessage).is_err() {
1434                            panic!("Could not send CguMessage to main thread")
1435                        }
1436                        main_thread_state = MainThreadState::Codegenning;
1437                    } else {
1438                        // The queue is full enough to not let the worker
1439                        // threads starve. Use the implicit Token to do some
1440                        // LLVM work too.
1441                        let (item, _) =
1442                            work_items.pop().expect("queue empty - queue_full_enough() broken?");
1443                        main_thread_state = MainThreadState::Lending;
1444                        spawn_work(
1445                            &cgcx,
1446                            &mut llvm_start_time,
1447                            get_worker_id(&mut free_worker_ids),
1448                            item,
1449                        );
1450                    }
1451                }
1452            } else if codegen_state == Completed {
1453                if running_with_any_token(main_thread_state, running_with_own_token) == 0
1454                    && work_items.is_empty()
1455                {
1456                    // All codegen work is done. Do we have LTO work to do?
1457                    if needs_fat_lto.is_empty()
1458                        && needs_thin_lto.is_empty()
1459                        && lto_import_only_modules.is_empty()
1460                    {
1461                        // Nothing more to do!
1462                        break;
1463                    }
1464
1465                    // We have LTO work to do. Perform the serial work here of
1466                    // figuring out what we're going to LTO and then push a
1467                    // bunch of work items onto our queue to do LTO. This all
1468                    // happens on the coordinator thread but it's very quick so
1469                    // we don't worry about tokens.
1470                    assert!(!started_lto);
1471                    started_lto = true;
1472
1473                    let needs_fat_lto = mem::take(&mut needs_fat_lto);
1474                    let needs_thin_lto = mem::take(&mut needs_thin_lto);
1475                    let import_only_modules = mem::take(&mut lto_import_only_modules);
1476
1477                    for (work, cost) in generate_lto_work(
1478                        &cgcx,
1479                        autodiff_items.clone(),
1480                        needs_fat_lto,
1481                        needs_thin_lto,
1482                        import_only_modules,
1483                    ) {
1484                        let insertion_index = work_items
1485                            .binary_search_by_key(&cost, |&(_, cost)| cost)
1486                            .unwrap_or_else(|e| e);
1487                        work_items.insert(insertion_index, (work, cost));
1488                        if cgcx.parallel {
1489                            helper.request_token();
1490                        }
1491                    }
1492                }
1493
1494                // In this branch, we know that everything has been codegened,
1495                // so it's just a matter of determining whether the implicit
1496                // Token is free to use for LLVM work.
1497                match main_thread_state {
1498                    MainThreadState::Idle => {
1499                        if let Some((item, _)) = work_items.pop() {
1500                            main_thread_state = MainThreadState::Lending;
1501                            spawn_work(
1502                                &cgcx,
1503                                &mut llvm_start_time,
1504                                get_worker_id(&mut free_worker_ids),
1505                                item,
1506                            );
1507                        } else {
1508                            // There is no unstarted work, so let the main thread
1509                            // take over for a running worker. Otherwise the
1510                            // implicit token would just go to waste.
1511                            // We reduce the `running` counter by one. The
1512                            // `tokens.truncate()` below will take care of
1513                            // giving the Token back.
1514                            assert!(running_with_own_token > 0);
1515                            running_with_own_token -= 1;
1516                            main_thread_state = MainThreadState::Lending;
1517                        }
1518                    }
1519                    MainThreadState::Codegenning => bug!(
1520                        "codegen worker should not be codegenning after \
1521                              codegen was already completed"
1522                    ),
1523                    MainThreadState::Lending => {
1524                        // Already making good use of that token
1525                    }
1526                }
1527            } else {
1528                // Don't queue up any more work if codegen was aborted, we're
1529                // just waiting for our existing children to finish.
1530                assert!(codegen_state == Aborted);
1531                if running_with_any_token(main_thread_state, running_with_own_token) == 0 {
1532                    break;
1533                }
1534            }
1535
1536            // Spin up what work we can, only doing this while we've got available
1537            // parallelism slots and work left to spawn.
1538            if codegen_state != Aborted {
1539                while running_with_own_token < tokens.len()
1540                    && let Some((item, _)) = work_items.pop()
1541                {
1542                    spawn_work(
1543                        &cgcx,
1544                        &mut llvm_start_time,
1545                        get_worker_id(&mut free_worker_ids),
1546                        item,
1547                    );
1548                    running_with_own_token += 1;
1549                }
1550            }
1551
1552            // Relinquish accidentally acquired extra tokens.
1553            tokens.truncate(running_with_own_token);
1554
1555            // If a thread exits successfully then we drop a token associated
1556            // with that worker and update our `running_with_own_token` count.
1557            // We may later re-acquire a token to continue running more work.
1558            // We may also not actually drop a token here if the worker was
1559            // running with an "ephemeral token".
1560            let mut free_worker = |worker_id| {
1561                if main_thread_state == MainThreadState::Lending {
1562                    main_thread_state = MainThreadState::Idle;
1563                } else {
1564                    running_with_own_token -= 1;
1565                }
1566
1567                free_worker_ids.push(worker_id);
1568            };
1569
1570            let msg = coordinator_receive.recv().unwrap();
1571            match *msg.downcast::<Message<B>>().ok().unwrap() {
1572                // Save the token locally and the next turn of the loop will use
1573                // this to spawn a new unit of work, or it may get dropped
1574                // immediately if we have no more work to spawn.
1575                Message::Token(token) => {
1576                    match token {
1577                        Ok(token) => {
1578                            tokens.push(token);
1579
1580                            if main_thread_state == MainThreadState::Lending {
1581                                // If the main thread token is used for LLVM work
1582                                // at the moment, we turn that thread into a regular
1583                                // LLVM worker thread, so the main thread is free
1584                                // to react to codegen demand.
1585                                main_thread_state = MainThreadState::Idle;
1586                                running_with_own_token += 1;
1587                            }
1588                        }
1589                        Err(e) => {
1590                            let msg = &format!("failed to acquire jobserver token: {e}");
1591                            shared_emitter.fatal(msg);
1592                            codegen_state = Aborted;
1593                        }
1594                    }
1595                }
1596
1597                Message::CodegenDone { llvm_work_item, cost } => {
1598                    // We keep the queue sorted by estimated processing cost,
1599                    // so that more expensive items are processed earlier. This
1600                    // is good for throughput as it gives the main thread more
1601                    // time to fill up the queue and it avoids scheduling
1602                    // expensive items to the end.
1603                    // Note, however, that this is not ideal for memory
1604                    // consumption, as LLVM module sizes are not evenly
1605                    // distributed.
1606                    let insertion_index = work_items.binary_search_by_key(&cost, |&(_, cost)| cost);
1607                    let insertion_index = match insertion_index {
1608                        Ok(idx) | Err(idx) => idx,
1609                    };
1610                    work_items.insert(insertion_index, (llvm_work_item, cost));
1611
1612                    if cgcx.parallel {
1613                        helper.request_token();
1614                    }
1615                    assert_eq!(main_thread_state, MainThreadState::Codegenning);
1616                    main_thread_state = MainThreadState::Idle;
1617                }
1618
1619                Message::AddAutoDiffItems(mut items) => {
1620                    autodiff_items.append(&mut items);
1621                }
1622
1623                Message::CodegenComplete => {
1624                    if codegen_state != Aborted {
1625                        codegen_state = Completed;
1626                    }
1627                    assert_eq!(main_thread_state, MainThreadState::Codegenning);
1628                    main_thread_state = MainThreadState::Idle;
1629                }
1630
1631                // If codegen is aborted that means translation was aborted due
1632                // to some normal-ish compiler error. In this situation we want
1633                // to exit as soon as possible, but we want to make sure all
1634                // existing work has finished. Flag codegen as being done, and
1635                // then conditions above will ensure no more work is spawned but
1636                // we'll keep executing this loop until `running_with_own_token`
1637                // hits 0.
1638                Message::CodegenAborted => {
1639                    codegen_state = Aborted;
1640                }
1641
1642                Message::WorkItem { result, worker_id } => {
1643                    free_worker(worker_id);
1644
1645                    match result {
1646                        Ok(WorkItemResult::Finished(compiled_module)) => {
1647                            match compiled_module.kind {
1648                                ModuleKind::Regular => {
1649                                    assert!(needs_link.is_empty());
1650                                    compiled_modules.push(compiled_module);
1651                                }
1652                                ModuleKind::Allocator => {
1653                                    assert!(compiled_allocator_module.is_none());
1654                                    compiled_allocator_module = Some(compiled_module);
1655                                }
1656                            }
1657                        }
1658                        Ok(WorkItemResult::NeedsLink(module)) => {
1659                            assert!(compiled_modules.is_empty());
1660                            needs_link.push(module);
1661                        }
1662                        Ok(WorkItemResult::NeedsFatLto(fat_lto_input)) => {
1663                            assert!(!started_lto);
1664                            assert!(needs_thin_lto.is_empty());
1665                            needs_fat_lto.push(fat_lto_input);
1666                        }
1667                        Ok(WorkItemResult::NeedsThinLto(name, thin_buffer)) => {
1668                            assert!(!started_lto);
1669                            assert!(needs_fat_lto.is_empty());
1670                            needs_thin_lto.push((name, thin_buffer));
1671                        }
1672                        Err(Some(WorkerFatalError)) => {
1673                            // Like `CodegenAborted`, wait for remaining work to finish.
1674                            codegen_state = Aborted;
1675                        }
1676                        Err(None) => {
1677                            // If the thread failed that means it panicked, so
1678                            // we abort immediately.
1679                            bug!("worker thread panicked");
1680                        }
1681                    }
1682                }
1683
1684                Message::AddImportOnlyModule { module_data, work_product } => {
1685                    assert!(!started_lto);
1686                    assert_eq!(codegen_state, Ongoing);
1687                    assert_eq!(main_thread_state, MainThreadState::Codegenning);
1688                    lto_import_only_modules.push((module_data, work_product));
1689                    main_thread_state = MainThreadState::Idle;
1690                }
1691            }
1692        }
1693
1694        if codegen_state == Aborted {
1695            return Err(());
1696        }
1697
1698        let needs_link = mem::take(&mut needs_link);
1699        if !needs_link.is_empty() {
1700            assert!(compiled_modules.is_empty());
1701            let dcx = cgcx.create_dcx();
1702            let dcx = dcx.handle();
1703            let module = B::run_link(&cgcx, dcx, needs_link).map_err(|_| ())?;
1704            let module =
1705                B::codegen(&cgcx, dcx, module, cgcx.config(ModuleKind::Regular)).map_err(|_| ())?;
1706            compiled_modules.push(module);
1707        }
1708
1709        // Drop to print timings
1710        drop(llvm_start_time);
1711
1712        // Regardless of what order these modules completed in, report them to
1713        // the backend in the same order every time to ensure that we're handing
1714        // out deterministic results.
1715        compiled_modules.sort_by(|a, b| a.name.cmp(&b.name));
1716
1717        Ok(CompiledModules {
1718            modules: compiled_modules,
1719            allocator_module: compiled_allocator_module,
1720        })
1721    })
1722    .expect("failed to spawn coordinator thread");
1723
1724    // A heuristic that determines if we have enough LLVM WorkItems in the
1725    // queue so that the main thread can do LLVM work instead of codegen
1726    fn queue_full_enough(items_in_queue: usize, workers_running: usize) -> bool {
1727        // This heuristic scales ahead-of-time codegen according to available
1728        // concurrency, as measured by `workers_running`. The idea is that the
1729        // more concurrency we have available, the more demand there will be for
1730        // work items, and the fuller the queue should be kept to meet demand.
1731        // An important property of this approach is that we codegen ahead of
1732        // time only as much as necessary, so as to keep fewer LLVM modules in
1733        // memory at once, thereby reducing memory consumption.
1734        //
1735        // When the number of workers running is less than the max concurrency
1736        // available to us, this heuristic can cause us to instruct the main
1737        // thread to work on an LLVM item (that is, tell it to "LLVM") instead
1738        // of codegen, even though it seems like it *should* be codegenning so
1739        // that we can create more work items and spawn more LLVM workers.
1740        //
1741        // But this is not a problem. When the main thread is told to LLVM,
1742        // according to this heuristic and how work is scheduled, there is
1743        // always at least one item in the queue, and therefore at least one
1744        // pending jobserver token request. If there *is* more concurrency
1745        // available, we will immediately receive a token, which will upgrade
1746        // the main thread's LLVM worker to a real one (conceptually), and free
1747        // up the main thread to codegen if necessary. On the other hand, if
1748        // there isn't more concurrency, then the main thread working on an LLVM
1749        // item is appropriate, as long as the queue is full enough for demand.
1750        //
1751        // Speaking of which, how full should we keep the queue? Probably less
1752        // full than you'd think. A lot has to go wrong for the queue not to be
1753        // full enough and for that to have a negative effect on compile times.
1754        //
1755        // Workers are unlikely to finish at exactly the same time, so when one
1756        // finishes and takes another work item off the queue, we often have
1757        // ample time to codegen at that point before the next worker finishes.
1758        // But suppose that codegen takes so long that the workers exhaust the
1759        // queue, and we have one or more workers that have nothing to work on.
1760        // Well, it might not be so bad. Of all the LLVM modules we create and
1761        // optimize, one has to finish last. It's not necessarily the case that
1762        // by losing some concurrency for a moment, we delay the point at which
1763        // that last LLVM module is finished and the rest of compilation can
1764        // proceed. Also, when we can't take advantage of some concurrency, we
1765        // give tokens back to the job server. That enables some other rustc to
1766        // potentially make use of the available concurrency. That could even
1767        // *decrease* overall compile time if we're lucky. But yes, if no other
1768        // rustc can make use of the concurrency, then we've squandered it.
1769        //
1770        // However, keeping the queue full is also beneficial when we have a
1771        // surge in available concurrency. Then items can be taken from the
1772        // queue immediately, without having to wait for codegen.
1773        //
1774        // So, the heuristic below tries to keep one item in the queue for every
1775        // four running workers. Based on limited benchmarking, this appears to
1776        // be more than sufficient to avoid increasing compilation times.
1777        let quarter_of_workers = workers_running - 3 * workers_running / 4;
1778        items_in_queue > 0 && items_in_queue >= quarter_of_workers
1779    }
1780}
1781
1782/// `FatalError` is explicitly not `Send`.
1783#[must_use]
1784pub(crate) struct WorkerFatalError;
1785
1786fn spawn_work<'a, B: ExtraBackendMethods>(
1787    cgcx: &'a CodegenContext<B>,
1788    llvm_start_time: &mut Option<VerboseTimingGuard<'a>>,
1789    worker_id: usize,
1790    work: WorkItem<B>,
1791) {
1792    if cgcx.config(work.module_kind()).time_module && llvm_start_time.is_none() {
1793        *llvm_start_time = Some(cgcx.prof.verbose_generic_activity("LLVM_passes"));
1794    }
1795
1796    let cgcx = cgcx.clone();
1797
1798    B::spawn_named_thread(cgcx.time_trace, work.short_description(), move || {
1799        // Set up a destructor which will fire off a message that we're done as
1800        // we exit.
1801        struct Bomb<B: ExtraBackendMethods> {
1802            coordinator_send: Sender<Box<dyn Any + Send>>,
1803            result: Option<Result<WorkItemResult<B>, FatalError>>,
1804            worker_id: usize,
1805        }
1806        impl<B: ExtraBackendMethods> Drop for Bomb<B> {
1807            fn drop(&mut self) {
1808                let worker_id = self.worker_id;
1809                let msg = match self.result.take() {
1810                    Some(Ok(result)) => Message::WorkItem::<B> { result: Ok(result), worker_id },
1811                    Some(Err(FatalError)) => {
1812                        Message::WorkItem::<B> { result: Err(Some(WorkerFatalError)), worker_id }
1813                    }
1814                    None => Message::WorkItem::<B> { result: Err(None), worker_id },
1815                };
1816                drop(self.coordinator_send.send(Box::new(msg)));
1817            }
1818        }
1819
1820        let mut bomb =
1821            Bomb::<B> { coordinator_send: cgcx.coordinator_send.clone(), result: None, worker_id };
1822
1823        // Execute the work itself, and if it finishes successfully then flag
1824        // ourselves as a success as well.
1825        //
1826        // Note that we ignore any `FatalError` coming out of `execute_work_item`,
1827        // as a diagnostic was already sent off to the main thread - just
1828        // surface that there was an error in this worker.
1829        bomb.result = {
1830            let module_config = cgcx.config(work.module_kind());
1831
1832            Some(match work {
1833                WorkItem::Optimize(m) => {
1834                    let _timer =
1835                        cgcx.prof.generic_activity_with_arg("codegen_module_optimize", &*m.name);
1836                    execute_optimize_work_item(&cgcx, m, module_config)
1837                }
1838                WorkItem::CopyPostLtoArtifacts(m) => {
1839                    let _timer = cgcx.prof.generic_activity_with_arg(
1840                        "codegen_copy_artifacts_from_incr_cache",
1841                        &*m.name,
1842                    );
1843                    Ok(execute_copy_from_cache_work_item(&cgcx, m, module_config))
1844                }
1845                WorkItem::LTO(m) => {
1846                    let _timer =
1847                        cgcx.prof.generic_activity_with_arg("codegen_module_perform_lto", m.name());
1848                    execute_lto_work_item(&cgcx, m, module_config)
1849                }
1850            })
1851        };
1852    })
1853    .expect("failed to spawn work thread");
1854}
1855
1856enum SharedEmitterMessage {
1857    Diagnostic(Diagnostic),
1858    InlineAsmError(SpanData, String, Level, Option<(String, Vec<InnerSpan>)>),
1859    Fatal(String),
1860}
1861
1862#[derive(Clone)]
1863pub struct SharedEmitter {
1864    sender: Sender<SharedEmitterMessage>,
1865}
1866
1867pub struct SharedEmitterMain {
1868    receiver: Receiver<SharedEmitterMessage>,
1869}
1870
1871impl SharedEmitter {
1872    fn new() -> (SharedEmitter, SharedEmitterMain) {
1873        let (sender, receiver) = channel();
1874
1875        (SharedEmitter { sender }, SharedEmitterMain { receiver })
1876    }
1877
1878    pub fn inline_asm_error(
1879        &self,
1880        span: SpanData,
1881        msg: String,
1882        level: Level,
1883        source: Option<(String, Vec<InnerSpan>)>,
1884    ) {
1885        drop(self.sender.send(SharedEmitterMessage::InlineAsmError(span, msg, level, source)));
1886    }
1887
1888    fn fatal(&self, msg: &str) {
1889        drop(self.sender.send(SharedEmitterMessage::Fatal(msg.to_string())));
1890    }
1891}
1892
1893impl Emitter for SharedEmitter {
1894    fn emit_diagnostic(
1895        &mut self,
1896        mut diag: rustc_errors::DiagInner,
1897        _registry: &rustc_errors::registry::Registry,
1898    ) {
1899        // Check that we aren't missing anything interesting when converting to
1900        // the cut-down local `DiagInner`.
1901        assert_eq!(diag.span, MultiSpan::new());
1902        assert_eq!(diag.suggestions, Suggestions::Enabled(vec![]));
1903        assert_eq!(diag.sort_span, rustc_span::DUMMY_SP);
1904        assert_eq!(diag.is_lint, None);
1905        // No sensible check for `diag.emitted_at`.
1906
1907        let args = mem::replace(&mut diag.args, DiagArgMap::default());
1908        drop(
1909            self.sender.send(SharedEmitterMessage::Diagnostic(Diagnostic {
1910                level: diag.level(),
1911                messages: diag.messages,
1912                code: diag.code,
1913                children: diag
1914                    .children
1915                    .into_iter()
1916                    .map(|child| Subdiagnostic { level: child.level, messages: child.messages })
1917                    .collect(),
1918                args,
1919            })),
1920        );
1921    }
1922
1923    fn source_map(&self) -> Option<&SourceMap> {
1924        None
1925    }
1926
1927    fn translator(&self) -> &Translator {
1928        panic!("shared emitter attempted to translate a diagnostic");
1929    }
1930}
1931
1932impl SharedEmitterMain {
1933    fn check(&self, sess: &Session, blocking: bool) {
1934        loop {
1935            let message = if blocking {
1936                match self.receiver.recv() {
1937                    Ok(message) => Ok(message),
1938                    Err(_) => Err(()),
1939                }
1940            } else {
1941                match self.receiver.try_recv() {
1942                    Ok(message) => Ok(message),
1943                    Err(_) => Err(()),
1944                }
1945            };
1946
1947            match message {
1948                Ok(SharedEmitterMessage::Diagnostic(diag)) => {
1949                    // The diagnostic has been received on the main thread.
1950                    // Convert it back to a full `Diagnostic` and emit.
1951                    let dcx = sess.dcx();
1952                    let mut d =
1953                        rustc_errors::DiagInner::new_with_messages(diag.level, diag.messages);
1954                    d.code = diag.code; // may be `None`, that's ok
1955                    d.children = diag
1956                        .children
1957                        .into_iter()
1958                        .map(|sub| rustc_errors::Subdiag {
1959                            level: sub.level,
1960                            messages: sub.messages,
1961                            span: MultiSpan::new(),
1962                        })
1963                        .collect();
1964                    d.args = diag.args;
1965                    dcx.emit_diagnostic(d);
1966                    sess.dcx().abort_if_errors();
1967                }
1968                Ok(SharedEmitterMessage::InlineAsmError(span, msg, level, source)) => {
1969                    assert_matches!(level, Level::Error | Level::Warning | Level::Note);
1970                    let mut err = Diag::<()>::new(sess.dcx(), level, msg);
1971                    if !span.is_dummy() {
1972                        err.span(span.span());
1973                    }
1974
1975                    // Point to the generated assembly if it is available.
1976                    if let Some((buffer, spans)) = source {
1977                        let source = sess
1978                            .source_map()
1979                            .new_source_file(FileName::inline_asm_source_code(&buffer), buffer);
1980                        let spans: Vec<_> = spans
1981                            .iter()
1982                            .map(|sp| {
1983                                Span::with_root_ctxt(
1984                                    source.normalized_byte_pos(sp.start as u32),
1985                                    source.normalized_byte_pos(sp.end as u32),
1986                                )
1987                            })
1988                            .collect();
1989                        err.span_note(spans, "instantiated into assembly here");
1990                    }
1991
1992                    err.emit();
1993                }
1994                Ok(SharedEmitterMessage::Fatal(msg)) => {
1995                    sess.dcx().fatal(msg);
1996                }
1997                Err(_) => {
1998                    break;
1999                }
2000            }
2001        }
2002    }
2003}
2004
2005pub struct Coordinator<B: ExtraBackendMethods> {
2006    pub sender: Sender<Box<dyn Any + Send>>,
2007    future: Option<thread::JoinHandle<Result<CompiledModules, ()>>>,
2008    // Only used for the Message type.
2009    phantom: PhantomData<B>,
2010}
2011
2012impl<B: ExtraBackendMethods> Coordinator<B> {
2013    fn join(mut self) -> std::thread::Result<Result<CompiledModules, ()>> {
2014        self.future.take().unwrap().join()
2015    }
2016}
2017
2018impl<B: ExtraBackendMethods> Drop for Coordinator<B> {
2019    fn drop(&mut self) {
2020        if let Some(future) = self.future.take() {
2021            // If we haven't joined yet, signal to the coordinator that it should spawn no more
2022            // work, and wait for worker threads to finish.
2023            drop(self.sender.send(Box::new(Message::CodegenAborted::<B>)));
2024            drop(future.join());
2025        }
2026    }
2027}
2028
2029pub struct OngoingCodegen<B: ExtraBackendMethods> {
2030    pub backend: B,
2031    pub crate_info: CrateInfo,
2032    pub codegen_worker_receive: Receiver<CguMessage>,
2033    pub shared_emitter_main: SharedEmitterMain,
2034    pub output_filenames: Arc<OutputFilenames>,
2035    pub coordinator: Coordinator<B>,
2036}
2037
2038impl<B: ExtraBackendMethods> OngoingCodegen<B> {
2039    pub fn join(self, sess: &Session) -> (CodegenResults, FxIndexMap<WorkProductId, WorkProduct>) {
2040        self.shared_emitter_main.check(sess, true);
2041        let compiled_modules = sess.time("join_worker_thread", || match self.coordinator.join() {
2042            Ok(Ok(compiled_modules)) => compiled_modules,
2043            Ok(Err(())) => {
2044                sess.dcx().abort_if_errors();
2045                panic!("expected abort due to worker thread errors")
2046            }
2047            Err(_) => {
2048                bug!("panic during codegen/LLVM phase");
2049            }
2050        });
2051
2052        sess.dcx().abort_if_errors();
2053
2054        let work_products =
2055            copy_all_cgu_workproducts_to_incr_comp_cache_dir(sess, &compiled_modules);
2056        produce_final_output_artifacts(sess, &compiled_modules, &self.output_filenames);
2057
2058        // FIXME: time_llvm_passes support - does this use a global context or
2059        // something?
2060        if sess.codegen_units().as_usize() == 1 && sess.opts.unstable_opts.time_llvm_passes {
2061            self.backend.print_pass_timings()
2062        }
2063
2064        if sess.print_llvm_stats() {
2065            self.backend.print_statistics()
2066        }
2067
2068        (
2069            CodegenResults {
2070                crate_info: self.crate_info,
2071
2072                modules: compiled_modules.modules,
2073                allocator_module: compiled_modules.allocator_module,
2074            },
2075            work_products,
2076        )
2077    }
2078
2079    pub(crate) fn codegen_finished(&self, tcx: TyCtxt<'_>) {
2080        self.wait_for_signal_to_codegen_item();
2081        self.check_for_errors(tcx.sess);
2082        drop(self.coordinator.sender.send(Box::new(Message::CodegenComplete::<B>)));
2083    }
2084
2085    pub(crate) fn submit_autodiff_items(&self, items: Vec<AutoDiffItem>) {
2086        drop(self.coordinator.sender.send(Box::new(Message::<B>::AddAutoDiffItems(items))));
2087    }
2088
2089    pub(crate) fn check_for_errors(&self, sess: &Session) {
2090        self.shared_emitter_main.check(sess, false);
2091    }
2092
2093    pub(crate) fn wait_for_signal_to_codegen_item(&self) {
2094        match self.codegen_worker_receive.recv() {
2095            Ok(CguMessage) => {
2096                // Ok to proceed.
2097            }
2098            Err(_) => {
2099                // One of the LLVM threads must have panicked, fall through so
2100                // error handling can be reached.
2101            }
2102        }
2103    }
2104}
2105
2106pub(crate) fn submit_codegened_module_to_llvm<B: ExtraBackendMethods>(
2107    _backend: &B,
2108    tx_to_llvm_workers: &Sender<Box<dyn Any + Send>>,
2109    module: ModuleCodegen<B::Module>,
2110    cost: u64,
2111) {
2112    let llvm_work_item = WorkItem::Optimize(module);
2113    drop(tx_to_llvm_workers.send(Box::new(Message::CodegenDone::<B> { llvm_work_item, cost })));
2114}
2115
2116pub(crate) fn submit_post_lto_module_to_llvm<B: ExtraBackendMethods>(
2117    _backend: &B,
2118    tx_to_llvm_workers: &Sender<Box<dyn Any + Send>>,
2119    module: CachedModuleCodegen,
2120) {
2121    let llvm_work_item = WorkItem::CopyPostLtoArtifacts(module);
2122    drop(tx_to_llvm_workers.send(Box::new(Message::CodegenDone::<B> { llvm_work_item, cost: 0 })));
2123}
2124
2125pub(crate) fn submit_pre_lto_module_to_llvm<B: ExtraBackendMethods>(
2126    _backend: &B,
2127    tcx: TyCtxt<'_>,
2128    tx_to_llvm_workers: &Sender<Box<dyn Any + Send>>,
2129    module: CachedModuleCodegen,
2130) {
2131    let filename = pre_lto_bitcode_filename(&module.name);
2132    let bc_path = in_incr_comp_dir_sess(tcx.sess, &filename);
2133    let file = fs::File::open(&bc_path)
2134        .unwrap_or_else(|e| panic!("failed to open bitcode file `{}`: {}", bc_path.display(), e));
2135
2136    let mmap = unsafe {
2137        Mmap::map(file).unwrap_or_else(|e| {
2138            panic!("failed to mmap bitcode file `{}`: {}", bc_path.display(), e)
2139        })
2140    };
2141    // Schedule the module to be loaded
2142    drop(tx_to_llvm_workers.send(Box::new(Message::AddImportOnlyModule::<B> {
2143        module_data: SerializedModule::FromUncompressedFile(mmap),
2144        work_product: module.source,
2145    })));
2146}
2147
2148fn pre_lto_bitcode_filename(module_name: &str) -> String {
2149    format!("{module_name}.{PRE_LTO_BC_EXT}")
2150}
2151
2152fn msvc_imps_needed(tcx: TyCtxt<'_>) -> bool {
2153    // This should never be true (because it's not supported). If it is true,
2154    // something is wrong with commandline arg validation.
2155    assert!(
2156        !(tcx.sess.opts.cg.linker_plugin_lto.enabled()
2157            && tcx.sess.target.is_like_windows
2158            && tcx.sess.opts.cg.prefer_dynamic)
2159    );
2160
2161    // We need to generate _imp__ symbol if we are generating an rlib or we include one
2162    // indirectly from ThinLTO. In theory these are not needed as ThinLTO could resolve
2163    // these, but it currently does not do so.
2164    let can_have_static_objects =
2165        tcx.sess.lto() == Lto::Thin || tcx.crate_types().contains(&CrateType::Rlib);
2166
2167    tcx.sess.target.is_like_windows &&
2168    can_have_static_objects   &&
2169    // ThinLTO can't handle this workaround in all cases, so we don't
2170    // emit the `__imp_` symbols. Instead we make them unnecessary by disallowing
2171    // dynamic linking when linker plugin LTO is enabled.
2172    !tcx.sess.opts.cg.linker_plugin_lto.enabled()
2173}