rustc_codegen_ssa/back/
write.rs

1use std::any::Any;
2use std::assert_matches::assert_matches;
3use std::marker::PhantomData;
4use std::path::{Path, PathBuf};
5use std::sync::Arc;
6use std::sync::mpsc::{Receiver, Sender, channel};
7use std::{fs, io, mem, str, thread};
8
9use rustc_abi::Size;
10use rustc_ast::attr;
11use rustc_ast::expand::autodiff_attrs::AutoDiffItem;
12use rustc_data_structures::fx::{FxHashMap, FxIndexMap};
13use rustc_data_structures::jobserver::{self, Acquired};
14use rustc_data_structures::memmap::Mmap;
15use rustc_data_structures::profiling::{SelfProfilerRef, VerboseTimingGuard};
16use rustc_errors::emitter::Emitter;
17use rustc_errors::translation::Translate;
18use rustc_errors::{
19    Diag, DiagArgMap, DiagCtxt, DiagMessage, ErrCode, FatalError, FluentBundle, Level, MultiSpan,
20    Style, Suggestions,
21};
22use rustc_fs_util::link_or_copy;
23use rustc_hir::def_id::{CrateNum, LOCAL_CRATE};
24use rustc_incremental::{
25    copy_cgu_workproduct_to_incr_comp_cache_dir, in_incr_comp_dir, in_incr_comp_dir_sess,
26};
27use rustc_metadata::EncodedMetadata;
28use rustc_metadata::fs::copy_to_stdout;
29use rustc_middle::bug;
30use rustc_middle::dep_graph::{WorkProduct, WorkProductId};
31use rustc_middle::middle::exported_symbols::SymbolExportInfo;
32use rustc_middle::ty::TyCtxt;
33use rustc_session::Session;
34use rustc_session::config::{
35    self, CrateType, Lto, OutFileName, OutputFilenames, OutputType, Passes, SwitchWithOptPath,
36};
37use rustc_span::source_map::SourceMap;
38use rustc_span::{FileName, InnerSpan, Span, SpanData, sym};
39use rustc_target::spec::{MergeFunctions, SanitizerSet};
40use tracing::debug;
41
42use super::link::{self, ensure_removed};
43use super::lto::{self, SerializedModule};
44use super::symbol_export::symbol_name_for_instance_in_crate;
45use crate::errors::{AutodiffWithoutLto, ErrorCreatingRemarkDir};
46use crate::traits::*;
47use crate::{
48    CachedModuleCodegen, CodegenResults, CompiledModule, CrateInfo, ModuleCodegen, ModuleKind,
49    errors,
50};
51
52const PRE_LTO_BC_EXT: &str = "pre-lto.bc";
53
54/// What kind of object file to emit.
55#[derive(Clone, Copy, PartialEq)]
56pub enum EmitObj {
57    // No object file.
58    None,
59
60    // Just uncompressed llvm bitcode. Provides easy compatibility with
61    // emscripten's ecc compiler, when used as the linker.
62    Bitcode,
63
64    // Object code, possibly augmented with a bitcode section.
65    ObjectCode(BitcodeSection),
66}
67
68/// What kind of llvm bitcode section to embed in an object file.
69#[derive(Clone, Copy, PartialEq)]
70pub enum BitcodeSection {
71    // No bitcode section.
72    None,
73
74    // A full, uncompressed bitcode section.
75    Full,
76}
77
78/// Module-specific configuration for `optimize_and_codegen`.
79pub struct ModuleConfig {
80    /// Names of additional optimization passes to run.
81    pub passes: Vec<String>,
82    /// Some(level) to optimize at a certain level, or None to run
83    /// absolutely no optimizations (used for the metadata module).
84    pub opt_level: Option<config::OptLevel>,
85
86    /// Some(level) to optimize binary size, or None to not affect program size.
87    pub opt_size: Option<config::OptLevel>,
88
89    pub pgo_gen: SwitchWithOptPath,
90    pub pgo_use: Option<PathBuf>,
91    pub pgo_sample_use: Option<PathBuf>,
92    pub debug_info_for_profiling: bool,
93    pub instrument_coverage: bool,
94
95    pub sanitizer: SanitizerSet,
96    pub sanitizer_recover: SanitizerSet,
97    pub sanitizer_dataflow_abilist: Vec<String>,
98    pub sanitizer_memory_track_origins: usize,
99
100    // Flags indicating which outputs to produce.
101    pub emit_pre_lto_bc: bool,
102    pub emit_no_opt_bc: bool,
103    pub emit_bc: bool,
104    pub emit_ir: bool,
105    pub emit_asm: bool,
106    pub emit_obj: EmitObj,
107    pub emit_thin_lto: bool,
108    pub emit_thin_lto_summary: bool,
109    pub bc_cmdline: String,
110
111    // Miscellaneous flags. These are mostly copied from command-line
112    // options.
113    pub verify_llvm_ir: bool,
114    pub lint_llvm_ir: bool,
115    pub no_prepopulate_passes: bool,
116    pub no_builtins: bool,
117    pub time_module: bool,
118    pub vectorize_loop: bool,
119    pub vectorize_slp: bool,
120    pub merge_functions: bool,
121    pub emit_lifetime_markers: bool,
122    pub llvm_plugins: Vec<String>,
123    pub autodiff: Vec<config::AutoDiff>,
124}
125
126impl ModuleConfig {
127    fn new(kind: ModuleKind, tcx: TyCtxt<'_>, no_builtins: bool) -> ModuleConfig {
128        // If it's a regular module, use `$regular`, otherwise use `$other`.
129        // `$regular` and `$other` are evaluated lazily.
130        macro_rules! if_regular {
131            ($regular: expr, $other: expr) => {
132                if let ModuleKind::Regular = kind { $regular } else { $other }
133            };
134        }
135
136        let sess = tcx.sess;
137        let opt_level_and_size = if_regular!(Some(sess.opts.optimize), None);
138
139        let save_temps = sess.opts.cg.save_temps;
140
141        let should_emit_obj = sess.opts.output_types.contains_key(&OutputType::Exe)
142            || match kind {
143                ModuleKind::Regular => sess.opts.output_types.contains_key(&OutputType::Object),
144                ModuleKind::Allocator => false,
145                ModuleKind::Metadata => sess.opts.output_types.contains_key(&OutputType::Metadata),
146            };
147
148        let emit_obj = if !should_emit_obj {
149            EmitObj::None
150        } else if sess.target.obj_is_bitcode
151            || (sess.opts.cg.linker_plugin_lto.enabled() && !no_builtins)
152        {
153            // This case is selected if the target uses objects as bitcode, or
154            // if linker plugin LTO is enabled. In the linker plugin LTO case
155            // the assumption is that the final link-step will read the bitcode
156            // and convert it to object code. This may be done by either the
157            // native linker or rustc itself.
158            //
159            // Note, however, that the linker-plugin-lto requested here is
160            // explicitly ignored for `#![no_builtins]` crates. These crates are
161            // specifically ignored by rustc's LTO passes and wouldn't work if
162            // loaded into the linker. These crates define symbols that LLVM
163            // lowers intrinsics to, and these symbol dependencies aren't known
164            // until after codegen. As a result any crate marked
165            // `#![no_builtins]` is assumed to not participate in LTO and
166            // instead goes on to generate object code.
167            EmitObj::Bitcode
168        } else if need_bitcode_in_object(tcx) {
169            EmitObj::ObjectCode(BitcodeSection::Full)
170        } else {
171            EmitObj::ObjectCode(BitcodeSection::None)
172        };
173
174        ModuleConfig {
175            passes: if_regular!(sess.opts.cg.passes.clone(), vec![]),
176
177            opt_level: opt_level_and_size,
178            opt_size: opt_level_and_size,
179
180            pgo_gen: if_regular!(
181                sess.opts.cg.profile_generate.clone(),
182                SwitchWithOptPath::Disabled
183            ),
184            pgo_use: if_regular!(sess.opts.cg.profile_use.clone(), None),
185            pgo_sample_use: if_regular!(sess.opts.unstable_opts.profile_sample_use.clone(), None),
186            debug_info_for_profiling: sess.opts.unstable_opts.debug_info_for_profiling,
187            instrument_coverage: if_regular!(sess.instrument_coverage(), false),
188
189            sanitizer: if_regular!(sess.opts.unstable_opts.sanitizer, SanitizerSet::empty()),
190            sanitizer_dataflow_abilist: if_regular!(
191                sess.opts.unstable_opts.sanitizer_dataflow_abilist.clone(),
192                Vec::new()
193            ),
194            sanitizer_recover: if_regular!(
195                sess.opts.unstable_opts.sanitizer_recover,
196                SanitizerSet::empty()
197            ),
198            sanitizer_memory_track_origins: if_regular!(
199                sess.opts.unstable_opts.sanitizer_memory_track_origins,
200                0
201            ),
202
203            emit_pre_lto_bc: if_regular!(
204                save_temps || need_pre_lto_bitcode_for_incr_comp(sess),
205                false
206            ),
207            emit_no_opt_bc: if_regular!(save_temps, false),
208            emit_bc: if_regular!(
209                save_temps || sess.opts.output_types.contains_key(&OutputType::Bitcode),
210                save_temps
211            ),
212            emit_ir: if_regular!(
213                sess.opts.output_types.contains_key(&OutputType::LlvmAssembly),
214                false
215            ),
216            emit_asm: if_regular!(
217                sess.opts.output_types.contains_key(&OutputType::Assembly),
218                false
219            ),
220            emit_obj,
221            emit_thin_lto: sess.opts.unstable_opts.emit_thin_lto,
222            emit_thin_lto_summary: if_regular!(
223                sess.opts.output_types.contains_key(&OutputType::ThinLinkBitcode),
224                false
225            ),
226            bc_cmdline: sess.target.bitcode_llvm_cmdline.to_string(),
227
228            verify_llvm_ir: sess.verify_llvm_ir(),
229            lint_llvm_ir: sess.opts.unstable_opts.lint_llvm_ir,
230            no_prepopulate_passes: sess.opts.cg.no_prepopulate_passes,
231            no_builtins: no_builtins || sess.target.no_builtins,
232
233            // Exclude metadata and allocator modules from time_passes output,
234            // since they throw off the "LLVM passes" measurement.
235            time_module: if_regular!(true, false),
236
237            // Copy what clang does by turning on loop vectorization at O2 and
238            // slp vectorization at O3.
239            vectorize_loop: !sess.opts.cg.no_vectorize_loops
240                && (sess.opts.optimize == config::OptLevel::More
241                    || sess.opts.optimize == config::OptLevel::Aggressive),
242            vectorize_slp: !sess.opts.cg.no_vectorize_slp
243                && sess.opts.optimize == config::OptLevel::Aggressive,
244
245            // Some targets (namely, NVPTX) interact badly with the
246            // MergeFunctions pass. This is because MergeFunctions can generate
247            // new function calls which may interfere with the target calling
248            // convention; e.g. for the NVPTX target, PTX kernels should not
249            // call other PTX kernels. MergeFunctions can also be configured to
250            // generate aliases instead, but aliases are not supported by some
251            // backends (again, NVPTX). Therefore, allow targets to opt out of
252            // the MergeFunctions pass, but otherwise keep the pass enabled (at
253            // O2 and O3) since it can be useful for reducing code size.
254            merge_functions: match sess
255                .opts
256                .unstable_opts
257                .merge_functions
258                .unwrap_or(sess.target.merge_functions)
259            {
260                MergeFunctions::Disabled => false,
261                MergeFunctions::Trampolines | MergeFunctions::Aliases => {
262                    use config::OptLevel::*;
263                    match sess.opts.optimize {
264                        Aggressive | More | SizeMin | Size => true,
265                        Less | No => false,
266                    }
267                }
268            },
269
270            emit_lifetime_markers: sess.emit_lifetime_markers(),
271            llvm_plugins: if_regular!(sess.opts.unstable_opts.llvm_plugins.clone(), vec![]),
272            autodiff: if_regular!(sess.opts.unstable_opts.autodiff.clone(), vec![]),
273        }
274    }
275
276    pub fn bitcode_needed(&self) -> bool {
277        self.emit_bc
278            || self.emit_thin_lto_summary
279            || self.emit_obj == EmitObj::Bitcode
280            || self.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full)
281    }
282
283    pub fn embed_bitcode(&self) -> bool {
284        self.emit_obj == EmitObj::ObjectCode(BitcodeSection::Full)
285    }
286}
287
288/// Configuration passed to the function returned by the `target_machine_factory`.
289pub struct TargetMachineFactoryConfig {
290    /// Split DWARF is enabled in LLVM by checking that `TM.MCOptions.SplitDwarfFile` isn't empty,
291    /// so the path to the dwarf object has to be provided when we create the target machine.
292    /// This can be ignored by backends which do not need it for their Split DWARF support.
293    pub split_dwarf_file: Option<PathBuf>,
294
295    /// The name of the output object file. Used for setting OutputFilenames in target options
296    /// so that LLVM can emit the CodeView S_OBJNAME record in pdb files
297    pub output_obj_file: Option<PathBuf>,
298}
299
300impl TargetMachineFactoryConfig {
301    pub fn new(
302        cgcx: &CodegenContext<impl WriteBackendMethods>,
303        module_name: &str,
304    ) -> TargetMachineFactoryConfig {
305        let split_dwarf_file = if cgcx.target_can_use_split_dwarf {
306            cgcx.output_filenames.split_dwarf_path(
307                cgcx.split_debuginfo,
308                cgcx.split_dwarf_kind,
309                Some(module_name),
310            )
311        } else {
312            None
313        };
314
315        let output_obj_file =
316            Some(cgcx.output_filenames.temp_path(OutputType::Object, Some(module_name)));
317        TargetMachineFactoryConfig { split_dwarf_file, output_obj_file }
318    }
319}
320
321pub type TargetMachineFactoryFn<B> = Arc<
322    dyn Fn(
323            TargetMachineFactoryConfig,
324        ) -> Result<
325            <B as WriteBackendMethods>::TargetMachine,
326            <B as WriteBackendMethods>::TargetMachineError,
327        > + Send
328        + Sync,
329>;
330
331type ExportedSymbols = FxHashMap<CrateNum, Arc<Vec<(String, SymbolExportInfo)>>>;
332
333/// Additional resources used by optimize_and_codegen (not module specific)
334#[derive(Clone)]
335pub struct CodegenContext<B: WriteBackendMethods> {
336    // Resources needed when running LTO
337    pub prof: SelfProfilerRef,
338    pub lto: Lto,
339    pub save_temps: bool,
340    pub fewer_names: bool,
341    pub time_trace: bool,
342    pub exported_symbols: Option<Arc<ExportedSymbols>>,
343    pub opts: Arc<config::Options>,
344    pub crate_types: Vec<CrateType>,
345    pub each_linked_rlib_for_lto: Vec<(CrateNum, PathBuf)>,
346    pub output_filenames: Arc<OutputFilenames>,
347    pub regular_module_config: Arc<ModuleConfig>,
348    pub metadata_module_config: Arc<ModuleConfig>,
349    pub allocator_module_config: Arc<ModuleConfig>,
350    pub tm_factory: TargetMachineFactoryFn<B>,
351    pub msvc_imps_needed: bool,
352    pub is_pe_coff: bool,
353    pub target_can_use_split_dwarf: bool,
354    pub target_arch: String,
355    pub target_is_like_osx: bool,
356    pub target_is_like_aix: bool,
357    pub split_debuginfo: rustc_target::spec::SplitDebuginfo,
358    pub split_dwarf_kind: rustc_session::config::SplitDwarfKind,
359    pub pointer_size: Size,
360
361    /// All commandline args used to invoke the compiler, with @file args fully expanded.
362    /// This will only be used within debug info, e.g. in the pdb file on windows
363    /// This is mainly useful for other tools that reads that debuginfo to figure out
364    /// how to call the compiler with the same arguments.
365    pub expanded_args: Vec<String>,
366
367    /// Emitter to use for diagnostics produced during codegen.
368    pub diag_emitter: SharedEmitter,
369    /// LLVM optimizations for which we want to print remarks.
370    pub remark: Passes,
371    /// Directory into which should the LLVM optimization remarks be written.
372    /// If `None`, they will be written to stderr.
373    pub remark_dir: Option<PathBuf>,
374    /// The incremental compilation session directory, or None if we are not
375    /// compiling incrementally
376    pub incr_comp_session_dir: Option<PathBuf>,
377    /// Channel back to the main control thread to send messages to
378    pub coordinator_send: Sender<Box<dyn Any + Send>>,
379    /// `true` if the codegen should be run in parallel.
380    ///
381    /// Depends on [`CodegenBackend::supports_parallel()`] and `-Zno_parallel_backend`.
382    pub parallel: bool,
383}
384
385impl<B: WriteBackendMethods> CodegenContext<B> {
386    pub fn create_dcx(&self) -> DiagCtxt {
387        DiagCtxt::new(Box::new(self.diag_emitter.clone()))
388    }
389
390    pub fn config(&self, kind: ModuleKind) -> &ModuleConfig {
391        match kind {
392            ModuleKind::Regular => &self.regular_module_config,
393            ModuleKind::Metadata => &self.metadata_module_config,
394            ModuleKind::Allocator => &self.allocator_module_config,
395        }
396    }
397}
398
399fn generate_lto_work<B: ExtraBackendMethods>(
400    cgcx: &CodegenContext<B>,
401    autodiff: Vec<AutoDiffItem>,
402    needs_fat_lto: Vec<FatLtoInput<B>>,
403    needs_thin_lto: Vec<(String, B::ThinBuffer)>,
404    import_only_modules: Vec<(SerializedModule<B::ModuleBuffer>, WorkProduct)>,
405) -> Vec<(WorkItem<B>, u64)> {
406    let _prof_timer = cgcx.prof.generic_activity("codegen_generate_lto_work");
407
408    if !needs_fat_lto.is_empty() {
409        assert!(needs_thin_lto.is_empty());
410        let mut module =
411            B::run_fat_lto(cgcx, needs_fat_lto, import_only_modules).unwrap_or_else(|e| e.raise());
412        if cgcx.lto == Lto::Fat && !autodiff.is_empty() {
413            let config = cgcx.config(ModuleKind::Regular);
414            module =
415                unsafe { module.autodiff(cgcx, autodiff, config).unwrap_or_else(|e| e.raise()) };
416        }
417        // We are adding a single work item, so the cost doesn't matter.
418        vec![(WorkItem::LTO(module), 0)]
419    } else {
420        if !autodiff.is_empty() {
421            let dcx = cgcx.create_dcx();
422            dcx.handle().emit_fatal(AutodiffWithoutLto {});
423        }
424        assert!(needs_fat_lto.is_empty());
425        let (lto_modules, copy_jobs) = B::run_thin_lto(cgcx, needs_thin_lto, import_only_modules)
426            .unwrap_or_else(|e| e.raise());
427        lto_modules
428            .into_iter()
429            .map(|module| {
430                let cost = module.cost();
431                (WorkItem::LTO(module), cost)
432            })
433            .chain(copy_jobs.into_iter().map(|wp| {
434                (
435                    WorkItem::CopyPostLtoArtifacts(CachedModuleCodegen {
436                        name: wp.cgu_name.clone(),
437                        source: wp,
438                    }),
439                    0, // copying is very cheap
440                )
441            }))
442            .collect()
443    }
444}
445
446struct CompiledModules {
447    modules: Vec<CompiledModule>,
448    allocator_module: Option<CompiledModule>,
449}
450
451fn need_bitcode_in_object(tcx: TyCtxt<'_>) -> bool {
452    let sess = tcx.sess;
453    sess.opts.cg.embed_bitcode
454        && tcx.crate_types().contains(&CrateType::Rlib)
455        && sess.opts.output_types.contains_key(&OutputType::Exe)
456}
457
458fn need_pre_lto_bitcode_for_incr_comp(sess: &Session) -> bool {
459    if sess.opts.incremental.is_none() {
460        return false;
461    }
462
463    match sess.lto() {
464        Lto::No => false,
465        Lto::Fat | Lto::Thin | Lto::ThinLocal => true,
466    }
467}
468
469pub(crate) fn start_async_codegen<B: ExtraBackendMethods>(
470    backend: B,
471    tcx: TyCtxt<'_>,
472    target_cpu: String,
473    metadata: EncodedMetadata,
474    metadata_module: Option<CompiledModule>,
475) -> OngoingCodegen<B> {
476    let (coordinator_send, coordinator_receive) = channel();
477
478    let crate_attrs = tcx.hir_attrs(rustc_hir::CRATE_HIR_ID);
479    let no_builtins = attr::contains_name(crate_attrs, sym::no_builtins);
480
481    let crate_info = CrateInfo::new(tcx, target_cpu);
482
483    let regular_config = ModuleConfig::new(ModuleKind::Regular, tcx, no_builtins);
484    let metadata_config = ModuleConfig::new(ModuleKind::Metadata, tcx, no_builtins);
485    let allocator_config = ModuleConfig::new(ModuleKind::Allocator, tcx, no_builtins);
486
487    let (shared_emitter, shared_emitter_main) = SharedEmitter::new();
488    let (codegen_worker_send, codegen_worker_receive) = channel();
489
490    let coordinator_thread = start_executing_work(
491        backend.clone(),
492        tcx,
493        &crate_info,
494        shared_emitter,
495        codegen_worker_send,
496        coordinator_receive,
497        Arc::new(regular_config),
498        Arc::new(metadata_config),
499        Arc::new(allocator_config),
500        coordinator_send.clone(),
501    );
502
503    OngoingCodegen {
504        backend,
505        metadata,
506        metadata_module,
507        crate_info,
508
509        codegen_worker_receive,
510        shared_emitter_main,
511        coordinator: Coordinator {
512            sender: coordinator_send,
513            future: Some(coordinator_thread),
514            phantom: PhantomData,
515        },
516        output_filenames: Arc::clone(tcx.output_filenames(())),
517    }
518}
519
520fn copy_all_cgu_workproducts_to_incr_comp_cache_dir(
521    sess: &Session,
522    compiled_modules: &CompiledModules,
523) -> FxIndexMap<WorkProductId, WorkProduct> {
524    let mut work_products = FxIndexMap::default();
525
526    if sess.opts.incremental.is_none() {
527        return work_products;
528    }
529
530    let _timer = sess.timer("copy_all_cgu_workproducts_to_incr_comp_cache_dir");
531
532    for module in compiled_modules.modules.iter().filter(|m| m.kind == ModuleKind::Regular) {
533        let mut files = Vec::new();
534        if let Some(object_file_path) = &module.object {
535            files.push((OutputType::Object.extension(), object_file_path.as_path()));
536        }
537        if let Some(dwarf_object_file_path) = &module.dwarf_object {
538            files.push(("dwo", dwarf_object_file_path.as_path()));
539        }
540        if let Some(path) = &module.assembly {
541            files.push((OutputType::Assembly.extension(), path.as_path()));
542        }
543        if let Some(path) = &module.llvm_ir {
544            files.push((OutputType::LlvmAssembly.extension(), path.as_path()));
545        }
546        if let Some(path) = &module.bytecode {
547            files.push((OutputType::Bitcode.extension(), path.as_path()));
548        }
549        if let Some((id, product)) = copy_cgu_workproduct_to_incr_comp_cache_dir(
550            sess,
551            &module.name,
552            files.as_slice(),
553            &module.links_from_incr_cache,
554        ) {
555            work_products.insert(id, product);
556        }
557    }
558
559    work_products
560}
561
562fn produce_final_output_artifacts(
563    sess: &Session,
564    compiled_modules: &CompiledModules,
565    crate_output: &OutputFilenames,
566) {
567    let mut user_wants_bitcode = false;
568    let mut user_wants_objects = false;
569
570    // Produce final compile outputs.
571    let copy_gracefully = |from: &Path, to: &OutFileName| match to {
572        OutFileName::Stdout if let Err(e) = copy_to_stdout(from) => {
573            sess.dcx().emit_err(errors::CopyPath::new(from, to.as_path(), e));
574        }
575        OutFileName::Real(path) if let Err(e) = fs::copy(from, path) => {
576            sess.dcx().emit_err(errors::CopyPath::new(from, path, e));
577        }
578        _ => {}
579    };
580
581    let copy_if_one_unit = |output_type: OutputType, keep_numbered: bool| {
582        if let [module] = &compiled_modules.modules[..] {
583            // 1) Only one codegen unit. In this case it's no difficulty
584            //    to copy `foo.0.x` to `foo.x`.
585            let module_name = Some(&module.name[..]);
586            let path = crate_output.temp_path(output_type, module_name);
587            let output = crate_output.path(output_type);
588            if !output_type.is_text_output() && output.is_tty() {
589                sess.dcx()
590                    .emit_err(errors::BinaryOutputToTty { shorthand: output_type.shorthand() });
591            } else {
592                copy_gracefully(&path, &output);
593            }
594            if !sess.opts.cg.save_temps && !keep_numbered {
595                // The user just wants `foo.x`, not `foo.#module-name#.x`.
596                ensure_removed(sess.dcx(), &path);
597            }
598        } else {
599            let extension = crate_output
600                .temp_path(output_type, None)
601                .extension()
602                .unwrap()
603                .to_str()
604                .unwrap()
605                .to_owned();
606
607            if crate_output.outputs.contains_explicit_name(&output_type) {
608                // 2) Multiple codegen units, with `--emit foo=some_name`. We have
609                //    no good solution for this case, so warn the user.
610                sess.dcx().emit_warn(errors::IgnoringEmitPath { extension });
611            } else if crate_output.single_output_file.is_some() {
612                // 3) Multiple codegen units, with `-o some_name`. We have
613                //    no good solution for this case, so warn the user.
614                sess.dcx().emit_warn(errors::IgnoringOutput { extension });
615            } else {
616                // 4) Multiple codegen units, but no explicit name. We
617                //    just leave the `foo.0.x` files in place.
618                // (We don't have to do any work in this case.)
619            }
620        }
621    };
622
623    // Flag to indicate whether the user explicitly requested bitcode.
624    // Otherwise, we produced it only as a temporary output, and will need
625    // to get rid of it.
626    for output_type in crate_output.outputs.keys() {
627        match *output_type {
628            OutputType::Bitcode => {
629                user_wants_bitcode = true;
630                // Copy to .bc, but always keep the .0.bc. There is a later
631                // check to figure out if we should delete .0.bc files, or keep
632                // them for making an rlib.
633                copy_if_one_unit(OutputType::Bitcode, true);
634            }
635            OutputType::ThinLinkBitcode => {
636                copy_if_one_unit(OutputType::ThinLinkBitcode, false);
637            }
638            OutputType::LlvmAssembly => {
639                copy_if_one_unit(OutputType::LlvmAssembly, false);
640            }
641            OutputType::Assembly => {
642                copy_if_one_unit(OutputType::Assembly, false);
643            }
644            OutputType::Object => {
645                user_wants_objects = true;
646                copy_if_one_unit(OutputType::Object, true);
647            }
648            OutputType::Mir | OutputType::Metadata | OutputType::Exe | OutputType::DepInfo => {}
649        }
650    }
651
652    // Clean up unwanted temporary files.
653
654    // We create the following files by default:
655    //  - #crate#.#module-name#.bc
656    //  - #crate#.#module-name#.o
657    //  - #crate#.crate.metadata.bc
658    //  - #crate#.crate.metadata.o
659    //  - #crate#.o (linked from crate.##.o)
660    //  - #crate#.bc (copied from crate.##.bc)
661    // We may create additional files if requested by the user (through
662    // `-C save-temps` or `--emit=` flags).
663
664    if !sess.opts.cg.save_temps {
665        // Remove the temporary .#module-name#.o objects. If the user didn't
666        // explicitly request bitcode (with --emit=bc), and the bitcode is not
667        // needed for building an rlib, then we must remove .#module-name#.bc as
668        // well.
669
670        // Specific rules for keeping .#module-name#.bc:
671        //  - If the user requested bitcode (`user_wants_bitcode`), and
672        //    codegen_units > 1, then keep it.
673        //  - If the user requested bitcode but codegen_units == 1, then we
674        //    can toss .#module-name#.bc because we copied it to .bc earlier.
675        //  - If we're not building an rlib and the user didn't request
676        //    bitcode, then delete .#module-name#.bc.
677        // If you change how this works, also update back::link::link_rlib,
678        // where .#module-name#.bc files are (maybe) deleted after making an
679        // rlib.
680        let needs_crate_object = crate_output.outputs.contains_key(&OutputType::Exe);
681
682        let keep_numbered_bitcode = user_wants_bitcode && sess.codegen_units().as_usize() > 1;
683
684        let keep_numbered_objects =
685            needs_crate_object || (user_wants_objects && sess.codegen_units().as_usize() > 1);
686
687        for module in compiled_modules.modules.iter() {
688            if !keep_numbered_objects {
689                if let Some(ref path) = module.object {
690                    ensure_removed(sess.dcx(), path);
691                }
692
693                if let Some(ref path) = module.dwarf_object {
694                    ensure_removed(sess.dcx(), path);
695                }
696            }
697
698            if let Some(ref path) = module.bytecode {
699                if !keep_numbered_bitcode {
700                    ensure_removed(sess.dcx(), path);
701                }
702            }
703        }
704
705        if !user_wants_bitcode
706            && let Some(ref allocator_module) = compiled_modules.allocator_module
707            && let Some(ref path) = allocator_module.bytecode
708        {
709            ensure_removed(sess.dcx(), path);
710        }
711    }
712
713    if sess.opts.json_artifact_notifications {
714        if let [module] = &compiled_modules.modules[..] {
715            module.for_each_output(|_path, ty| {
716                if sess.opts.output_types.contains_key(&ty) {
717                    let descr = ty.shorthand();
718                    // for single cgu file is renamed to drop cgu specific suffix
719                    // so we regenerate it the same way
720                    let path = crate_output.path(ty);
721                    sess.dcx().emit_artifact_notification(path.as_path(), descr);
722                }
723            });
724        } else {
725            for module in &compiled_modules.modules {
726                module.for_each_output(|path, ty| {
727                    if sess.opts.output_types.contains_key(&ty) {
728                        let descr = ty.shorthand();
729                        sess.dcx().emit_artifact_notification(&path, descr);
730                    }
731                });
732            }
733        }
734    }
735
736    // We leave the following files around by default:
737    //  - #crate#.o
738    //  - #crate#.crate.metadata.o
739    //  - #crate#.bc
740    // These are used in linking steps and will be cleaned up afterward.
741}
742
743pub(crate) enum WorkItem<B: WriteBackendMethods> {
744    /// Optimize a newly codegened, totally unoptimized module.
745    Optimize(ModuleCodegen<B::Module>),
746    /// Copy the post-LTO artifacts from the incremental cache to the output
747    /// directory.
748    CopyPostLtoArtifacts(CachedModuleCodegen),
749    /// Performs (Thin)LTO on the given module.
750    LTO(lto::LtoModuleCodegen<B>),
751}
752
753impl<B: WriteBackendMethods> WorkItem<B> {
754    fn module_kind(&self) -> ModuleKind {
755        match *self {
756            WorkItem::Optimize(ref m) => m.kind,
757            WorkItem::CopyPostLtoArtifacts(_) | WorkItem::LTO(_) => ModuleKind::Regular,
758        }
759    }
760
761    /// Generate a short description of this work item suitable for use as a thread name.
762    fn short_description(&self) -> String {
763        // `pthread_setname()` on *nix ignores anything beyond the first 15
764        // bytes. Use short descriptions to maximize the space available for
765        // the module name.
766        #[cfg(not(windows))]
767        fn desc(short: &str, _long: &str, name: &str) -> String {
768            // The short label is three bytes, and is followed by a space. That
769            // leaves 11 bytes for the CGU name. How we obtain those 11 bytes
770            // depends on the CGU name form.
771            //
772            // - Non-incremental, e.g. `regex.f10ba03eb5ec7975-cgu.0`: the part
773            //   before the `-cgu.0` is the same for every CGU, so use the
774            //   `cgu.0` part. The number suffix will be different for each
775            //   CGU.
776            //
777            // - Incremental (normal), e.g. `2i52vvl2hco29us0`: use the whole
778            //   name because each CGU will have a unique ASCII hash, and the
779            //   first 11 bytes will be enough to identify it.
780            //
781            // - Incremental (with `-Zhuman-readable-cgu-names`), e.g.
782            //   `regex.f10ba03eb5ec7975-re_builder.volatile`: use the whole
783            //   name. The first 11 bytes won't be enough to uniquely identify
784            //   it, but no obvious substring will, and this is a rarely used
785            //   option so it doesn't matter much.
786            //
787            assert_eq!(short.len(), 3);
788            let name = if let Some(index) = name.find("-cgu.") {
789                &name[index + 1..] // +1 skips the leading '-'.
790            } else {
791                name
792            };
793            format!("{short} {name}")
794        }
795
796        // Windows has no thread name length limit, so use more descriptive names.
797        #[cfg(windows)]
798        fn desc(_short: &str, long: &str, name: &str) -> String {
799            format!("{long} {name}")
800        }
801
802        match self {
803            WorkItem::Optimize(m) => desc("opt", "optimize module", &m.name),
804            WorkItem::CopyPostLtoArtifacts(m) => desc("cpy", "copy LTO artifacts for", &m.name),
805            WorkItem::LTO(m) => desc("lto", "LTO module", m.name()),
806        }
807    }
808}
809
810/// A result produced by the backend.
811pub(crate) enum WorkItemResult<B: WriteBackendMethods> {
812    /// The backend has finished compiling a CGU, nothing more required.
813    Finished(CompiledModule),
814
815    /// The backend has finished compiling a CGU, which now needs linking
816    /// because `-Zcombine-cgu` was specified.
817    NeedsLink(ModuleCodegen<B::Module>),
818
819    /// The backend has finished compiling a CGU, which now needs to go through
820    /// fat LTO.
821    NeedsFatLto(FatLtoInput<B>),
822
823    /// The backend has finished compiling a CGU, which now needs to go through
824    /// thin LTO.
825    NeedsThinLto(String, B::ThinBuffer),
826}
827
828pub enum FatLtoInput<B: WriteBackendMethods> {
829    Serialized { name: String, buffer: B::ModuleBuffer },
830    InMemory(ModuleCodegen<B::Module>),
831}
832
833/// Actual LTO type we end up choosing based on multiple factors.
834pub(crate) enum ComputedLtoType {
835    No,
836    Thin,
837    Fat,
838}
839
840pub(crate) fn compute_per_cgu_lto_type(
841    sess_lto: &Lto,
842    opts: &config::Options,
843    sess_crate_types: &[CrateType],
844    module_kind: ModuleKind,
845) -> ComputedLtoType {
846    // Metadata modules never participate in LTO regardless of the lto
847    // settings.
848    if module_kind == ModuleKind::Metadata {
849        return ComputedLtoType::No;
850    }
851
852    // If the linker does LTO, we don't have to do it. Note that we
853    // keep doing full LTO, if it is requested, as not to break the
854    // assumption that the output will be a single module.
855    let linker_does_lto = opts.cg.linker_plugin_lto.enabled();
856
857    // When we're automatically doing ThinLTO for multi-codegen-unit
858    // builds we don't actually want to LTO the allocator modules if
859    // it shows up. This is due to various linker shenanigans that
860    // we'll encounter later.
861    let is_allocator = module_kind == ModuleKind::Allocator;
862
863    // We ignore a request for full crate graph LTO if the crate type
864    // is only an rlib, as there is no full crate graph to process,
865    // that'll happen later.
866    //
867    // This use case currently comes up primarily for targets that
868    // require LTO so the request for LTO is always unconditionally
869    // passed down to the backend, but we don't actually want to do
870    // anything about it yet until we've got a final product.
871    let is_rlib = matches!(sess_crate_types, [CrateType::Rlib]);
872
873    match sess_lto {
874        Lto::ThinLocal if !linker_does_lto && !is_allocator => ComputedLtoType::Thin,
875        Lto::Thin if !linker_does_lto && !is_rlib => ComputedLtoType::Thin,
876        Lto::Fat if !is_rlib => ComputedLtoType::Fat,
877        _ => ComputedLtoType::No,
878    }
879}
880
881fn execute_optimize_work_item<B: ExtraBackendMethods>(
882    cgcx: &CodegenContext<B>,
883    mut module: ModuleCodegen<B::Module>,
884    module_config: &ModuleConfig,
885) -> Result<WorkItemResult<B>, FatalError> {
886    let dcx = cgcx.create_dcx();
887    let dcx = dcx.handle();
888
889    unsafe {
890        B::optimize(cgcx, dcx, &mut module, module_config)?;
891    }
892
893    // After we've done the initial round of optimizations we need to
894    // decide whether to synchronously codegen this module or ship it
895    // back to the coordinator thread for further LTO processing (which
896    // has to wait for all the initial modules to be optimized).
897
898    let lto_type = compute_per_cgu_lto_type(&cgcx.lto, &cgcx.opts, &cgcx.crate_types, module.kind);
899
900    // If we're doing some form of incremental LTO then we need to be sure to
901    // save our module to disk first.
902    let bitcode = if cgcx.config(module.kind).emit_pre_lto_bc {
903        let filename = pre_lto_bitcode_filename(&module.name);
904        cgcx.incr_comp_session_dir.as_ref().map(|path| path.join(&filename))
905    } else {
906        None
907    };
908
909    match lto_type {
910        ComputedLtoType::No => finish_intra_module_work(cgcx, module, module_config),
911        ComputedLtoType::Thin => {
912            let (name, thin_buffer) = B::prepare_thin(module, false);
913            if let Some(path) = bitcode {
914                fs::write(&path, thin_buffer.data()).unwrap_or_else(|e| {
915                    panic!("Error writing pre-lto-bitcode file `{}`: {}", path.display(), e);
916                });
917            }
918            Ok(WorkItemResult::NeedsThinLto(name, thin_buffer))
919        }
920        ComputedLtoType::Fat => match bitcode {
921            Some(path) => {
922                let (name, buffer) = B::serialize_module(module);
923                fs::write(&path, buffer.data()).unwrap_or_else(|e| {
924                    panic!("Error writing pre-lto-bitcode file `{}`: {}", path.display(), e);
925                });
926                Ok(WorkItemResult::NeedsFatLto(FatLtoInput::Serialized { name, buffer }))
927            }
928            None => Ok(WorkItemResult::NeedsFatLto(FatLtoInput::InMemory(module))),
929        },
930    }
931}
932
933fn execute_copy_from_cache_work_item<B: ExtraBackendMethods>(
934    cgcx: &CodegenContext<B>,
935    module: CachedModuleCodegen,
936    module_config: &ModuleConfig,
937) -> WorkItemResult<B> {
938    let incr_comp_session_dir = cgcx.incr_comp_session_dir.as_ref().unwrap();
939
940    let mut links_from_incr_cache = Vec::new();
941
942    let mut load_from_incr_comp_dir = |output_path: PathBuf, saved_path: &str| {
943        let source_file = in_incr_comp_dir(incr_comp_session_dir, saved_path);
944        debug!(
945            "copying preexisting module `{}` from {:?} to {}",
946            module.name,
947            source_file,
948            output_path.display()
949        );
950        match link_or_copy(&source_file, &output_path) {
951            Ok(_) => {
952                links_from_incr_cache.push(source_file);
953                Some(output_path)
954            }
955            Err(error) => {
956                cgcx.create_dcx().handle().emit_err(errors::CopyPathBuf {
957                    source_file,
958                    output_path,
959                    error,
960                });
961                None
962            }
963        }
964    };
965
966    let dwarf_object =
967        module.source.saved_files.get("dwo").as_ref().and_then(|saved_dwarf_object_file| {
968            let dwarf_obj_out = cgcx
969                .output_filenames
970                .split_dwarf_path(cgcx.split_debuginfo, cgcx.split_dwarf_kind, Some(&module.name))
971                .expect(
972                    "saved dwarf object in work product but `split_dwarf_path` returned `None`",
973                );
974            load_from_incr_comp_dir(dwarf_obj_out, saved_dwarf_object_file)
975        });
976
977    let mut load_from_incr_cache = |perform, output_type: OutputType| {
978        if perform {
979            let saved_file = module.source.saved_files.get(output_type.extension())?;
980            let output_path = cgcx.output_filenames.temp_path(output_type, Some(&module.name));
981            load_from_incr_comp_dir(output_path, &saved_file)
982        } else {
983            None
984        }
985    };
986
987    let should_emit_obj = module_config.emit_obj != EmitObj::None;
988    let assembly = load_from_incr_cache(module_config.emit_asm, OutputType::Assembly);
989    let llvm_ir = load_from_incr_cache(module_config.emit_ir, OutputType::LlvmAssembly);
990    let bytecode = load_from_incr_cache(module_config.emit_bc, OutputType::Bitcode);
991    let object = load_from_incr_cache(should_emit_obj, OutputType::Object);
992    if should_emit_obj && object.is_none() {
993        cgcx.create_dcx().handle().emit_fatal(errors::NoSavedObjectFile { cgu_name: &module.name })
994    }
995
996    WorkItemResult::Finished(CompiledModule {
997        links_from_incr_cache,
998        name: module.name,
999        kind: ModuleKind::Regular,
1000        object,
1001        dwarf_object,
1002        bytecode,
1003        assembly,
1004        llvm_ir,
1005    })
1006}
1007
1008fn execute_lto_work_item<B: ExtraBackendMethods>(
1009    cgcx: &CodegenContext<B>,
1010    module: lto::LtoModuleCodegen<B>,
1011    module_config: &ModuleConfig,
1012) -> Result<WorkItemResult<B>, FatalError> {
1013    let module = unsafe { module.optimize(cgcx)? };
1014    finish_intra_module_work(cgcx, module, module_config)
1015}
1016
1017fn finish_intra_module_work<B: ExtraBackendMethods>(
1018    cgcx: &CodegenContext<B>,
1019    module: ModuleCodegen<B::Module>,
1020    module_config: &ModuleConfig,
1021) -> Result<WorkItemResult<B>, FatalError> {
1022    let dcx = cgcx.create_dcx();
1023    let dcx = dcx.handle();
1024
1025    if !cgcx.opts.unstable_opts.combine_cgu
1026        || module.kind == ModuleKind::Metadata
1027        || module.kind == ModuleKind::Allocator
1028    {
1029        let module = unsafe { B::codegen(cgcx, dcx, module, module_config)? };
1030        Ok(WorkItemResult::Finished(module))
1031    } else {
1032        Ok(WorkItemResult::NeedsLink(module))
1033    }
1034}
1035
1036/// Messages sent to the coordinator.
1037pub(crate) enum Message<B: WriteBackendMethods> {
1038    /// A jobserver token has become available. Sent from the jobserver helper
1039    /// thread.
1040    Token(io::Result<Acquired>),
1041
1042    /// The backend has finished processing a work item for a codegen unit.
1043    /// Sent from a backend worker thread.
1044    WorkItem { result: Result<WorkItemResult<B>, Option<WorkerFatalError>>, worker_id: usize },
1045
1046    /// A vector containing all the AutoDiff tasks that we have to pass to Enzyme.
1047    AddAutoDiffItems(Vec<AutoDiffItem>),
1048
1049    /// The frontend has finished generating something (backend IR or a
1050    /// post-LTO artifact) for a codegen unit, and it should be passed to the
1051    /// backend. Sent from the main thread.
1052    CodegenDone { llvm_work_item: WorkItem<B>, cost: u64 },
1053
1054    /// Similar to `CodegenDone`, but for reusing a pre-LTO artifact
1055    /// Sent from the main thread.
1056    AddImportOnlyModule {
1057        module_data: SerializedModule<B::ModuleBuffer>,
1058        work_product: WorkProduct,
1059    },
1060
1061    /// The frontend has finished generating everything for all codegen units.
1062    /// Sent from the main thread.
1063    CodegenComplete,
1064
1065    /// Some normal-ish compiler error occurred, and codegen should be wound
1066    /// down. Sent from the main thread.
1067    CodegenAborted,
1068}
1069
1070/// A message sent from the coordinator thread to the main thread telling it to
1071/// process another codegen unit.
1072pub struct CguMessage;
1073
1074// A cut-down version of `rustc_errors::DiagInner` that impls `Send`, which
1075// can be used to send diagnostics from codegen threads to the main thread.
1076// It's missing the following fields from `rustc_errors::DiagInner`.
1077// - `span`: it doesn't impl `Send`.
1078// - `suggestions`: it doesn't impl `Send`, and isn't used for codegen
1079//   diagnostics.
1080// - `sort_span`: it doesn't impl `Send`.
1081// - `is_lint`: lints aren't relevant during codegen.
1082// - `emitted_at`: not used for codegen diagnostics.
1083struct Diagnostic {
1084    level: Level,
1085    messages: Vec<(DiagMessage, Style)>,
1086    code: Option<ErrCode>,
1087    children: Vec<Subdiagnostic>,
1088    args: DiagArgMap,
1089}
1090
1091// A cut-down version of `rustc_errors::Subdiag` that impls `Send`. It's
1092// missing the following fields from `rustc_errors::Subdiag`.
1093// - `span`: it doesn't impl `Send`.
1094pub(crate) struct Subdiagnostic {
1095    level: Level,
1096    messages: Vec<(DiagMessage, Style)>,
1097}
1098
1099#[derive(PartialEq, Clone, Copy, Debug)]
1100enum MainThreadState {
1101    /// Doing nothing.
1102    Idle,
1103
1104    /// Doing codegen, i.e. MIR-to-LLVM-IR conversion.
1105    Codegenning,
1106
1107    /// Idle, but lending the compiler process's Token to an LLVM thread so it can do useful work.
1108    Lending,
1109}
1110
1111fn start_executing_work<B: ExtraBackendMethods>(
1112    backend: B,
1113    tcx: TyCtxt<'_>,
1114    crate_info: &CrateInfo,
1115    shared_emitter: SharedEmitter,
1116    codegen_worker_send: Sender<CguMessage>,
1117    coordinator_receive: Receiver<Box<dyn Any + Send>>,
1118    regular_config: Arc<ModuleConfig>,
1119    metadata_config: Arc<ModuleConfig>,
1120    allocator_config: Arc<ModuleConfig>,
1121    tx_to_llvm_workers: Sender<Box<dyn Any + Send>>,
1122) -> thread::JoinHandle<Result<CompiledModules, ()>> {
1123    let coordinator_send = tx_to_llvm_workers;
1124    let sess = tcx.sess;
1125
1126    let mut each_linked_rlib_for_lto = Vec::new();
1127    drop(link::each_linked_rlib(crate_info, None, &mut |cnum, path| {
1128        if link::ignored_for_lto(sess, crate_info, cnum) {
1129            return;
1130        }
1131        each_linked_rlib_for_lto.push((cnum, path.to_path_buf()));
1132    }));
1133
1134    // Compute the set of symbols we need to retain when doing LTO (if we need to)
1135    let exported_symbols = {
1136        let mut exported_symbols = FxHashMap::default();
1137
1138        let copy_symbols = |cnum| {
1139            let symbols = tcx
1140                .exported_symbols(cnum)
1141                .iter()
1142                .map(|&(s, lvl)| (symbol_name_for_instance_in_crate(tcx, s, cnum), lvl))
1143                .collect();
1144            Arc::new(symbols)
1145        };
1146
1147        match sess.lto() {
1148            Lto::No => None,
1149            Lto::ThinLocal => {
1150                exported_symbols.insert(LOCAL_CRATE, copy_symbols(LOCAL_CRATE));
1151                Some(Arc::new(exported_symbols))
1152            }
1153            Lto::Fat | Lto::Thin => {
1154                exported_symbols.insert(LOCAL_CRATE, copy_symbols(LOCAL_CRATE));
1155                for &(cnum, ref _path) in &each_linked_rlib_for_lto {
1156                    exported_symbols.insert(cnum, copy_symbols(cnum));
1157                }
1158                Some(Arc::new(exported_symbols))
1159            }
1160        }
1161    };
1162
1163    // First up, convert our jobserver into a helper thread so we can use normal
1164    // mpsc channels to manage our messages and such.
1165    // After we've requested tokens then we'll, when we can,
1166    // get tokens on `coordinator_receive` which will
1167    // get managed in the main loop below.
1168    let coordinator_send2 = coordinator_send.clone();
1169    let helper = jobserver::client()
1170        .into_helper_thread(move |token| {
1171            drop(coordinator_send2.send(Box::new(Message::Token::<B>(token))));
1172        })
1173        .expect("failed to spawn helper thread");
1174
1175    let ol =
1176        if tcx.sess.opts.unstable_opts.no_codegen || !tcx.sess.opts.output_types.should_codegen() {
1177            // If we know that we won’t be doing codegen, create target machines without optimisation.
1178            config::OptLevel::No
1179        } else {
1180            tcx.backend_optimization_level(())
1181        };
1182    let backend_features = tcx.global_backend_features(());
1183
1184    let remark_dir = if let Some(ref dir) = sess.opts.unstable_opts.remark_dir {
1185        let result = fs::create_dir_all(dir).and_then(|_| dir.canonicalize());
1186        match result {
1187            Ok(dir) => Some(dir),
1188            Err(error) => sess.dcx().emit_fatal(ErrorCreatingRemarkDir { error }),
1189        }
1190    } else {
1191        None
1192    };
1193
1194    let cgcx = CodegenContext::<B> {
1195        crate_types: tcx.crate_types().to_vec(),
1196        each_linked_rlib_for_lto,
1197        lto: sess.lto(),
1198        fewer_names: sess.fewer_names(),
1199        save_temps: sess.opts.cg.save_temps,
1200        time_trace: sess.opts.unstable_opts.llvm_time_trace,
1201        opts: Arc::new(sess.opts.clone()),
1202        prof: sess.prof.clone(),
1203        exported_symbols,
1204        remark: sess.opts.cg.remark.clone(),
1205        remark_dir,
1206        incr_comp_session_dir: sess.incr_comp_session_dir_opt().map(|r| r.clone()),
1207        coordinator_send,
1208        expanded_args: tcx.sess.expanded_args.clone(),
1209        diag_emitter: shared_emitter.clone(),
1210        output_filenames: Arc::clone(tcx.output_filenames(())),
1211        regular_module_config: regular_config,
1212        metadata_module_config: metadata_config,
1213        allocator_module_config: allocator_config,
1214        tm_factory: backend.target_machine_factory(tcx.sess, ol, backend_features),
1215        msvc_imps_needed: msvc_imps_needed(tcx),
1216        is_pe_coff: tcx.sess.target.is_like_windows,
1217        target_can_use_split_dwarf: tcx.sess.target_can_use_split_dwarf(),
1218        target_arch: tcx.sess.target.arch.to_string(),
1219        target_is_like_osx: tcx.sess.target.is_like_osx,
1220        target_is_like_aix: tcx.sess.target.is_like_aix,
1221        split_debuginfo: tcx.sess.split_debuginfo(),
1222        split_dwarf_kind: tcx.sess.opts.unstable_opts.split_dwarf_kind,
1223        parallel: backend.supports_parallel() && !sess.opts.unstable_opts.no_parallel_backend,
1224        pointer_size: tcx.data_layout.pointer_size,
1225    };
1226
1227    // This is the "main loop" of parallel work happening for parallel codegen.
1228    // It's here that we manage parallelism, schedule work, and work with
1229    // messages coming from clients.
1230    //
1231    // There are a few environmental pre-conditions that shape how the system
1232    // is set up:
1233    //
1234    // - Error reporting can only happen on the main thread because that's the
1235    //   only place where we have access to the compiler `Session`.
1236    // - LLVM work can be done on any thread.
1237    // - Codegen can only happen on the main thread.
1238    // - Each thread doing substantial work must be in possession of a `Token`
1239    //   from the `Jobserver`.
1240    // - The compiler process always holds one `Token`. Any additional `Tokens`
1241    //   have to be requested from the `Jobserver`.
1242    //
1243    // Error Reporting
1244    // ===============
1245    // The error reporting restriction is handled separately from the rest: We
1246    // set up a `SharedEmitter` that holds an open channel to the main thread.
1247    // When an error occurs on any thread, the shared emitter will send the
1248    // error message to the receiver main thread (`SharedEmitterMain`). The
1249    // main thread will periodically query this error message queue and emit
1250    // any error messages it has received. It might even abort compilation if
1251    // it has received a fatal error. In this case we rely on all other threads
1252    // being torn down automatically with the main thread.
1253    // Since the main thread will often be busy doing codegen work, error
1254    // reporting will be somewhat delayed, since the message queue can only be
1255    // checked in between two work packages.
1256    //
1257    // Work Processing Infrastructure
1258    // ==============================
1259    // The work processing infrastructure knows three major actors:
1260    //
1261    // - the coordinator thread,
1262    // - the main thread, and
1263    // - LLVM worker threads
1264    //
1265    // The coordinator thread is running a message loop. It instructs the main
1266    // thread about what work to do when, and it will spawn off LLVM worker
1267    // threads as open LLVM WorkItems become available.
1268    //
1269    // The job of the main thread is to codegen CGUs into LLVM work packages
1270    // (since the main thread is the only thread that can do this). The main
1271    // thread will block until it receives a message from the coordinator, upon
1272    // which it will codegen one CGU, send it to the coordinator and block
1273    // again. This way the coordinator can control what the main thread is
1274    // doing.
1275    //
1276    // The coordinator keeps a queue of LLVM WorkItems, and when a `Token` is
1277    // available, it will spawn off a new LLVM worker thread and let it process
1278    // a WorkItem. When a LLVM worker thread is done with its WorkItem,
1279    // it will just shut down, which also frees all resources associated with
1280    // the given LLVM module, and sends a message to the coordinator that the
1281    // WorkItem has been completed.
1282    //
1283    // Work Scheduling
1284    // ===============
1285    // The scheduler's goal is to minimize the time it takes to complete all
1286    // work there is, however, we also want to keep memory consumption low
1287    // if possible. These two goals are at odds with each other: If memory
1288    // consumption were not an issue, we could just let the main thread produce
1289    // LLVM WorkItems at full speed, assuring maximal utilization of
1290    // Tokens/LLVM worker threads. However, since codegen is usually faster
1291    // than LLVM processing, the queue of LLVM WorkItems would fill up and each
1292    // WorkItem potentially holds on to a substantial amount of memory.
1293    //
1294    // So the actual goal is to always produce just enough LLVM WorkItems as
1295    // not to starve our LLVM worker threads. That means, once we have enough
1296    // WorkItems in our queue, we can block the main thread, so it does not
1297    // produce more until we need them.
1298    //
1299    // Doing LLVM Work on the Main Thread
1300    // ----------------------------------
1301    // Since the main thread owns the compiler process's implicit `Token`, it is
1302    // wasteful to keep it blocked without doing any work. Therefore, what we do
1303    // in this case is: We spawn off an additional LLVM worker thread that helps
1304    // reduce the queue. The work it is doing corresponds to the implicit
1305    // `Token`. The coordinator will mark the main thread as being busy with
1306    // LLVM work. (The actual work happens on another OS thread but we just care
1307    // about `Tokens`, not actual threads).
1308    //
1309    // When any LLVM worker thread finishes while the main thread is marked as
1310    // "busy with LLVM work", we can do a little switcheroo: We give the Token
1311    // of the just finished thread to the LLVM worker thread that is working on
1312    // behalf of the main thread's implicit Token, thus freeing up the main
1313    // thread again. The coordinator can then again decide what the main thread
1314    // should do. This allows the coordinator to make decisions at more points
1315    // in time.
1316    //
1317    // Striking a Balance between Throughput and Memory Consumption
1318    // ------------------------------------------------------------
1319    // Since our two goals, (1) use as many Tokens as possible and (2) keep
1320    // memory consumption as low as possible, are in conflict with each other,
1321    // we have to find a trade off between them. Right now, the goal is to keep
1322    // all workers busy, which means that no worker should find the queue empty
1323    // when it is ready to start.
1324    // How do we do achieve this? Good question :) We actually never know how
1325    // many `Tokens` are potentially available so it's hard to say how much to
1326    // fill up the queue before switching the main thread to LLVM work. Also we
1327    // currently don't have a means to estimate how long a running LLVM worker
1328    // will still be busy with it's current WorkItem. However, we know the
1329    // maximal count of available Tokens that makes sense (=the number of CPU
1330    // cores), so we can take a conservative guess. The heuristic we use here
1331    // is implemented in the `queue_full_enough()` function.
1332    //
1333    // Some Background on Jobservers
1334    // -----------------------------
1335    // It's worth also touching on the management of parallelism here. We don't
1336    // want to just spawn a thread per work item because while that's optimal
1337    // parallelism it may overload a system with too many threads or violate our
1338    // configuration for the maximum amount of cpu to use for this process. To
1339    // manage this we use the `jobserver` crate.
1340    //
1341    // Job servers are an artifact of GNU make and are used to manage
1342    // parallelism between processes. A jobserver is a glorified IPC semaphore
1343    // basically. Whenever we want to run some work we acquire the semaphore,
1344    // and whenever we're done with that work we release the semaphore. In this
1345    // manner we can ensure that the maximum number of parallel workers is
1346    // capped at any one point in time.
1347    //
1348    // LTO and the coordinator thread
1349    // ------------------------------
1350    //
1351    // The final job the coordinator thread is responsible for is managing LTO
1352    // and how that works. When LTO is requested what we'll do is collect all
1353    // optimized LLVM modules into a local vector on the coordinator. Once all
1354    // modules have been codegened and optimized we hand this to the `lto`
1355    // module for further optimization. The `lto` module will return back a list
1356    // of more modules to work on, which the coordinator will continue to spawn
1357    // work for.
1358    //
1359    // Each LLVM module is automatically sent back to the coordinator for LTO if
1360    // necessary. There's already optimizations in place to avoid sending work
1361    // back to the coordinator if LTO isn't requested.
1362    return B::spawn_named_thread(cgcx.time_trace, "coordinator".to_string(), move || {
1363        let mut worker_id_counter = 0;
1364        let mut free_worker_ids = Vec::new();
1365        let mut get_worker_id = |free_worker_ids: &mut Vec<usize>| {
1366            if let Some(id) = free_worker_ids.pop() {
1367                id
1368            } else {
1369                let id = worker_id_counter;
1370                worker_id_counter += 1;
1371                id
1372            }
1373        };
1374
1375        // This is where we collect codegen units that have gone all the way
1376        // through codegen and LLVM.
1377        let mut autodiff_items = Vec::new();
1378        let mut compiled_modules = vec![];
1379        let mut compiled_allocator_module = None;
1380        let mut needs_link = Vec::new();
1381        let mut needs_fat_lto = Vec::new();
1382        let mut needs_thin_lto = Vec::new();
1383        let mut lto_import_only_modules = Vec::new();
1384        let mut started_lto = false;
1385
1386        /// Possible state transitions:
1387        /// - Ongoing -> Completed
1388        /// - Ongoing -> Aborted
1389        /// - Completed -> Aborted
1390        #[derive(Debug, PartialEq)]
1391        enum CodegenState {
1392            Ongoing,
1393            Completed,
1394            Aborted,
1395        }
1396        use CodegenState::*;
1397        let mut codegen_state = Ongoing;
1398
1399        // This is the queue of LLVM work items that still need processing.
1400        let mut work_items = Vec::<(WorkItem<B>, u64)>::new();
1401
1402        // This are the Jobserver Tokens we currently hold. Does not include
1403        // the implicit Token the compiler process owns no matter what.
1404        let mut tokens = Vec::new();
1405
1406        let mut main_thread_state = MainThreadState::Idle;
1407
1408        // How many LLVM worker threads are running while holding a Token. This
1409        // *excludes* any that the main thread is lending a Token to.
1410        let mut running_with_own_token = 0;
1411
1412        // How many LLVM worker threads are running in total. This *includes*
1413        // any that the main thread is lending a Token to.
1414        let running_with_any_token = |main_thread_state, running_with_own_token| {
1415            running_with_own_token
1416                + if main_thread_state == MainThreadState::Lending { 1 } else { 0 }
1417        };
1418
1419        let mut llvm_start_time: Option<VerboseTimingGuard<'_>> = None;
1420
1421        // Run the message loop while there's still anything that needs message
1422        // processing. Note that as soon as codegen is aborted we simply want to
1423        // wait for all existing work to finish, so many of the conditions here
1424        // only apply if codegen hasn't been aborted as they represent pending
1425        // work to be done.
1426        loop {
1427            // While there are still CGUs to be codegened, the coordinator has
1428            // to decide how to utilize the compiler processes implicit Token:
1429            // For codegenning more CGU or for running them through LLVM.
1430            if codegen_state == Ongoing {
1431                if main_thread_state == MainThreadState::Idle {
1432                    // Compute the number of workers that will be running once we've taken as many
1433                    // items from the work queue as we can, plus one for the main thread. It's not
1434                    // critically important that we use this instead of just
1435                    // `running_with_own_token`, but it prevents the `queue_full_enough` heuristic
1436                    // from fluctuating just because a worker finished up and we decreased the
1437                    // `running_with_own_token` count, even though we're just going to increase it
1438                    // right after this when we put a new worker to work.
1439                    let extra_tokens = tokens.len().checked_sub(running_with_own_token).unwrap();
1440                    let additional_running = std::cmp::min(extra_tokens, work_items.len());
1441                    let anticipated_running = running_with_own_token + additional_running + 1;
1442
1443                    if !queue_full_enough(work_items.len(), anticipated_running) {
1444                        // The queue is not full enough, process more codegen units:
1445                        if codegen_worker_send.send(CguMessage).is_err() {
1446                            panic!("Could not send CguMessage to main thread")
1447                        }
1448                        main_thread_state = MainThreadState::Codegenning;
1449                    } else {
1450                        // The queue is full enough to not let the worker
1451                        // threads starve. Use the implicit Token to do some
1452                        // LLVM work too.
1453                        let (item, _) =
1454                            work_items.pop().expect("queue empty - queue_full_enough() broken?");
1455                        main_thread_state = MainThreadState::Lending;
1456                        spawn_work(
1457                            &cgcx,
1458                            &mut llvm_start_time,
1459                            get_worker_id(&mut free_worker_ids),
1460                            item,
1461                        );
1462                    }
1463                }
1464            } else if codegen_state == Completed {
1465                if running_with_any_token(main_thread_state, running_with_own_token) == 0
1466                    && work_items.is_empty()
1467                {
1468                    // All codegen work is done. Do we have LTO work to do?
1469                    if needs_fat_lto.is_empty()
1470                        && needs_thin_lto.is_empty()
1471                        && lto_import_only_modules.is_empty()
1472                    {
1473                        // Nothing more to do!
1474                        break;
1475                    }
1476
1477                    // We have LTO work to do. Perform the serial work here of
1478                    // figuring out what we're going to LTO and then push a
1479                    // bunch of work items onto our queue to do LTO. This all
1480                    // happens on the coordinator thread but it's very quick so
1481                    // we don't worry about tokens.
1482                    assert!(!started_lto);
1483                    started_lto = true;
1484
1485                    let needs_fat_lto = mem::take(&mut needs_fat_lto);
1486                    let needs_thin_lto = mem::take(&mut needs_thin_lto);
1487                    let import_only_modules = mem::take(&mut lto_import_only_modules);
1488
1489                    for (work, cost) in generate_lto_work(
1490                        &cgcx,
1491                        autodiff_items.clone(),
1492                        needs_fat_lto,
1493                        needs_thin_lto,
1494                        import_only_modules,
1495                    ) {
1496                        let insertion_index = work_items
1497                            .binary_search_by_key(&cost, |&(_, cost)| cost)
1498                            .unwrap_or_else(|e| e);
1499                        work_items.insert(insertion_index, (work, cost));
1500                        if cgcx.parallel {
1501                            helper.request_token();
1502                        }
1503                    }
1504                }
1505
1506                // In this branch, we know that everything has been codegened,
1507                // so it's just a matter of determining whether the implicit
1508                // Token is free to use for LLVM work.
1509                match main_thread_state {
1510                    MainThreadState::Idle => {
1511                        if let Some((item, _)) = work_items.pop() {
1512                            main_thread_state = MainThreadState::Lending;
1513                            spawn_work(
1514                                &cgcx,
1515                                &mut llvm_start_time,
1516                                get_worker_id(&mut free_worker_ids),
1517                                item,
1518                            );
1519                        } else {
1520                            // There is no unstarted work, so let the main thread
1521                            // take over for a running worker. Otherwise the
1522                            // implicit token would just go to waste.
1523                            // We reduce the `running` counter by one. The
1524                            // `tokens.truncate()` below will take care of
1525                            // giving the Token back.
1526                            assert!(running_with_own_token > 0);
1527                            running_with_own_token -= 1;
1528                            main_thread_state = MainThreadState::Lending;
1529                        }
1530                    }
1531                    MainThreadState::Codegenning => bug!(
1532                        "codegen worker should not be codegenning after \
1533                              codegen was already completed"
1534                    ),
1535                    MainThreadState::Lending => {
1536                        // Already making good use of that token
1537                    }
1538                }
1539            } else {
1540                // Don't queue up any more work if codegen was aborted, we're
1541                // just waiting for our existing children to finish.
1542                assert!(codegen_state == Aborted);
1543                if running_with_any_token(main_thread_state, running_with_own_token) == 0 {
1544                    break;
1545                }
1546            }
1547
1548            // Spin up what work we can, only doing this while we've got available
1549            // parallelism slots and work left to spawn.
1550            if codegen_state != Aborted {
1551                while running_with_own_token < tokens.len()
1552                    && let Some((item, _)) = work_items.pop()
1553                {
1554                    spawn_work(
1555                        &cgcx,
1556                        &mut llvm_start_time,
1557                        get_worker_id(&mut free_worker_ids),
1558                        item,
1559                    );
1560                    running_with_own_token += 1;
1561                }
1562            }
1563
1564            // Relinquish accidentally acquired extra tokens.
1565            tokens.truncate(running_with_own_token);
1566
1567            // If a thread exits successfully then we drop a token associated
1568            // with that worker and update our `running_with_own_token` count.
1569            // We may later re-acquire a token to continue running more work.
1570            // We may also not actually drop a token here if the worker was
1571            // running with an "ephemeral token".
1572            let mut free_worker = |worker_id| {
1573                if main_thread_state == MainThreadState::Lending {
1574                    main_thread_state = MainThreadState::Idle;
1575                } else {
1576                    running_with_own_token -= 1;
1577                }
1578
1579                free_worker_ids.push(worker_id);
1580            };
1581
1582            let msg = coordinator_receive.recv().unwrap();
1583            match *msg.downcast::<Message<B>>().ok().unwrap() {
1584                // Save the token locally and the next turn of the loop will use
1585                // this to spawn a new unit of work, or it may get dropped
1586                // immediately if we have no more work to spawn.
1587                Message::Token(token) => {
1588                    match token {
1589                        Ok(token) => {
1590                            tokens.push(token);
1591
1592                            if main_thread_state == MainThreadState::Lending {
1593                                // If the main thread token is used for LLVM work
1594                                // at the moment, we turn that thread into a regular
1595                                // LLVM worker thread, so the main thread is free
1596                                // to react to codegen demand.
1597                                main_thread_state = MainThreadState::Idle;
1598                                running_with_own_token += 1;
1599                            }
1600                        }
1601                        Err(e) => {
1602                            let msg = &format!("failed to acquire jobserver token: {e}");
1603                            shared_emitter.fatal(msg);
1604                            codegen_state = Aborted;
1605                        }
1606                    }
1607                }
1608
1609                Message::CodegenDone { llvm_work_item, cost } => {
1610                    // We keep the queue sorted by estimated processing cost,
1611                    // so that more expensive items are processed earlier. This
1612                    // is good for throughput as it gives the main thread more
1613                    // time to fill up the queue and it avoids scheduling
1614                    // expensive items to the end.
1615                    // Note, however, that this is not ideal for memory
1616                    // consumption, as LLVM module sizes are not evenly
1617                    // distributed.
1618                    let insertion_index = work_items.binary_search_by_key(&cost, |&(_, cost)| cost);
1619                    let insertion_index = match insertion_index {
1620                        Ok(idx) | Err(idx) => idx,
1621                    };
1622                    work_items.insert(insertion_index, (llvm_work_item, cost));
1623
1624                    if cgcx.parallel {
1625                        helper.request_token();
1626                    }
1627                    assert_eq!(main_thread_state, MainThreadState::Codegenning);
1628                    main_thread_state = MainThreadState::Idle;
1629                }
1630
1631                Message::AddAutoDiffItems(mut items) => {
1632                    autodiff_items.append(&mut items);
1633                }
1634
1635                Message::CodegenComplete => {
1636                    if codegen_state != Aborted {
1637                        codegen_state = Completed;
1638                    }
1639                    assert_eq!(main_thread_state, MainThreadState::Codegenning);
1640                    main_thread_state = MainThreadState::Idle;
1641                }
1642
1643                // If codegen is aborted that means translation was aborted due
1644                // to some normal-ish compiler error. In this situation we want
1645                // to exit as soon as possible, but we want to make sure all
1646                // existing work has finished. Flag codegen as being done, and
1647                // then conditions above will ensure no more work is spawned but
1648                // we'll keep executing this loop until `running_with_own_token`
1649                // hits 0.
1650                Message::CodegenAborted => {
1651                    codegen_state = Aborted;
1652                }
1653
1654                Message::WorkItem { result, worker_id } => {
1655                    free_worker(worker_id);
1656
1657                    match result {
1658                        Ok(WorkItemResult::Finished(compiled_module)) => {
1659                            match compiled_module.kind {
1660                                ModuleKind::Regular => {
1661                                    assert!(needs_link.is_empty());
1662                                    compiled_modules.push(compiled_module);
1663                                }
1664                                ModuleKind::Allocator => {
1665                                    assert!(compiled_allocator_module.is_none());
1666                                    compiled_allocator_module = Some(compiled_module);
1667                                }
1668                                ModuleKind::Metadata => bug!("Should be handled separately"),
1669                            }
1670                        }
1671                        Ok(WorkItemResult::NeedsLink(module)) => {
1672                            assert!(compiled_modules.is_empty());
1673                            needs_link.push(module);
1674                        }
1675                        Ok(WorkItemResult::NeedsFatLto(fat_lto_input)) => {
1676                            assert!(!started_lto);
1677                            assert!(needs_thin_lto.is_empty());
1678                            needs_fat_lto.push(fat_lto_input);
1679                        }
1680                        Ok(WorkItemResult::NeedsThinLto(name, thin_buffer)) => {
1681                            assert!(!started_lto);
1682                            assert!(needs_fat_lto.is_empty());
1683                            needs_thin_lto.push((name, thin_buffer));
1684                        }
1685                        Err(Some(WorkerFatalError)) => {
1686                            // Like `CodegenAborted`, wait for remaining work to finish.
1687                            codegen_state = Aborted;
1688                        }
1689                        Err(None) => {
1690                            // If the thread failed that means it panicked, so
1691                            // we abort immediately.
1692                            bug!("worker thread panicked");
1693                        }
1694                    }
1695                }
1696
1697                Message::AddImportOnlyModule { module_data, work_product } => {
1698                    assert!(!started_lto);
1699                    assert_eq!(codegen_state, Ongoing);
1700                    assert_eq!(main_thread_state, MainThreadState::Codegenning);
1701                    lto_import_only_modules.push((module_data, work_product));
1702                    main_thread_state = MainThreadState::Idle;
1703                }
1704            }
1705        }
1706
1707        if codegen_state == Aborted {
1708            return Err(());
1709        }
1710
1711        let needs_link = mem::take(&mut needs_link);
1712        if !needs_link.is_empty() {
1713            assert!(compiled_modules.is_empty());
1714            let dcx = cgcx.create_dcx();
1715            let dcx = dcx.handle();
1716            let module = B::run_link(&cgcx, dcx, needs_link).map_err(|_| ())?;
1717            let module = unsafe {
1718                B::codegen(&cgcx, dcx, module, cgcx.config(ModuleKind::Regular)).map_err(|_| ())?
1719            };
1720            compiled_modules.push(module);
1721        }
1722
1723        // Drop to print timings
1724        drop(llvm_start_time);
1725
1726        // Regardless of what order these modules completed in, report them to
1727        // the backend in the same order every time to ensure that we're handing
1728        // out deterministic results.
1729        compiled_modules.sort_by(|a, b| a.name.cmp(&b.name));
1730
1731        Ok(CompiledModules {
1732            modules: compiled_modules,
1733            allocator_module: compiled_allocator_module,
1734        })
1735    })
1736    .expect("failed to spawn coordinator thread");
1737
1738    // A heuristic that determines if we have enough LLVM WorkItems in the
1739    // queue so that the main thread can do LLVM work instead of codegen
1740    fn queue_full_enough(items_in_queue: usize, workers_running: usize) -> bool {
1741        // This heuristic scales ahead-of-time codegen according to available
1742        // concurrency, as measured by `workers_running`. The idea is that the
1743        // more concurrency we have available, the more demand there will be for
1744        // work items, and the fuller the queue should be kept to meet demand.
1745        // An important property of this approach is that we codegen ahead of
1746        // time only as much as necessary, so as to keep fewer LLVM modules in
1747        // memory at once, thereby reducing memory consumption.
1748        //
1749        // When the number of workers running is less than the max concurrency
1750        // available to us, this heuristic can cause us to instruct the main
1751        // thread to work on an LLVM item (that is, tell it to "LLVM") instead
1752        // of codegen, even though it seems like it *should* be codegenning so
1753        // that we can create more work items and spawn more LLVM workers.
1754        //
1755        // But this is not a problem. When the main thread is told to LLVM,
1756        // according to this heuristic and how work is scheduled, there is
1757        // always at least one item in the queue, and therefore at least one
1758        // pending jobserver token request. If there *is* more concurrency
1759        // available, we will immediately receive a token, which will upgrade
1760        // the main thread's LLVM worker to a real one (conceptually), and free
1761        // up the main thread to codegen if necessary. On the other hand, if
1762        // there isn't more concurrency, then the main thread working on an LLVM
1763        // item is appropriate, as long as the queue is full enough for demand.
1764        //
1765        // Speaking of which, how full should we keep the queue? Probably less
1766        // full than you'd think. A lot has to go wrong for the queue not to be
1767        // full enough and for that to have a negative effect on compile times.
1768        //
1769        // Workers are unlikely to finish at exactly the same time, so when one
1770        // finishes and takes another work item off the queue, we often have
1771        // ample time to codegen at that point before the next worker finishes.
1772        // But suppose that codegen takes so long that the workers exhaust the
1773        // queue, and we have one or more workers that have nothing to work on.
1774        // Well, it might not be so bad. Of all the LLVM modules we create and
1775        // optimize, one has to finish last. It's not necessarily the case that
1776        // by losing some concurrency for a moment, we delay the point at which
1777        // that last LLVM module is finished and the rest of compilation can
1778        // proceed. Also, when we can't take advantage of some concurrency, we
1779        // give tokens back to the job server. That enables some other rustc to
1780        // potentially make use of the available concurrency. That could even
1781        // *decrease* overall compile time if we're lucky. But yes, if no other
1782        // rustc can make use of the concurrency, then we've squandered it.
1783        //
1784        // However, keeping the queue full is also beneficial when we have a
1785        // surge in available concurrency. Then items can be taken from the
1786        // queue immediately, without having to wait for codegen.
1787        //
1788        // So, the heuristic below tries to keep one item in the queue for every
1789        // four running workers. Based on limited benchmarking, this appears to
1790        // be more than sufficient to avoid increasing compilation times.
1791        let quarter_of_workers = workers_running - 3 * workers_running / 4;
1792        items_in_queue > 0 && items_in_queue >= quarter_of_workers
1793    }
1794}
1795
1796/// `FatalError` is explicitly not `Send`.
1797#[must_use]
1798pub(crate) struct WorkerFatalError;
1799
1800fn spawn_work<'a, B: ExtraBackendMethods>(
1801    cgcx: &'a CodegenContext<B>,
1802    llvm_start_time: &mut Option<VerboseTimingGuard<'a>>,
1803    worker_id: usize,
1804    work: WorkItem<B>,
1805) {
1806    if cgcx.config(work.module_kind()).time_module && llvm_start_time.is_none() {
1807        *llvm_start_time = Some(cgcx.prof.verbose_generic_activity("LLVM_passes"));
1808    }
1809
1810    let cgcx = cgcx.clone();
1811
1812    B::spawn_named_thread(cgcx.time_trace, work.short_description(), move || {
1813        // Set up a destructor which will fire off a message that we're done as
1814        // we exit.
1815        struct Bomb<B: ExtraBackendMethods> {
1816            coordinator_send: Sender<Box<dyn Any + Send>>,
1817            result: Option<Result<WorkItemResult<B>, FatalError>>,
1818            worker_id: usize,
1819        }
1820        impl<B: ExtraBackendMethods> Drop for Bomb<B> {
1821            fn drop(&mut self) {
1822                let worker_id = self.worker_id;
1823                let msg = match self.result.take() {
1824                    Some(Ok(result)) => Message::WorkItem::<B> { result: Ok(result), worker_id },
1825                    Some(Err(FatalError)) => {
1826                        Message::WorkItem::<B> { result: Err(Some(WorkerFatalError)), worker_id }
1827                    }
1828                    None => Message::WorkItem::<B> { result: Err(None), worker_id },
1829                };
1830                drop(self.coordinator_send.send(Box::new(msg)));
1831            }
1832        }
1833
1834        let mut bomb =
1835            Bomb::<B> { coordinator_send: cgcx.coordinator_send.clone(), result: None, worker_id };
1836
1837        // Execute the work itself, and if it finishes successfully then flag
1838        // ourselves as a success as well.
1839        //
1840        // Note that we ignore any `FatalError` coming out of `execute_work_item`,
1841        // as a diagnostic was already sent off to the main thread - just
1842        // surface that there was an error in this worker.
1843        bomb.result = {
1844            let module_config = cgcx.config(work.module_kind());
1845
1846            Some(match work {
1847                WorkItem::Optimize(m) => {
1848                    let _timer =
1849                        cgcx.prof.generic_activity_with_arg("codegen_module_optimize", &*m.name);
1850                    execute_optimize_work_item(&cgcx, m, module_config)
1851                }
1852                WorkItem::CopyPostLtoArtifacts(m) => {
1853                    let _timer = cgcx.prof.generic_activity_with_arg(
1854                        "codegen_copy_artifacts_from_incr_cache",
1855                        &*m.name,
1856                    );
1857                    Ok(execute_copy_from_cache_work_item(&cgcx, m, module_config))
1858                }
1859                WorkItem::LTO(m) => {
1860                    let _timer =
1861                        cgcx.prof.generic_activity_with_arg("codegen_module_perform_lto", m.name());
1862                    execute_lto_work_item(&cgcx, m, module_config)
1863                }
1864            })
1865        };
1866    })
1867    .expect("failed to spawn work thread");
1868}
1869
1870enum SharedEmitterMessage {
1871    Diagnostic(Diagnostic),
1872    InlineAsmError(SpanData, String, Level, Option<(String, Vec<InnerSpan>)>),
1873    Fatal(String),
1874}
1875
1876#[derive(Clone)]
1877pub struct SharedEmitter {
1878    sender: Sender<SharedEmitterMessage>,
1879}
1880
1881pub struct SharedEmitterMain {
1882    receiver: Receiver<SharedEmitterMessage>,
1883}
1884
1885impl SharedEmitter {
1886    fn new() -> (SharedEmitter, SharedEmitterMain) {
1887        let (sender, receiver) = channel();
1888
1889        (SharedEmitter { sender }, SharedEmitterMain { receiver })
1890    }
1891
1892    pub fn inline_asm_error(
1893        &self,
1894        span: SpanData,
1895        msg: String,
1896        level: Level,
1897        source: Option<(String, Vec<InnerSpan>)>,
1898    ) {
1899        drop(self.sender.send(SharedEmitterMessage::InlineAsmError(span, msg, level, source)));
1900    }
1901
1902    fn fatal(&self, msg: &str) {
1903        drop(self.sender.send(SharedEmitterMessage::Fatal(msg.to_string())));
1904    }
1905}
1906
1907impl Translate for SharedEmitter {
1908    fn fluent_bundle(&self) -> Option<&FluentBundle> {
1909        None
1910    }
1911
1912    fn fallback_fluent_bundle(&self) -> &FluentBundle {
1913        panic!("shared emitter attempted to translate a diagnostic");
1914    }
1915}
1916
1917impl Emitter for SharedEmitter {
1918    fn emit_diagnostic(
1919        &mut self,
1920        mut diag: rustc_errors::DiagInner,
1921        _registry: &rustc_errors::registry::Registry,
1922    ) {
1923        // Check that we aren't missing anything interesting when converting to
1924        // the cut-down local `DiagInner`.
1925        assert_eq!(diag.span, MultiSpan::new());
1926        assert_eq!(diag.suggestions, Suggestions::Enabled(vec![]));
1927        assert_eq!(diag.sort_span, rustc_span::DUMMY_SP);
1928        assert_eq!(diag.is_lint, None);
1929        // No sensible check for `diag.emitted_at`.
1930
1931        let args = mem::replace(&mut diag.args, DiagArgMap::default());
1932        drop(
1933            self.sender.send(SharedEmitterMessage::Diagnostic(Diagnostic {
1934                level: diag.level(),
1935                messages: diag.messages,
1936                code: diag.code,
1937                children: diag
1938                    .children
1939                    .into_iter()
1940                    .map(|child| Subdiagnostic { level: child.level, messages: child.messages })
1941                    .collect(),
1942                args,
1943            })),
1944        );
1945    }
1946
1947    fn source_map(&self) -> Option<&SourceMap> {
1948        None
1949    }
1950}
1951
1952impl SharedEmitterMain {
1953    fn check(&self, sess: &Session, blocking: bool) {
1954        loop {
1955            let message = if blocking {
1956                match self.receiver.recv() {
1957                    Ok(message) => Ok(message),
1958                    Err(_) => Err(()),
1959                }
1960            } else {
1961                match self.receiver.try_recv() {
1962                    Ok(message) => Ok(message),
1963                    Err(_) => Err(()),
1964                }
1965            };
1966
1967            match message {
1968                Ok(SharedEmitterMessage::Diagnostic(diag)) => {
1969                    // The diagnostic has been received on the main thread.
1970                    // Convert it back to a full `Diagnostic` and emit.
1971                    let dcx = sess.dcx();
1972                    let mut d =
1973                        rustc_errors::DiagInner::new_with_messages(diag.level, diag.messages);
1974                    d.code = diag.code; // may be `None`, that's ok
1975                    d.children = diag
1976                        .children
1977                        .into_iter()
1978                        .map(|sub| rustc_errors::Subdiag {
1979                            level: sub.level,
1980                            messages: sub.messages,
1981                            span: MultiSpan::new(),
1982                        })
1983                        .collect();
1984                    d.args = diag.args;
1985                    dcx.emit_diagnostic(d);
1986                    sess.dcx().abort_if_errors();
1987                }
1988                Ok(SharedEmitterMessage::InlineAsmError(span, msg, level, source)) => {
1989                    assert_matches!(level, Level::Error | Level::Warning | Level::Note);
1990                    let mut err = Diag::<()>::new(sess.dcx(), level, msg);
1991                    if !span.is_dummy() {
1992                        err.span(span.span());
1993                    }
1994
1995                    // Point to the generated assembly if it is available.
1996                    if let Some((buffer, spans)) = source {
1997                        let source = sess
1998                            .source_map()
1999                            .new_source_file(FileName::inline_asm_source_code(&buffer), buffer);
2000                        let spans: Vec<_> = spans
2001                            .iter()
2002                            .map(|sp| {
2003                                Span::with_root_ctxt(
2004                                    source.normalized_byte_pos(sp.start as u32),
2005                                    source.normalized_byte_pos(sp.end as u32),
2006                                )
2007                            })
2008                            .collect();
2009                        err.span_note(spans, "instantiated into assembly here");
2010                    }
2011
2012                    err.emit();
2013                }
2014                Ok(SharedEmitterMessage::Fatal(msg)) => {
2015                    sess.dcx().fatal(msg);
2016                }
2017                Err(_) => {
2018                    break;
2019                }
2020            }
2021        }
2022    }
2023}
2024
2025pub struct Coordinator<B: ExtraBackendMethods> {
2026    pub sender: Sender<Box<dyn Any + Send>>,
2027    future: Option<thread::JoinHandle<Result<CompiledModules, ()>>>,
2028    // Only used for the Message type.
2029    phantom: PhantomData<B>,
2030}
2031
2032impl<B: ExtraBackendMethods> Coordinator<B> {
2033    fn join(mut self) -> std::thread::Result<Result<CompiledModules, ()>> {
2034        self.future.take().unwrap().join()
2035    }
2036}
2037
2038impl<B: ExtraBackendMethods> Drop for Coordinator<B> {
2039    fn drop(&mut self) {
2040        if let Some(future) = self.future.take() {
2041            // If we haven't joined yet, signal to the coordinator that it should spawn no more
2042            // work, and wait for worker threads to finish.
2043            drop(self.sender.send(Box::new(Message::CodegenAborted::<B>)));
2044            drop(future.join());
2045        }
2046    }
2047}
2048
2049pub struct OngoingCodegen<B: ExtraBackendMethods> {
2050    pub backend: B,
2051    pub metadata: EncodedMetadata,
2052    pub metadata_module: Option<CompiledModule>,
2053    pub crate_info: CrateInfo,
2054    pub codegen_worker_receive: Receiver<CguMessage>,
2055    pub shared_emitter_main: SharedEmitterMain,
2056    pub output_filenames: Arc<OutputFilenames>,
2057    pub coordinator: Coordinator<B>,
2058}
2059
2060impl<B: ExtraBackendMethods> OngoingCodegen<B> {
2061    pub fn join(self, sess: &Session) -> (CodegenResults, FxIndexMap<WorkProductId, WorkProduct>) {
2062        self.shared_emitter_main.check(sess, true);
2063        let compiled_modules = sess.time("join_worker_thread", || match self.coordinator.join() {
2064            Ok(Ok(compiled_modules)) => compiled_modules,
2065            Ok(Err(())) => {
2066                sess.dcx().abort_if_errors();
2067                panic!("expected abort due to worker thread errors")
2068            }
2069            Err(_) => {
2070                bug!("panic during codegen/LLVM phase");
2071            }
2072        });
2073
2074        sess.dcx().abort_if_errors();
2075
2076        let work_products =
2077            copy_all_cgu_workproducts_to_incr_comp_cache_dir(sess, &compiled_modules);
2078        produce_final_output_artifacts(sess, &compiled_modules, &self.output_filenames);
2079
2080        // FIXME: time_llvm_passes support - does this use a global context or
2081        // something?
2082        if sess.codegen_units().as_usize() == 1 && sess.opts.unstable_opts.time_llvm_passes {
2083            self.backend.print_pass_timings()
2084        }
2085
2086        if sess.print_llvm_stats() {
2087            self.backend.print_statistics()
2088        }
2089
2090        (
2091            CodegenResults {
2092                metadata: self.metadata,
2093                crate_info: self.crate_info,
2094
2095                modules: compiled_modules.modules,
2096                allocator_module: compiled_modules.allocator_module,
2097                metadata_module: self.metadata_module,
2098            },
2099            work_products,
2100        )
2101    }
2102
2103    pub(crate) fn codegen_finished(&self, tcx: TyCtxt<'_>) {
2104        self.wait_for_signal_to_codegen_item();
2105        self.check_for_errors(tcx.sess);
2106        drop(self.coordinator.sender.send(Box::new(Message::CodegenComplete::<B>)));
2107    }
2108
2109    pub(crate) fn submit_autodiff_items(&self, items: Vec<AutoDiffItem>) {
2110        drop(self.coordinator.sender.send(Box::new(Message::<B>::AddAutoDiffItems(items))));
2111    }
2112
2113    pub(crate) fn check_for_errors(&self, sess: &Session) {
2114        self.shared_emitter_main.check(sess, false);
2115    }
2116
2117    pub(crate) fn wait_for_signal_to_codegen_item(&self) {
2118        match self.codegen_worker_receive.recv() {
2119            Ok(CguMessage) => {
2120                // Ok to proceed.
2121            }
2122            Err(_) => {
2123                // One of the LLVM threads must have panicked, fall through so
2124                // error handling can be reached.
2125            }
2126        }
2127    }
2128}
2129
2130pub(crate) fn submit_codegened_module_to_llvm<B: ExtraBackendMethods>(
2131    _backend: &B,
2132    tx_to_llvm_workers: &Sender<Box<dyn Any + Send>>,
2133    module: ModuleCodegen<B::Module>,
2134    cost: u64,
2135) {
2136    let llvm_work_item = WorkItem::Optimize(module);
2137    drop(tx_to_llvm_workers.send(Box::new(Message::CodegenDone::<B> { llvm_work_item, cost })));
2138}
2139
2140pub(crate) fn submit_post_lto_module_to_llvm<B: ExtraBackendMethods>(
2141    _backend: &B,
2142    tx_to_llvm_workers: &Sender<Box<dyn Any + Send>>,
2143    module: CachedModuleCodegen,
2144) {
2145    let llvm_work_item = WorkItem::CopyPostLtoArtifacts(module);
2146    drop(tx_to_llvm_workers.send(Box::new(Message::CodegenDone::<B> { llvm_work_item, cost: 0 })));
2147}
2148
2149pub(crate) fn submit_pre_lto_module_to_llvm<B: ExtraBackendMethods>(
2150    _backend: &B,
2151    tcx: TyCtxt<'_>,
2152    tx_to_llvm_workers: &Sender<Box<dyn Any + Send>>,
2153    module: CachedModuleCodegen,
2154) {
2155    let filename = pre_lto_bitcode_filename(&module.name);
2156    let bc_path = in_incr_comp_dir_sess(tcx.sess, &filename);
2157    let file = fs::File::open(&bc_path)
2158        .unwrap_or_else(|e| panic!("failed to open bitcode file `{}`: {}", bc_path.display(), e));
2159
2160    let mmap = unsafe {
2161        Mmap::map(file).unwrap_or_else(|e| {
2162            panic!("failed to mmap bitcode file `{}`: {}", bc_path.display(), e)
2163        })
2164    };
2165    // Schedule the module to be loaded
2166    drop(tx_to_llvm_workers.send(Box::new(Message::AddImportOnlyModule::<B> {
2167        module_data: SerializedModule::FromUncompressedFile(mmap),
2168        work_product: module.source,
2169    })));
2170}
2171
2172fn pre_lto_bitcode_filename(module_name: &str) -> String {
2173    format!("{module_name}.{PRE_LTO_BC_EXT}")
2174}
2175
2176fn msvc_imps_needed(tcx: TyCtxt<'_>) -> bool {
2177    // This should never be true (because it's not supported). If it is true,
2178    // something is wrong with commandline arg validation.
2179    assert!(
2180        !(tcx.sess.opts.cg.linker_plugin_lto.enabled()
2181            && tcx.sess.target.is_like_windows
2182            && tcx.sess.opts.cg.prefer_dynamic)
2183    );
2184
2185    // We need to generate _imp__ symbol if we are generating an rlib or we include one
2186    // indirectly from ThinLTO. In theory these are not needed as ThinLTO could resolve
2187    // these, but it currently does not do so.
2188    let can_have_static_objects =
2189        tcx.sess.lto() == Lto::Thin || tcx.crate_types().iter().any(|ct| *ct == CrateType::Rlib);
2190
2191    tcx.sess.target.is_like_windows &&
2192    can_have_static_objects   &&
2193    // ThinLTO can't handle this workaround in all cases, so we don't
2194    // emit the `__imp_` symbols. Instead we make them unnecessary by disallowing
2195    // dynamic linking when linker plugin LTO is enabled.
2196    !tcx.sess.opts.cg.linker_plugin_lto.enabled()
2197}