Skip to content

Multiverse Analysis

This class orchestrates a multiverse analysis.

Attributes:

Name Type Description
dimensions

A dictionary where keys are dimension names and values are lists of possible values for each dimension.

constraints

Optional dictionary where keys are dimension names and values are lists of constraints. Each constraint is a dictionary with: - value: The value of the dimension that the constraint applies to. - allowed_if: A dictionary of dimension-value pairs that must be present for the constraint to be allowed. - forbidden_if: A dictionary of dimension-value pairs that must not be present for the constraint to be allowed. Only one of allowed_if and forbidden_if can be present in a constraint. Example: constraints = { "dimension1": [ { "value": "value1", "allowed_if": {"dimension2": "value2"} }, { "value": "value3", "forbidden_if": {"dimension4": "value4"} } ] }

seed

The seed to use for the analysis.

cell_timeout

A timeout (in seconds) for each cell in the notebook.

stop_on_error

Whether to stop the analysis if an error occurs.

run_no int

The number of the current run.

new_run bool

Whether this is a new run or not.

output_dir Path

The directory to store the output in.

universe_file Path

The Path to the universe file to run.

grid Optional[List[Dict[str, Any]]]

Optional list of dictionaries containing the settings for different universes.

Source code in multiversum/multiverse.py
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
class MultiverseAnalysis:
    """
    This class orchestrates a multiverse analysis.

    Attributes:
        dimensions: A dictionary where keys are dimension names and values are lists of possible values for each dimension.
        constraints: Optional dictionary where keys are dimension names and values are lists of constraints. Each constraint is a dictionary with:
            - value: The value of the dimension that the constraint applies to.
            - allowed_if: A dictionary of dimension-value pairs that must be present for the constraint to be allowed.
            - forbidden_if: A dictionary of dimension-value pairs that must not be present for the constraint to be allowed.
            Only one of allowed_if and forbidden_if can be present in a constraint.
            Example:
                constraints = {
                    "dimension1": [
                        {
                            "value": "value1",
                            "allowed_if": {"dimension2": "value2"}
                        },
                        {
                            "value": "value3",
                            "forbidden_if": {"dimension4": "value4"}
                        }
                    ]
                }
        seed: The seed to use for the analysis.
        cell_timeout: A timeout (in seconds) for each cell in the notebook.
        stop_on_error: Whether to stop the analysis if an error occurs.
        run_no: The number of the current run.
        new_run: Whether this is a new run or not.
        output_dir: The directory to store the output in.
        universe_file: The Path to the universe file to run.
        grid: Optional list of dictionaries containing the settings for different universes.
    """

    dimensions = None
    constraints = None
    seed = DEFAULT_SEED
    cell_timeout = None
    stop_on_error = DEFAULT_STOP_ON_ERROR

    run_no: int
    new_run: bool
    output_dir: Path
    universe_file: Path

    grid: Optional[List[Dict[str, Any]]] = None

    def __init__(
        self,
        dimensions: Optional[Dict] = None,
        config: Union[Path, Config, None] = None,
        universe: Path = None,
        output_dir: Path = Path("./output"),
        run_no: Optional[int] = None,
        new_run: bool = True,
        seed: Optional[int] = None,
        stop_on_error: Optional[bool] = None,
        cell_timeout: Optional[int] = None,
    ) -> None:
        """
        Initializes a new MultiverseAnalysis instance.

        Args:
            dimensions: A dictionary where keys are dimension names and values are lists of possible values for each dimension.
                Each dimension corresponds to a decision.
            config: A Path to a TOML, JSON or Python file containing the
                analysis configuration. Supported configuration options can be
                found in the Config class. If a Python file is used, it should
                contain a dictionary / config object named "config".
                Will automatically search for multiverse.toml / .json / .py.
            universe: The Path to the universe_file to run. Either an
                ipython / jupyter notebook (.ipynb) or a python script (.py).
            output_dir: The directory to store the output in.
            run_no: The number of the current run. Defaults to an automatically
                incrementing integer number if new_run is True or the last run if
                new_run is False.
            new_run: Whether this is a new run or not. Defaults to True.
            seed: The seed to use for the analysis.
            stop_on_error: Whether to stop the analysis if an error occurs.
            cell_timeout: A timeout (in seconds) for each cell in the notebook.
        """
        # Check for configuration file and parse it
        config_file = search_files(file=config, default_files=DEFAULT_CONFIG_FILES)
        if isinstance(config_file, Path):
            if config_file.suffix == ".toml":
                with open(config_file, "rb") as fp:
                    config = tomllib.load(fp)
            elif config_file.suffix == ".json":
                with open(config_file, "r") as fp:
                    config = json.load(fp)
            elif config_file.suffix == ".py":
                results = runpy.run_path(str(config_file))
                config = results["config"]
            else:
                raise ValueError(
                    "Only .toml, .json and .py files are supported as config."
                )
        # Convert config to Config object
        if isinstance(config, dict):
            config = Config(**config)

        # Read settings from config (or args)
        self.read_config_value(config, "dimensions", dimensions)
        self.read_config_value(config, "seed", seed)
        self.read_config_value(config, "stop_on_error", stop_on_error)
        self.read_config_value(config, "cell_timeout", cell_timeout)
        self.read_config_value(config, "constraints")

        universe_file = search_files(
            file=universe, default_files=DEFAULT_UNIVERSE_FILES
        )
        self.universe_file = universe_file

        self.output_dir = output_dir
        if self.output_dir is not None:
            self.output_dir.mkdir(parents=True, exist_ok=True)

        self.run_no = (
            run_no if run_no is not None else self.read_counter(increment=new_run)
        )

        if self.dimensions is None:
            raise ValueError(
                "Dimensions need to be specified either directly or in a config."
            )

    def read_config_value(
        self, config: Optional[Config], key: str, overwrite_value: Optional[Any] = None
    ):
        config_value = getattr(config, key) if config is not None else None

        if overwrite_value is not None:
            if config_value is not None:
                logger.warning(
                    f"Overwriting config value {key} ({config_value}) with {overwrite_value} as it was passed directly."
                )
            setattr(self, key, overwrite_value)
        elif config_value is not None:
            # Use value from config
            setattr(self, key, config_value)

    def get_run_dir(self, sub_directory: Optional[str] = None) -> Path:
        """
        Get the directory for the current run.

        Args:
            sub_directory: An optional sub-directory to append to the run directory.

        Returns:
            A Path object for the current run directory.
        """
        run_dir = self.output_dir / "runs" / str(self.run_no)
        target_dir = run_dir / sub_directory if sub_directory is not None else run_dir
        target_dir.mkdir(parents=True, exist_ok=True)
        return target_dir

    def read_counter(self, increment: bool) -> int:
        """
        Read the counter from the output directory.

        Args:
            increment: Whether to increment the counter after reading.

        Returns:
            The current value of the counter.
        """

        # Use a self-incrementing counter via counter.txt
        counter_filepath = self.output_dir / "counter.txt"
        if counter_filepath.is_file():
            with open(counter_filepath, "r") as fp:
                run_no = int(fp.read())
        else:
            run_no = 0
        if increment:
            run_no += 1
        with open(counter_filepath, "w") as fp:
            fp.write(str(run_no))

        return run_no

    def generate_grid(self, save: bool = True) -> List[Dict[str, Any]]:
        """
        Generate the multiverse grid from the stored dimensions.

        Args:
            save: Whether to save the multiverse grid to a JSON file.

        Returns:
            A list of dicts containing the settings for different universes.
        """
        self.grid = generate_multiverse_grid(self.dimensions, self.constraints)
        if save:
            with open(self.output_dir / "multiverse_grid.json", "w") as fp:
                json.dump(self.grid, fp, indent=2)
        return self.grid

    def aggregate_data(
        self, include_errors: bool = True, save: bool = True
    ) -> pd.DataFrame:
        """
        Aggregate the data from all universes into a single DataFrame.

        Args:
            include_errors: Whether to include error information.
            save: Whether to save the aggregated data to a file.

        Returns:
            A pandas DataFrame containing the aggregated data from all universes.
        """
        data_dir = self.get_run_dir(sub_directory="data")
        csv_files = list(data_dir.glob("*.csv"))

        if include_errors:
            error_dir = self.get_run_dir(sub_directory=ERRORS_DIR_NAME)
            csv_files += list(error_dir.glob("*.csv"))

        if len(csv_files) == 0:
            logger.warning("No data files to aggregate, returning empty dataframe.")
            df = pd.DataFrame({"mv_universe_id": []})
        else:
            df = pd.concat((pd.read_csv(f) for f in csv_files), ignore_index=True)

        if save:
            df.to_csv(data_dir / ("agg_" + str(self.run_no) + "_run_outputs.csv.gz"))

        return df

    def check_missing_universes(self) -> MissingUniverseInfo:
        """
        Check if any universes from the multiverse have not yet been visited.

        Returns:
            A dictionary containing the missing universe ids, additional
                universe ids (i.e. not in the current multiverse_grid)
                and the dictionaries for the missing universes.
        """
        multiverse_dict = add_ids_to_multiverse_grid(self.generate_grid(save=False))
        all_universe_ids = set(multiverse_dict.keys())

        aggregated_data = self.aggregate_data(include_errors=False, save=False)
        universe_ids_with_data = set(aggregated_data["mv_universe_id"])

        missing_universe_ids = all_universe_ids - universe_ids_with_data
        extra_universe_ids = universe_ids_with_data - all_universe_ids
        missing_universes = [multiverse_dict[u_id] for u_id in missing_universe_ids]

        if len(missing_universe_ids) > 0 or len(extra_universe_ids) > 0:
            logger.warning(
                f"Found missing {len(missing_universe_ids)} / "
                f"additional {len(extra_universe_ids)} universe ids!"
            )

        return {
            "missing_universe_ids": missing_universe_ids,
            "extra_universe_ids": extra_universe_ids,
            "missing_universes": missing_universes,
        }

    def examine_multiverse(
        self, multiverse_grid: List[Dict[str, Any]] = None, n_jobs: int = -2
    ) -> None:
        """
        Run the analysis for all universes in the multiverse.

        Args:
            multiverse_grid: A list of dictionaries containing the settings for different universes.
            n_jobs: The number of jobs to run in parallel. Defaults to -2 (all CPUs but one).

        Returns:
            None
        """
        if multiverse_grid is None:
            multiverse_grid = self.grid or self.generate_grid(save=False)

        # Run analysis for all universes
        with Progress(
            TextColumn("[progress.description]{task.description}"),
            SpinnerColumn(),
            TaskProgressColumn(),
            BarColumn(bar_width=None),
            MofNCompleteColumn(),
            TextColumn("•"),
            TimeElapsedColumn(),
            TextColumn("•"),
            TimeRemainingColumn(),
            expand=True,
        ) as progress:
            task_id = progress.add_task("Running", total=len(multiverse_grid))
            if n_jobs == 1:
                logger.info("Running in single-threaded mode (njobs = 1).")
                for universe_params in multiverse_grid:
                    self.visit_universe(universe_params)
                    progress.update(task_id, advance=1)
                    # Somehow automatic updating is not working in single threaded mode, so we manually refresh
                    progress.refresh()
            else:
                logger.info(
                    f"Running in parallel mode (njobs = {n_jobs}; {cpu_count()} CPUs detected)."
                )
                with rich_joblib(progress, task_id):
                    # For n_jobs below -1, (n_cpus + 1 + n_jobs) are used.
                    # Thus for n_jobs = -2, all CPUs but one are used
                    Parallel(n_jobs=n_jobs)(
                        delayed(self.visit_universe)(universe_params)
                        for universe_params in multiverse_grid
                    )

    def visit_universe(self, universe_dimensions: Dict[str, str]) -> None:
        """
        Run the complete analysis for a single universe.

        Output from the analysis will be saved to a file in the run's output
        directory.

        Args:
            universe_dimensions: A dictionary containing the parameters
                for the universe.

        Returns:
            None
        """
        # Generate universe ID
        universe_id = generate_universe_id(universe_dimensions)
        logger.debug(f"Visiting universe: {universe_id}")

        # Clean up any old error fiels
        error_path = self._get_error_filepath(universe_id)
        if error_path.is_file():
            logger.warning(
                f"Removing old error file: {error_path}. This should only happen during a re-run."
            )
            error_path.unlink()

        universe_filetype = self.universe_file.suffix

        # Generate final command
        output_dir = self.get_run_dir(sub_directory="universes")
        output_filename = f"nb_{self.run_no}-{universe_id}{universe_filetype}"
        output_path = output_dir / output_filename

        # Ensure output dir exists
        output_dir.mkdir(parents=True, exist_ok=True)

        # Prepare settings dictionary
        settings = {
            "universe_id": universe_id,
            "dimensions": universe_dimensions,
            "run_no": self.run_no,
            "output_dir": str(self.output_dir),
            "seed": self.seed,
        }
        settings_str = json.dumps(settings, sort_keys=True)

        try:
            with warnings.catch_warnings(record=True) as w:
                warnings.simplefilter("always")
                if universe_filetype == ".ipynb":
                    self.execute_notebook_via_api(
                        input_path=str(self.universe_file),
                        output_path=str(output_path),
                        parameters={
                            "settings": settings_str,
                        },
                    )
                elif universe_filetype == ".py":
                    self.execute_python_script(
                        input_path=str(self.universe_file),
                        output_path=str(output_path),
                        parameters=settings,
                    )
                else:
                    raise ValueError("Universe file must be a .ipynb or .py file.")

                for warning in w:
                    logger.warning(
                        f"Warning in universe {universe_id}: {warning.message}"
                    )
        except Exception as e:
            error_filename = "E_" + output_filename
            logger.error(f"Error in universe {universe_id} ({error_filename})")
            # Rename notebook file to indicate error
            error_output_path = output_dir / error_filename
            output_path.rename(error_output_path)
            if self.stop_on_error:
                raise e
            else:
                logger.exception(e)
                self.save_error(universe_id, universe_dimensions, e)

    def _get_error_filepath(self, universe_id: str) -> Path:
        error_dir = self.get_run_dir(sub_directory=ERRORS_DIR_NAME)
        error_filename = "e_" + str(self.run_no) + "-" + universe_id + ".csv"

        return error_dir / error_filename

    def save_error(self, universe_id: str, dimensions: dict, error: Exception) -> None:
        """
        Save an error to a file.

        Args:
            universe_id: The ID of the universe that caused the error.
            error: The exception that was raised.

        Returns:
            None
        """
        error_type = type(error).__name__
        if error_type == "PapermillExecutionError":
            error_type = error.ename

        df_error = add_universe_info_to_df(
            pd.DataFrame(
                {
                    "mv_error_type": [error_type],
                    "mv_error": [str(error)],
                }
            ),
            universe_id=universe_id,
            run_no=self.run_no,
            dimensions=dimensions,
        )
        error_path = self._get_error_filepath(universe_id)
        df_error.to_csv(error_path, index=False)

    def execute_notebook_via_api(
        self, input_path: str, output_path: str, parameters: Dict[str, str]
    ):
        """
        Executes a notebook via the papermill python API.

        Args:
            input_path: The path to the input notebook.
            output_path: The path to the output notebook.
            parameters: A dictionary containing the parameters for the notebook.

        Returns:
            None
        """
        pm.execute_notebook(
            input_path,
            output_path,
            parameters=parameters,
            progress_bar=False,
            kernel_manager_class="multiversum.IPCKernelManager.IPCKernelManager",
            execution_timeout=self.cell_timeout,
        )

    def execute_python_script(
        self, input_path: str, output_path: Optional[str], parameters: Dict[str, Any]
    ):
        global_dict = {SCRIPT_GLOBAL_OVERWRITE_NAME: parameters}

        if output_path is not None:
            # Capture output
            script_output_capture = io.StringIO()
            with contextlib.redirect_stdout(script_output_capture):
                runpy.run_path(input_path, init_globals=global_dict)
        else:
            # Keep output as-is
            runpy.run_path(input_path, init_globals=global_dict)

        # Copy input file to output file if no output file is specified
        if output_path is not None:
            with open(input_path, "r") as input_file:
                with open(output_path, "w") as output_file:
                    # Prepend brief statement and parameters
                    output_file.write("# Generated by multiversum\n")
                    output_file.write(
                        "# Note: This file is only for illustrative purposes and the analysis itself may behave slightly differently.\n"
                    )
                    output_file.write(
                        f"{SCRIPT_GLOBAL_OVERWRITE_NAME} = {json.dumps(parameters, indent=4)}\n\n"
                    )
                    # Prepend output from running the script
                    script_output = script_output_capture.getvalue()
                    script_output_escaped = script_output.replace("\n", "\n# ")
                    output_file.write(f"# Output:\n# {script_output_escaped}\n\n")

                    # Copy over script
                    output_file.write(input_file.read())

    def generate_minimal_grid(self) -> List[Dict[str, Any]]:
        """
        Generate a minimal multiverse grid that contains each unique option at least once.

        This creates a smaller grid compared to the full factorial design, where each unique
        option in each dimension appears at least once. This can be useful for testing or
        quick validation of all options.

        Returns:
            A list of dicts containing the settings for different universes.
        """
        return generate_minimal_multiverse_grid(self.dimensions, self.constraints)

__init__(dimensions=None, config=None, universe=None, output_dir=Path('./output'), run_no=None, new_run=True, seed=None, stop_on_error=None, cell_timeout=None)

Initializes a new MultiverseAnalysis instance.

Parameters:

Name Type Description Default
dimensions Optional[Dict]

A dictionary where keys are dimension names and values are lists of possible values for each dimension. Each dimension corresponds to a decision.

None
config Union[Path, Config, None]

A Path to a TOML, JSON or Python file containing the analysis configuration. Supported configuration options can be found in the Config class. If a Python file is used, it should contain a dictionary / config object named "config". Will automatically search for multiverse.toml / .json / .py.

None
universe Path

The Path to the universe_file to run. Either an ipython / jupyter notebook (.ipynb) or a python script (.py).

None
output_dir Path

The directory to store the output in.

Path('./output')
run_no Optional[int]

The number of the current run. Defaults to an automatically incrementing integer number if new_run is True or the last run if new_run is False.

None
new_run bool

Whether this is a new run or not. Defaults to True.

True
seed Optional[int]

The seed to use for the analysis.

None
stop_on_error Optional[bool]

Whether to stop the analysis if an error occurs.

None
cell_timeout Optional[int]

A timeout (in seconds) for each cell in the notebook.

None
Source code in multiversum/multiverse.py
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
def __init__(
    self,
    dimensions: Optional[Dict] = None,
    config: Union[Path, Config, None] = None,
    universe: Path = None,
    output_dir: Path = Path("./output"),
    run_no: Optional[int] = None,
    new_run: bool = True,
    seed: Optional[int] = None,
    stop_on_error: Optional[bool] = None,
    cell_timeout: Optional[int] = None,
) -> None:
    """
    Initializes a new MultiverseAnalysis instance.

    Args:
        dimensions: A dictionary where keys are dimension names and values are lists of possible values for each dimension.
            Each dimension corresponds to a decision.
        config: A Path to a TOML, JSON or Python file containing the
            analysis configuration. Supported configuration options can be
            found in the Config class. If a Python file is used, it should
            contain a dictionary / config object named "config".
            Will automatically search for multiverse.toml / .json / .py.
        universe: The Path to the universe_file to run. Either an
            ipython / jupyter notebook (.ipynb) or a python script (.py).
        output_dir: The directory to store the output in.
        run_no: The number of the current run. Defaults to an automatically
            incrementing integer number if new_run is True or the last run if
            new_run is False.
        new_run: Whether this is a new run or not. Defaults to True.
        seed: The seed to use for the analysis.
        stop_on_error: Whether to stop the analysis if an error occurs.
        cell_timeout: A timeout (in seconds) for each cell in the notebook.
    """
    # Check for configuration file and parse it
    config_file = search_files(file=config, default_files=DEFAULT_CONFIG_FILES)
    if isinstance(config_file, Path):
        if config_file.suffix == ".toml":
            with open(config_file, "rb") as fp:
                config = tomllib.load(fp)
        elif config_file.suffix == ".json":
            with open(config_file, "r") as fp:
                config = json.load(fp)
        elif config_file.suffix == ".py":
            results = runpy.run_path(str(config_file))
            config = results["config"]
        else:
            raise ValueError(
                "Only .toml, .json and .py files are supported as config."
            )
    # Convert config to Config object
    if isinstance(config, dict):
        config = Config(**config)

    # Read settings from config (or args)
    self.read_config_value(config, "dimensions", dimensions)
    self.read_config_value(config, "seed", seed)
    self.read_config_value(config, "stop_on_error", stop_on_error)
    self.read_config_value(config, "cell_timeout", cell_timeout)
    self.read_config_value(config, "constraints")

    universe_file = search_files(
        file=universe, default_files=DEFAULT_UNIVERSE_FILES
    )
    self.universe_file = universe_file

    self.output_dir = output_dir
    if self.output_dir is not None:
        self.output_dir.mkdir(parents=True, exist_ok=True)

    self.run_no = (
        run_no if run_no is not None else self.read_counter(increment=new_run)
    )

    if self.dimensions is None:
        raise ValueError(
            "Dimensions need to be specified either directly or in a config."
        )

aggregate_data(include_errors=True, save=True)

Aggregate the data from all universes into a single DataFrame.

Parameters:

Name Type Description Default
include_errors bool

Whether to include error information.

True
save bool

Whether to save the aggregated data to a file.

True

Returns:

Type Description
DataFrame

A pandas DataFrame containing the aggregated data from all universes.

Source code in multiversum/multiverse.py
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
def aggregate_data(
    self, include_errors: bool = True, save: bool = True
) -> pd.DataFrame:
    """
    Aggregate the data from all universes into a single DataFrame.

    Args:
        include_errors: Whether to include error information.
        save: Whether to save the aggregated data to a file.

    Returns:
        A pandas DataFrame containing the aggregated data from all universes.
    """
    data_dir = self.get_run_dir(sub_directory="data")
    csv_files = list(data_dir.glob("*.csv"))

    if include_errors:
        error_dir = self.get_run_dir(sub_directory=ERRORS_DIR_NAME)
        csv_files += list(error_dir.glob("*.csv"))

    if len(csv_files) == 0:
        logger.warning("No data files to aggregate, returning empty dataframe.")
        df = pd.DataFrame({"mv_universe_id": []})
    else:
        df = pd.concat((pd.read_csv(f) for f in csv_files), ignore_index=True)

    if save:
        df.to_csv(data_dir / ("agg_" + str(self.run_no) + "_run_outputs.csv.gz"))

    return df

check_missing_universes()

Check if any universes from the multiverse have not yet been visited.

Returns:

Type Description
MissingUniverseInfo

A dictionary containing the missing universe ids, additional universe ids (i.e. not in the current multiverse_grid) and the dictionaries for the missing universes.

Source code in multiversum/multiverse.py
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
def check_missing_universes(self) -> MissingUniverseInfo:
    """
    Check if any universes from the multiverse have not yet been visited.

    Returns:
        A dictionary containing the missing universe ids, additional
            universe ids (i.e. not in the current multiverse_grid)
            and the dictionaries for the missing universes.
    """
    multiverse_dict = add_ids_to_multiverse_grid(self.generate_grid(save=False))
    all_universe_ids = set(multiverse_dict.keys())

    aggregated_data = self.aggregate_data(include_errors=False, save=False)
    universe_ids_with_data = set(aggregated_data["mv_universe_id"])

    missing_universe_ids = all_universe_ids - universe_ids_with_data
    extra_universe_ids = universe_ids_with_data - all_universe_ids
    missing_universes = [multiverse_dict[u_id] for u_id in missing_universe_ids]

    if len(missing_universe_ids) > 0 or len(extra_universe_ids) > 0:
        logger.warning(
            f"Found missing {len(missing_universe_ids)} / "
            f"additional {len(extra_universe_ids)} universe ids!"
        )

    return {
        "missing_universe_ids": missing_universe_ids,
        "extra_universe_ids": extra_universe_ids,
        "missing_universes": missing_universes,
    }

examine_multiverse(multiverse_grid=None, n_jobs=-2)

Run the analysis for all universes in the multiverse.

Parameters:

Name Type Description Default
multiverse_grid List[Dict[str, Any]]

A list of dictionaries containing the settings for different universes.

None
n_jobs int

The number of jobs to run in parallel. Defaults to -2 (all CPUs but one).

-2

Returns:

Type Description
None

None

Source code in multiversum/multiverse.py
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
def examine_multiverse(
    self, multiverse_grid: List[Dict[str, Any]] = None, n_jobs: int = -2
) -> None:
    """
    Run the analysis for all universes in the multiverse.

    Args:
        multiverse_grid: A list of dictionaries containing the settings for different universes.
        n_jobs: The number of jobs to run in parallel. Defaults to -2 (all CPUs but one).

    Returns:
        None
    """
    if multiverse_grid is None:
        multiverse_grid = self.grid or self.generate_grid(save=False)

    # Run analysis for all universes
    with Progress(
        TextColumn("[progress.description]{task.description}"),
        SpinnerColumn(),
        TaskProgressColumn(),
        BarColumn(bar_width=None),
        MofNCompleteColumn(),
        TextColumn("•"),
        TimeElapsedColumn(),
        TextColumn("•"),
        TimeRemainingColumn(),
        expand=True,
    ) as progress:
        task_id = progress.add_task("Running", total=len(multiverse_grid))
        if n_jobs == 1:
            logger.info("Running in single-threaded mode (njobs = 1).")
            for universe_params in multiverse_grid:
                self.visit_universe(universe_params)
                progress.update(task_id, advance=1)
                # Somehow automatic updating is not working in single threaded mode, so we manually refresh
                progress.refresh()
        else:
            logger.info(
                f"Running in parallel mode (njobs = {n_jobs}; {cpu_count()} CPUs detected)."
            )
            with rich_joblib(progress, task_id):
                # For n_jobs below -1, (n_cpus + 1 + n_jobs) are used.
                # Thus for n_jobs = -2, all CPUs but one are used
                Parallel(n_jobs=n_jobs)(
                    delayed(self.visit_universe)(universe_params)
                    for universe_params in multiverse_grid
                )

execute_notebook_via_api(input_path, output_path, parameters)

Executes a notebook via the papermill python API.

Parameters:

Name Type Description Default
input_path str

The path to the input notebook.

required
output_path str

The path to the output notebook.

required
parameters Dict[str, str]

A dictionary containing the parameters for the notebook.

required

Returns:

Type Description

None

Source code in multiversum/multiverse.py
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
def execute_notebook_via_api(
    self, input_path: str, output_path: str, parameters: Dict[str, str]
):
    """
    Executes a notebook via the papermill python API.

    Args:
        input_path: The path to the input notebook.
        output_path: The path to the output notebook.
        parameters: A dictionary containing the parameters for the notebook.

    Returns:
        None
    """
    pm.execute_notebook(
        input_path,
        output_path,
        parameters=parameters,
        progress_bar=False,
        kernel_manager_class="multiversum.IPCKernelManager.IPCKernelManager",
        execution_timeout=self.cell_timeout,
    )

generate_grid(save=True)

Generate the multiverse grid from the stored dimensions.

Parameters:

Name Type Description Default
save bool

Whether to save the multiverse grid to a JSON file.

True

Returns:

Type Description
List[Dict[str, Any]]

A list of dicts containing the settings for different universes.

Source code in multiversum/multiverse.py
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
def generate_grid(self, save: bool = True) -> List[Dict[str, Any]]:
    """
    Generate the multiverse grid from the stored dimensions.

    Args:
        save: Whether to save the multiverse grid to a JSON file.

    Returns:
        A list of dicts containing the settings for different universes.
    """
    self.grid = generate_multiverse_grid(self.dimensions, self.constraints)
    if save:
        with open(self.output_dir / "multiverse_grid.json", "w") as fp:
            json.dump(self.grid, fp, indent=2)
    return self.grid

generate_minimal_grid()

Generate a minimal multiverse grid that contains each unique option at least once.

This creates a smaller grid compared to the full factorial design, where each unique option in each dimension appears at least once. This can be useful for testing or quick validation of all options.

Returns:

Type Description
List[Dict[str, Any]]

A list of dicts containing the settings for different universes.

Source code in multiversum/multiverse.py
580
581
582
583
584
585
586
587
588
589
590
591
def generate_minimal_grid(self) -> List[Dict[str, Any]]:
    """
    Generate a minimal multiverse grid that contains each unique option at least once.

    This creates a smaller grid compared to the full factorial design, where each unique
    option in each dimension appears at least once. This can be useful for testing or
    quick validation of all options.

    Returns:
        A list of dicts containing the settings for different universes.
    """
    return generate_minimal_multiverse_grid(self.dimensions, self.constraints)

get_run_dir(sub_directory=None)

Get the directory for the current run.

Parameters:

Name Type Description Default
sub_directory Optional[str]

An optional sub-directory to append to the run directory.

None

Returns:

Type Description
Path

A Path object for the current run directory.

Source code in multiversum/multiverse.py
239
240
241
242
243
244
245
246
247
248
249
250
251
252
def get_run_dir(self, sub_directory: Optional[str] = None) -> Path:
    """
    Get the directory for the current run.

    Args:
        sub_directory: An optional sub-directory to append to the run directory.

    Returns:
        A Path object for the current run directory.
    """
    run_dir = self.output_dir / "runs" / str(self.run_no)
    target_dir = run_dir / sub_directory if sub_directory is not None else run_dir
    target_dir.mkdir(parents=True, exist_ok=True)
    return target_dir

read_counter(increment)

Read the counter from the output directory.

Parameters:

Name Type Description Default
increment bool

Whether to increment the counter after reading.

required

Returns:

Type Description
int

The current value of the counter.

Source code in multiversum/multiverse.py
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
def read_counter(self, increment: bool) -> int:
    """
    Read the counter from the output directory.

    Args:
        increment: Whether to increment the counter after reading.

    Returns:
        The current value of the counter.
    """

    # Use a self-incrementing counter via counter.txt
    counter_filepath = self.output_dir / "counter.txt"
    if counter_filepath.is_file():
        with open(counter_filepath, "r") as fp:
            run_no = int(fp.read())
    else:
        run_no = 0
    if increment:
        run_no += 1
    with open(counter_filepath, "w") as fp:
        fp.write(str(run_no))

    return run_no

save_error(universe_id, dimensions, error)

Save an error to a file.

Parameters:

Name Type Description Default
universe_id str

The ID of the universe that caused the error.

required
error Exception

The exception that was raised.

required

Returns:

Type Description
None

None

Source code in multiversum/multiverse.py
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
def save_error(self, universe_id: str, dimensions: dict, error: Exception) -> None:
    """
    Save an error to a file.

    Args:
        universe_id: The ID of the universe that caused the error.
        error: The exception that was raised.

    Returns:
        None
    """
    error_type = type(error).__name__
    if error_type == "PapermillExecutionError":
        error_type = error.ename

    df_error = add_universe_info_to_df(
        pd.DataFrame(
            {
                "mv_error_type": [error_type],
                "mv_error": [str(error)],
            }
        ),
        universe_id=universe_id,
        run_no=self.run_no,
        dimensions=dimensions,
    )
    error_path = self._get_error_filepath(universe_id)
    df_error.to_csv(error_path, index=False)

visit_universe(universe_dimensions)

Run the complete analysis for a single universe.

Output from the analysis will be saved to a file in the run's output directory.

Parameters:

Name Type Description Default
universe_dimensions Dict[str, str]

A dictionary containing the parameters for the universe.

required

Returns:

Type Description
None

None

Source code in multiversum/multiverse.py
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
def visit_universe(self, universe_dimensions: Dict[str, str]) -> None:
    """
    Run the complete analysis for a single universe.

    Output from the analysis will be saved to a file in the run's output
    directory.

    Args:
        universe_dimensions: A dictionary containing the parameters
            for the universe.

    Returns:
        None
    """
    # Generate universe ID
    universe_id = generate_universe_id(universe_dimensions)
    logger.debug(f"Visiting universe: {universe_id}")

    # Clean up any old error fiels
    error_path = self._get_error_filepath(universe_id)
    if error_path.is_file():
        logger.warning(
            f"Removing old error file: {error_path}. This should only happen during a re-run."
        )
        error_path.unlink()

    universe_filetype = self.universe_file.suffix

    # Generate final command
    output_dir = self.get_run_dir(sub_directory="universes")
    output_filename = f"nb_{self.run_no}-{universe_id}{universe_filetype}"
    output_path = output_dir / output_filename

    # Ensure output dir exists
    output_dir.mkdir(parents=True, exist_ok=True)

    # Prepare settings dictionary
    settings = {
        "universe_id": universe_id,
        "dimensions": universe_dimensions,
        "run_no": self.run_no,
        "output_dir": str(self.output_dir),
        "seed": self.seed,
    }
    settings_str = json.dumps(settings, sort_keys=True)

    try:
        with warnings.catch_warnings(record=True) as w:
            warnings.simplefilter("always")
            if universe_filetype == ".ipynb":
                self.execute_notebook_via_api(
                    input_path=str(self.universe_file),
                    output_path=str(output_path),
                    parameters={
                        "settings": settings_str,
                    },
                )
            elif universe_filetype == ".py":
                self.execute_python_script(
                    input_path=str(self.universe_file),
                    output_path=str(output_path),
                    parameters=settings,
                )
            else:
                raise ValueError("Universe file must be a .ipynb or .py file.")

            for warning in w:
                logger.warning(
                    f"Warning in universe {universe_id}: {warning.message}"
                )
    except Exception as e:
        error_filename = "E_" + output_filename
        logger.error(f"Error in universe {universe_id} ({error_filename})")
        # Rename notebook file to indicate error
        error_output_path = output_dir / error_filename
        output_path.rename(error_output_path)
        if self.stop_on_error:
            raise e
        else:
            logger.exception(e)
            self.save_error(universe_id, universe_dimensions, e)