Skip to content

Commit

Permalink
various pr changes
Browse files Browse the repository at this point in the history
  • Loading branch information
stevebachmeier committed Jan 8, 2025
1 parent 1b03395 commit 5707193
Show file tree
Hide file tree
Showing 7 changed files with 51 additions and 39 deletions.
8 changes: 4 additions & 4 deletions src/easylink/cli.py
Original file line number Diff line number Diff line change
Expand Up @@ -89,9 +89,9 @@ def run(
) -> None:
"""Runs a pipeline from the command line.
In addition to running the pipeline, this command will also generate the
DAG image. If you only want to generate the image without actually running
the pipeline, use the ``easylink generate-dag`` command.
In addition to running the pipeline, this command will also generate the directed
acyclic graph (DAG) image. If you only want to generate the image without actually
running the pipeline, use the ``easylink generate-dag`` command.
"""
configure_logging_to_terminal(verbose)
logger.info("Running pipeline")
Expand Down Expand Up @@ -120,7 +120,7 @@ def generate_dag(
output_dir: str | None,
no_timestamp: bool,
) -> None:
"""Generates an image of the proposed pipeline DAG.
"""Generates an image of the proposed pipeline directed acyclic graph (DAG).
This command only generates the DAG image of the pipeline; it does not actually
run it. To run the pipeline, use the ``easylink run`` command.
Expand Down
3 changes: 2 additions & 1 deletion src/easylink/configuration.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,8 @@
Configuration
=============
This module is responsible for managing an easylink run's configuration.
This module is responsible for managing an easylink run's configuration as defined
by various user-input specification files.
"""

Expand Down
42 changes: 26 additions & 16 deletions src/easylink/graph_components.py
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
Graph Components
================
This module is responsible for defining the abstractions that represent the graph
components of a pipeline.
This module is responsible for defining the modular building-block objects that
can be composed to create graph representations of pipelines.
"""

Expand All @@ -24,11 +24,16 @@

@dataclass(frozen=True)
class InputSlot:
"""An abstraction representing a single input slot to a specific :class:`~easylink.step.Step`.
"""An abstraction representing a single input slot to a specific node.
In order to pass data between :class:`Steps<easylink.step.Step>`, an InputSlot
of one Step can be connected to an :class:`OutputSlot` of another Step via an :class:`EdgeParams`
instance.
``InputSlots`` represent distinct semantic categories of input files, between
which a node must be able to differentiate. In order to pass data between nodes,
an ``InputSlot`` of one node can be connected to an :class:`OutputSlot` of another
node via an :class:`EdgeParams` instance.
Notes
-----
Nodes can be either :class:`Steps<easylink.step.Step>` or :class:`Implementations<~easylink.implementation.Implementation>`.
"""

name: str
Expand All @@ -44,18 +49,19 @@ class InputSlot:

@dataclass(frozen=True)
class OutputSlot:
"""An abstraction representing a single output slot from a specific :class:`~easylink.step.Step`.
"""An abstraction representing a single output slot from a specific node.
In order to pass data between :class:`Steps<easylink.step.Step>`, an OutputSlot
of one Step can be connected to an :class:`InputSlot` of another Step via an :class:`EdgeParams`
instance.
In order to pass data between nodes, an OutputSlot of one node can be connected
to an :class:`InputSlot` of another node via an :class:`EdgeParams` instance.
Notes
-----
Nodes can be either :class:`Steps<easylink.step.Step>` or :class:`Implementations<~easylink.implementation.Implementation>`.
Input data is validated via the :class:`InputSlot's<InputSlot>` required
:attr:`~InputSlot.validator` attribute. In order to prevent multiple
validations of the same files (since outputs of one :class:`~easylink.step.Step`
can be inputs to another), no such validator is stored here on the OutputSlot.
validations of the same files (since outputs of one node can be inputs to another),
no such validator is stored here on the OutputSlot.
"""

name: str
Expand All @@ -64,10 +70,14 @@ class OutputSlot:

@dataclass(frozen=True)
class EdgeParams:
"""A representation of an edge between two nodes (:class:`Steps<easylink.step.Step>`) in a graph.
"""A representation of an edge between two nodes in a graph.
EdgeParams connect the :class:`OutputSlot` of a source Step to the :class:`InputSlot`
of a target Step.
EdgeParams connect the :class:`OutputSlot` of a source node to the :class:`InputSlot`
of a target node.
Notes
-----
Nodes can be either :class:`Steps<easylink.step.Step>` or :class:`Implementations<~easylink.implementation.Implementation>`.
"""

source_node: str
Expand Down Expand Up @@ -110,7 +120,7 @@ def from_graph_edge(


class StepGraph(nx.MultiDiGraph):
"""A DAG of :class:`Steps<easylink.step.Step>` and the data dependencies between them.
"""A directed acyclic graph (DAG) of :class:`Steps<easylink.step.Step>` and the data dependencies between them.
StepGraphs are DAGs with :class:`Steps<easylink.step.Step>`
for nodes and the file dependencies between them for edges. Multiple edges
Expand Down
28 changes: 14 additions & 14 deletions src/easylink/implementation.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@
===============
This module is responsible for defining the abstractions that represent actual
implementations of steps in a pipeline.
implementations of steps in a pipeline. Typically, these abstractions contain
information about what container to run for the step and other related details.
"""

Expand Down Expand Up @@ -136,7 +137,7 @@ def outputs(self) -> dict[str, list[str]]:


class NullImplementation:
"""An object with a partial :class:`Implementation` interface that represents that no container needs to run.
"""An partial :class:`Implementation` interface that represents that no container needs to run.
The primary use case for this class is when adding an :class:`~easylink.step.IOStep` -
which does not have a corresponding :class:`Implementation` - to an
Expand Down Expand Up @@ -170,11 +171,10 @@ def __init__(
names to their instances."""
self.schema_steps = [self.name]
"""The requested :class:`~easylink.pipeline_schema.PipelineSchema`
:class:`~easylink.step.Step` names for which this NullImplementation is
expected to be responsible."""
:class:`~easylink.step.Step` names this ``NullImplementation`` implements."""
self.combined_name = None
"""The name of the combined implementation that this NullImplementation
is part of. This is definitionally None for a NullImplementation."""
"""The name of the combined implementation of which ``NullImplementation``
is a constituent. This is definitionally None for a ``NullImplementation``."""


class PartialImplementation:
Expand All @@ -194,12 +194,12 @@ class PartialImplementation:
Parameters
----------
combined_name
The name of the combined implementation that this PartialImplementation
is part of.
The name of the combined implementation of which this ``PartialImplementation``
is a part.
schema_step
The requested :class:`~easylink.pipeline_schema.PipelineSchema`
:class:`~easylink.step.Step` name for which this PartialImplementation is
expected to be responsible.
:class:`~easylink.step.Step` name that this ``PartialImplementation``
partially implements.
input_slots
The :class:`InputSlots<easylink.graph_components.InputSlot>` for this PartialImplementation.
output_slots
Expand All @@ -215,12 +215,12 @@ def __init__(
output_slots: Iterable["OutputSlot"] = (),
):
self.combined_name = combined_name
"""The name of the combined implementation that this PartialImplementation
is part of."""
"""The name of the combined implementation of which this ``PartialImplementation``
is a part."""
self.schema_step = schema_step
"""The requested :class:`~easylink.pipeline_schema.PipelineSchema`
:class:`~easylink.step.Step` name for which this PartialImplementation is
expected to be responsible."""
:class:`~easylink.step.Step` name that this ``PartialImplementation``
partially implements."""
self.input_slots = {slot.name: slot for slot in input_slots}
"""A mapping of :class:`InputSlots<easylink.graph_components.InputSlot>`
names to their instances."""
Expand Down
6 changes: 3 additions & 3 deletions src/easylink/pipeline_graph.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,9 @@
class PipelineGraph(ImplementationGraph):
"""The structure of the pipeline.
The PipelineGraph is a DAG composed of Implementations and their file dependencies.
It is created by "flattening" the PipelineSchema (a nested StepGraph) with parameters
set in the configuration.
The PipelineGraph is a directed acyclic graph (DAG) composed of Implementations
and their file dependencies. It is created by "flattening" the PipelineSchema
(a nested StepGraph) with parameters set in the configuration.
"""

def __init__(self, config: Config) -> None:
Expand Down
2 changes: 1 addition & 1 deletion src/easylink/rule.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ def _build_rule(self) -> str:
class TargetRule(Rule):
"""A rule that defines the final output of the pipeline.
Snakemake will determine the DAG based on this target.
Snakemake will determine the directed acyclic graph (DAG) based on this target.
"""

target_files: list[str]
Expand Down
1 change: 1 addition & 0 deletions src/easylink/step.py
Original file line number Diff line number Diff line change
Expand Up @@ -671,6 +671,7 @@ def _update_step_graph(self, num_repeats) -> StepGraph:
graph.add_node_from_step(node)
for edge in edges:
graph.add_edge_from_params(edge)
breakpoint()
return graph

def _update_slot_mappings(self, num_repeats) -> dict:
Expand Down

0 comments on commit 5707193

Please sign in to comment.