Warning: This document is for an old version of Pydra: A simple dataflow engine with scalable semantics. The main version is master.

Library API (application programmer interface)

The Pydra workflow engine.

Pydra is a rewrite of the Nipype engine with mapping and joining as first-class operations. It forms the core of the Nipype 2.0 ecosystem.

pydra.check_latest_version()

pydra.set_input_validator(flag=False)

Subpackages

• pydra.engine package
  • AuditFlag
    • AuditFlag.ALL
    • AuditFlag.NONE
    • AuditFlag.PROV
    • AuditFlag.RESOURCE
  • DockerTask
    • DockerTask.container_args
    • DockerTask.init
  • ShellCommandTask
    • ShellCommandTask.cmdline
    • ShellCommandTask.command_args
    • ShellCommandTask.input_spec
    • ShellCommandTask.output_spec
  • Submitter
    • Submitter.close()
    • Submitter.expand_runnable()
    • Submitter.expand_workflow()
    • Submitter.submit_from_call()
  • Workflow
    • Workflow.add()
    • Workflow.checksum
    • Workflow.create_connections()
    • Workflow.create_dotfile()
    • Workflow.graph_sorted
    • Workflow.nodes
    • Workflow.set_output()
  • Submodules
    • pydra.engine.audit module
      • Audit
        • Audit.audit_check()
        • Audit.audit_message()
        • Audit.audit_task()
        • Audit.finalize_audit()
        • Audit.monitor()
        • Audit.start_audit()
    • pydra.engine.boutiques module
      • BoshTask
    • pydra.engine.core module
      • TaskBase
        • TaskBase.audit_flags
        • TaskBase.cache_dir
        • TaskBase.cache_locations
        • TaskBase.can_resume
        • TaskBase.checksum
        • TaskBase.checksum_states()
        • TaskBase.combine()
        • TaskBase.cont_dim
        • TaskBase.done
        • TaskBase.errored
        • TaskBase.generated_output_names
        • TaskBase.get_input_el()
        • TaskBase.help()
        • TaskBase.output_dir
        • TaskBase.output_names
        • TaskBase.pickle_task()
        • TaskBase.result()
        • TaskBase.set_state()
        • TaskBase.split()
        • TaskBase.uid
        • TaskBase.version
      • Workflow
        • Workflow.add()
        • Workflow.checksum
        • Workflow.create_connections()
        • Workflow.create_dotfile()
        • Workflow.graph_sorted
        • Workflow.nodes
        • Workflow.set_output()
      • is_lazy()
      • is_task()
      • is_workflow()
    • pydra.engine.graph module
      • DiGraph
        • DiGraph.add_edges()
        • DiGraph.add_edges_description()
        • DiGraph.add_nodes()
        • DiGraph.calculate_max_paths()
        • DiGraph.copy()
        • DiGraph.create_dotfile_detailed()
        • DiGraph.create_dotfile_nested()
        • DiGraph.create_dotfile_simple()
        • DiGraph.edges
        • DiGraph.edges_names
        • DiGraph.export_graph()
        • DiGraph.nodes
        • DiGraph.nodes_details
        • DiGraph.nodes_names_map
        • DiGraph.remove_nodes()
        • DiGraph.remove_nodes_connections()
        • DiGraph.remove_previous_connections()
        • DiGraph.remove_successors_nodes()
        • DiGraph.sorted_nodes
        • DiGraph.sorted_nodes_names
        • DiGraph.sorting()
    • pydra.engine.helpers module
      • PydraFileLock
      • argstr_formatting()
      • copyfile_workflow()
      • create_checksum()
      • custom_validator()
      • ensure_list()
      • execute()
      • gather_runtime_info()
      • get_available_cpus()
      • get_open_loop()
      • hash_function()
      • hash_value()
      • load_and_run()
      • load_and_run_async()
      • load_result()
      • load_task()
      • make_klass()
      • output_from_inputfields()
      • position_sort()
      • print_help()
      • read_and_display()
      • read_and_display_async()
      • read_stream_and_display()
      • record_error()
      • save()
      • task_hash()
    • pydra.engine.helpers_file module
      • copyfile()
      • copyfile_input()
      • copyfiles()
      • ensure_list()
      • fname_presuffix()
      • get_related_files()
      • hash_dir()
      • hash_file()
      • is_container()
      • is_existing_file()
      • is_local_file()
      • on_cifs()
      • related_filetype_sets
      • split_filename()
      • template_update()
      • template_update_single()
    • pydra.engine.helpers_state module
      • PydraStateError
      • add_name_combiner()
      • add_name_splitter()
      • combine_final_groups()
      • converter_groups_to_input()
      • flatten()
      • input_shape()
      • inputs_types_to_dict()
      • iter_splits()
      • map_splits()
      • remove_inp_from_splitter_rpn()
      • rpn2splitter()
      • splits_groups()
      • splitter2rpn()
    • pydra.engine.specs module
      • BaseSpec
        • BaseSpec.check_fields_input_spec()
        • BaseSpec.check_metadata()
        • BaseSpec.collect_additional_outputs()
        • BaseSpec.copyfile_input()
        • BaseSpec.hash
        • BaseSpec.retrieve_values()
        • BaseSpec.template_update()
      • ContainerSpec
        • ContainerSpec.container
        • ContainerSpec.container_xargs
        • ContainerSpec.image
      • Directory
      • DockerSpec
        • DockerSpec.container
      • File
      • FunctionSpec
        • FunctionSpec.check_metadata()
      • LazyField
        • LazyField.get_value()
      • MultiInputFile
      • MultiInputObj
        • MultiInputObj.converter()
      • MultiOutputFile
      • MultiOutputObj
        • MultiOutputObj.converter()
      • Result
        • Result.errored
        • Result.get_output_field()
        • Result.output
        • Result.runtime
      • Runtime
        • Runtime.cpu_peak_percent
        • Runtime.rss_peak_gb
        • Runtime.vms_peak_gb
      • RuntimeSpec
        • RuntimeSpec.container
        • RuntimeSpec.network
        • RuntimeSpec.outdir
      • ShellOutSpec
        • ShellOutSpec.collect_additional_outputs()
        • ShellOutSpec.generated_output_names()
        • ShellOutSpec.return_code
        • ShellOutSpec.stderr
        • ShellOutSpec.stdout
      • ShellSpec
        • ShellSpec.args
        • ShellSpec.check_metadata()
        • ShellSpec.executable
        • ShellSpec.retrieve_values()
      • SingularitySpec
        • SingularitySpec.container
      • SpecInfo
        • SpecInfo.bases
        • SpecInfo.fields
        • SpecInfo.name
      • TaskHook
        • TaskHook.post_run
        • TaskHook.post_run_task
        • TaskHook.pre_run
        • TaskHook.pre_run_task
        • TaskHook.reset()
      • attr_fields()
      • attr_fields_dict()
      • donothing()
      • path_to_string()
    • pydra.engine.state module
      • State
        • State.name
        • State.splitter
        • State.splitter_rpn_compact
        • State.splitter_rpn
        • State.combiner
        • State.splitter_final
        • State.other_states
        • State.inner_inputs
        • State.states_ind
        • State.states_val
        • State.inputs_ind
        • State.group_for_inputs
        • State.group_for_inputs_final
        • State.groups_stack_final
        • State.final_combined_ind_mapping
        • State.combiner
        • State.combiner_validation()
        • State.current_combiner
        • State.current_combiner_all
        • State.current_splitter
        • State.current_splitter_rpn
        • State.inner_inputs
        • State.other_states
        • State.prepare_inputs()
        • State.prepare_states()
        • State.prepare_states_combined_ind()
        • State.prepare_states_ind()
        • State.prepare_states_val()
        • State.prev_state_combiner
        • State.prev_state_combiner_all
        • State.prev_state_splitter
        • State.prev_state_splitter_rpn
        • State.prev_state_splitter_rpn_compact
        • State.set_input_groups()
        • State.splits()
        • State.splitter
        • State.splitter_final
        • State.splitter_rpn
        • State.splitter_rpn_compact
        • State.splitter_rpn_final
        • State.splitter_validation()
        • State.update_connections()
    • pydra.engine.submitter module
      • Submitter
        • Submitter.close()
        • Submitter.expand_runnable()
        • Submitter.expand_workflow()
        • Submitter.submit_from_call()
      • get_runnable_tasks()
      • is_runnable()
      • prepare_runnable_with_state()
    • pydra.engine.task module
      • ContainerTask
        • ContainerTask.bind_paths()
        • ContainerTask.binds()
        • ContainerTask.container_check()
      • DockerTask
        • DockerTask.container_args
        • DockerTask.init
      • FunctionTask
      • ShellCommandTask
        • ShellCommandTask.cmdline
        • ShellCommandTask.command_args
        • ShellCommandTask.input_spec
        • ShellCommandTask.output_spec
      • SingularityTask
        • SingularityTask.container_args
        • SingularityTask.init
      • split_cmd()
    • pydra.engine.workers module
      • ConcurrentFuturesWorker
        • ConcurrentFuturesWorker.close()
        • ConcurrentFuturesWorker.exec_as_coro()
        • ConcurrentFuturesWorker.run_el()
      • DaskWorker
        • DaskWorker.close()
        • DaskWorker.exec_dask()
        • DaskWorker.run_el()
      • DistributedWorker
        • DistributedWorker.fetch_finished()
        • DistributedWorker.max_jobs
      • SGEWorker
        • SGEWorker.check_for_results_files()
        • SGEWorker.get_output_by_task_pkl()
        • SGEWorker.get_tasks_to_run()
        • SGEWorker.run_el()
        • SGEWorker.submit_array_job()
      • SerialWorker
        • SerialWorker.close()
        • SerialWorker.exec_serial()
        • SerialWorker.fetch_finished()
        • SerialWorker.run_el()
      • SlurmWorker
        • SlurmWorker.run_el()
      • Worker
        • Worker.close()
        • Worker.fetch_finished()
        • Worker.run_el()
• pydra.mark package
  • Submodules
    • pydra.mark.functions module
      • annotate()
      • task()
• pydra.tasks package
• pydra.utils package
  • Submodules
    • pydra.utils.messenger module
      • AuditFlag
        • AuditFlag.ALL
        • AuditFlag.NONE
        • AuditFlag.PROV
        • AuditFlag.RESOURCE
      • FileMessenger
        • FileMessenger.send()
      • Messenger
        • Messenger.send()
      • PrintMessenger
        • PrintMessenger.send()
      • RemoteRESTMessenger
        • RemoteRESTMessenger.send()
      • RuntimeHooks
        • RuntimeHooks.resource_monitor_post_stop
        • RuntimeHooks.resource_monitor_pre_start
        • RuntimeHooks.task_execute_post_exit
        • RuntimeHooks.task_execute_pre_entry
        • RuntimeHooks.task_run_entry
        • RuntimeHooks.task_run_exit
      • collect_messages()
      • gen_uuid()
      • make_message()
      • now()
      • send_message()
    • pydra.utils.profiler module
      • ResourceMonitor
        • ResourceMonitor.fname
        • ResourceMonitor.run()
        • ResourceMonitor.stop()
      • get_max_resources_used()
      • get_system_total_memory_gb()
      • log_nodes_cb()