diff --git a/.gitignore b/.gitignore index 5bb4e110325..b091fbc48e0 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,10 @@ tests/test_orca/images/linux/failed/ doc/python/raw.githubusercontent.com/ +docs/ +docs_tmp/ +pages/examples/ + # Don't ignore dataset files !*.csv.gz !*.geojson.gz diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ad666ea805c..f4cfaf94447 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,7 +8,7 @@ then explains the technical aspects of preparing your contribution. ## Code of Conduct -Please note that all contributos are required to abide by our [Code of Conduct](CODE_OF_CONDUCT.md). +Please note that all contributos are required to abide by our Code of Conduct. ## Different Ways to Contribute @@ -19,7 +19,7 @@ it is important to understand the structure of the code and the repository. - The [`plotly.graph_objects`](https://plotly.com/python/graph-objects/) module (usually imported as `go`) is [generated from the plotly.js schema](https://plotly.com/python/figure-structure/), so changes to be made in this package need to be contributed to [plotly.js](https://github.com/plotly/plotly.js) - or to the code generation system in `./codegen/`. + or to the code generation system in `./bin/codegen/`. Code generation creates traces and layout classes that have a direct correspondence to their JavaScript counterparts, while higher-level methods that work on figures regardless of the current schema (e.g., `BaseFigure.for_each_trace`) are defined in `plotly/basedatatypes.py`. @@ -38,16 +38,17 @@ it is important to understand the structure of the code and the repository. These are organized in subdirectories according to what they test: see the "Setup" section below for more details. -- Documentation is found in `doc/`, and its structure is described in [its README file](doc/README.md). +- Documentation is found in `doc/`, and its structure is described in its README file. The documentation is a great place to start contributing, since you can add or modify examples without setting up a full environment. -Code and documentation are not the only way to contribute: -you can also help by: +Code and documentation are not the only way to contribute. +You can also help by: - Reporting bugs at . Please take a moment to see if your problem has already been reported, and if so, add a comment to the existing issue; - we will try to prioritize those that affect the most people. + we will try to prioritize those that affect the most people + and that are accompanied by small, runnable examples. - Submitting feature requests (also at ). Again, please add a comment to an existing issue if the feature you want has already been requested. @@ -219,11 +220,11 @@ Once you have done that, run the `updateplotlyjs` command: ```bash -python commands.py updateplotlyjs +python bin/updatejs.py ``` This downloads new versions of `plot-schema.json` and `plotly.min.js` from the `plotly/plotly.js` GitHub repository -and places them in `plotly/package_data`. +and places them in `resources` and `plotly/package_data` respectively. It then regenerates all of the `graph_objs` classes based on the new schema. ### Using a Development Branch of Plotly.js @@ -232,7 +233,8 @@ If your development branch is in [the plotly.js repository](https://github.com/p you can update to development versions of `plotly.js` with this command: ```bash -python commands.py updateplotlyjsdev --devrepo reponame --devbranch branchname +# FIXME commands.py didn't provide --devrepo or --devbranch +python bin/updatejs.py --dev --devrepo reponame --devbranch branchname ``` This fetches the `plotly.js` in the CircleCI artifact of the branch `branchname` of the repo `reponame`. @@ -255,5 +257,6 @@ You can then run the following command *in your local plotly.py repository*: ```bash -python commands.py updateplotlyjsdev --local /path/to/your/plotly.js/ +# FIXME: commands.py didn't provide --local +python bin/updatejs.py --dev --local /path/to/your/plotly.js/ ``` diff --git a/Makefile b/Makefile new file mode 100644 index 00000000000..56f5a2895ba --- /dev/null +++ b/Makefile @@ -0,0 +1,78 @@ +# Manage plotly.py project. + +RUN = uv run +PACKAGE_DIRS = _plotly_utils plotly +CODE_DIRS = ${PACKAGE_DIRS} scripts + +ifdef MKDOCS_ALL +EXAMPLE_SRC = $(wildcard doc/python/*.md) +else +EXAMPLE_SRC = doc/python/cone-plot.md doc/python/strip-charts.md +endif + +EXAMPLE_DST = $(patsubst doc/python/%.md,pages/examples/%.md,${EXAMPLE_SRC}) + +## commands: show available commands +commands: + @grep -h -E '^##' ${MAKEFILE_LIST} | sed -e 's/## //g' | column -t -s ':' + +## docs: rebuild documentation +.PHONY: docs +docs: + ${RUN} mkdocs build + +## docs-lint: check documentation +docs-lint: + ${RUN} pydoclint ${PACKAGE_DIRS} + +## docs-tmp: rebuild documentation saving Markdown in ./tmp +docs-tmp: + MKDOCS_TEMP_DIR=./docs_tmp ${RUN} mkdocs build + +## examples-batch: generate Markdown for all doc/python +examples-batch: + ${RUN} bin/run_markdown.py --outdir pages/examples --inline --verbose 1 ${EXAMPLE_SRC} + +## examples: generate Markdown for individual doc/python +examples: ${EXAMPLE_DST} + +pages/examples/%.md: doc/python/%.md + ${RUN} bin/run_markdown.py --outdir pages/examples --inline --verbose 2 $< + +## format: reformat code +format: + ${RUN} ruff format ${CODE_DIRS} + +## generate: generate code +generate: + ${RUN} bin/generate_code.py --codedir plotly + ${RUN} ruff format plotly + +## lint: check the code +lint: + ${RUN} ruff check ${CODE_DIRS} + +## test: run tests +test: + ${RUN} pytest tests + +## updatejs: update JavaScript bundle +updatejs: + ${RUN} bin/updatejs.py --codedir plotly + +## --: -- + +## clean: clean up repository +clean: + @find . -name '*~' -delete + @find . -name '.DS_Store' -delete + @rm -rf .coverage + @rm -rf .pytest_cache + @rm -rf .ruff_cache + @rm -rf dist + @rm -rf docs + @rm -rf pages/examples + +## sync: update Python packages +sync: + uv sync --extra dev diff --git a/README.md b/README.md index 11f117aca59..8e6ef158f46 100644 --- a/README.md +++ b/README.md @@ -76,13 +76,13 @@ Built on top of [plotly.js](https://github.com/plotly/plotly.js), `plotly.py` is ## Installation -plotly.py may be installed using pip +plotly.py may be installed using pip: ``` pip install plotly ``` -or conda. +or conda: ``` conda install -c conda-forge plotly @@ -90,8 +90,7 @@ conda install -c conda-forge plotly ### Jupyter Widget Support -For use as a Jupyter widget, install `jupyter` and `anywidget` -packages using `pip`: +For use as a Jupyter widget, install the `jupyter` and `anywidget` packages using `pip`: ``` pip install jupyter anywidget @@ -112,14 +111,14 @@ command line utility (legacy as of `plotly` version 4.9). #### Kaleido -The [`kaleido`](https://github.com/plotly/Kaleido) package has no dependencies and can be installed -using pip +The [`kaleido`](https://github.com/plotly/Kaleido) package has no dependencies +and can be installed using pip: ``` pip install -U kaleido ``` -or conda +or conda: ``` conda install -c conda-forge python-kaleido @@ -129,13 +128,13 @@ conda install -c conda-forge python-kaleido Some plotly.py features rely on fairly large geographic shape files. The county choropleth figure factory is one such example. These shape files are distributed as a -separate `plotly-geo` package. This package can be installed using pip... +separate `plotly-geo` package. This package can be installed using pip: ``` pip install plotly-geo==1.0.0 ``` -or conda +or conda: ``` conda install -c plotly plotly-geo=1.0.0 @@ -145,7 +144,7 @@ conda install -c plotly plotly-geo=1.0.0 ## Copyright and Licenses -Code and documentation copyright 2019 Plotly, Inc. +Code and documentation copyright Plotly, Inc. Code released under the [MIT license](https://github.com/plotly/plotly.py/blob/main/LICENSE.txt). diff --git a/_plotly_utils/basevalidators.py b/_plotly_utils/basevalidators.py index 0d7e387bbc3..aaf7b3ff694 100644 --- a/_plotly_utils/basevalidators.py +++ b/_plotly_utils/basevalidators.py @@ -2370,26 +2370,26 @@ def __init__(self, plotly_name, parent_name, data_class_str, data_docs, **kwargs @staticmethod def compute_graph_obj_module_str(data_class_str, parent_name): if parent_name == "frame" and data_class_str in ["Data", "Layout"]: - # Special case. There are no graph_objs.frame.Data or - # graph_objs.frame.Layout classes. These are remapped to - # graph_objs.Data and graph_objs.Layout + # Special case. There are no graph_objects.frame.Data or + # graph_objects.frame.Layout classes. These are remapped to + # graph_objects.Data and graph_objects.Layout parent_parts = parent_name.split(".") - module_str = ".".join(["plotly.graph_objs"] + parent_parts[1:]) + module_str = ".".join(["plotly.graph_objects"] + parent_parts[1:]) elif parent_name == "layout.template" and data_class_str == "Layout": # Remap template's layout to regular layout - module_str = "plotly.graph_objs" + module_str = "plotly.graph_objects" elif "layout.template.data" in parent_name: # Remap template's traces to regular traces parent_name = parent_name.replace("layout.template.data.", "") if parent_name: - module_str = "plotly.graph_objs." + parent_name + module_str = "plotly.graph_objects." + parent_name else: - module_str = "plotly.graph_objs" + module_str = "plotly.graph_objects" elif parent_name: - module_str = "plotly.graph_objs." + parent_name + module_str = "plotly.graph_objects." + parent_name else: - module_str = "plotly.graph_objs" + module_str = "plotly.graph_objects" return module_str @@ -2580,7 +2580,7 @@ def description(self): def get_trace_class(self, trace_name): # Import trace classes if trace_name not in self._class_map: - trace_module = import_module("plotly.graph_objs") + trace_module = import_module("plotly.graph_objects") trace_class_name = self.class_strs_map[trace_name] self._class_map[trace_name] = getattr(trace_module, trace_class_name) @@ -2591,7 +2591,7 @@ def validate_coerce(self, v, skip_invalid=False, _validate=True): # Import Histogram2dcontour, this is the deprecated name of the # Histogram2dContour trace. - from plotly.graph_objs import Histogram2dcontour + from plotly.graph_objects import Histogram2dcontour if v is None: v = [] diff --git a/bin/check-all-md.py b/bin/check-all-md.py new file mode 100644 index 00000000000..c06b00e3393 --- /dev/null +++ b/bin/check-all-md.py @@ -0,0 +1,17 @@ +from pathlib import Path +import os +import sys +from run_markdown import _parse_md + +TMP_FILE = "tmp.py" + +for filename in sys.argv[1:]: + content = Path(filename).read_text() + blocks = _parse_md(content) + for i, block in enumerate(blocks): + Path(TMP_FILE).write_text(block["code"].strip()) + sys.stdout.write(f"\n{'=' * 40}\n{filename}: {i}\n") + sys.stdout.flush() + sys.stdout.write(f"{'-' * 40}\n") + sys.stdout.flush() + os.system(f"python {TMP_FILE} > /dev/null") diff --git a/codegen/__init__.py b/bin/codegen/__init__.py similarity index 76% rename from codegen/__init__.py rename to bin/codegen/__init__.py index b299fa36045..6e2368fa094 100644 --- a/codegen/__init__.py +++ b/bin/codegen/__init__.py @@ -1,14 +1,12 @@ import json import os -import os.path as opath import shutil -import subprocess import sys from codegen.datatypes import build_datatype_py, write_datatype_py # noqa: F401 from codegen.compatibility import ( write_deprecated_datatypes, - write_graph_objs_graph_objs, + write_graph_objects_graph_objects, DEPRECATED_DATATYPES, ) from codegen.figure import write_figure_classes @@ -62,7 +60,7 @@ def preprocess_schema(plotly_schema): matching the structure of `figure.layout` and `traceTemplate` is a dict matching the structure of the trace with type `trace_type` (e.g. 'scatter'). Alternatively, this may be specified as an instance of -plotly.graph_objs.layout.Template. +plotly.graph_objects.layout.Template. Trace templates are applied cyclically to traces of each type. Container arrays (eg `annotations`) have special @@ -87,45 +85,30 @@ def preprocess_schema(plotly_schema): items["colorscale"] = items.pop("concentrationscales") -def make_paths(outdir): - """Make various paths needed for formatting and linting.""" +def make_paths(codedir): + """Make various paths needed for code generation.""" - validators_dir = opath.join(outdir, "validators") - graph_objs_dir = opath.join(outdir, "graph_objs") - graph_objects_path = opath.join(outdir, "graph_objects", "__init__.py") - return validators_dir, graph_objs_dir, graph_objects_path + validators_dir = codedir / "validators" + graph_objects_dir = codedir / "graph_objects" + graph_objs_path = codedir / "graph_objs" / "__init__.py" + return validators_dir, graph_objects_dir, graph_objs_path -def lint_code(outdir): - """Check Python code using settings in pyproject.toml.""" - - subprocess.call(["ruff", "check", *make_paths(outdir)]) - - -def reformat_code(outdir): - """Reformat Python code using settings in pyproject.toml.""" - - subprocess.call(["ruff", "format", *make_paths(outdir)]) - - -def perform_codegen(outdir, noformat=False): - """Generate code (and possibly reformat).""" +def perform_codegen(codedir, noformat=False): + """Generate code.""" # Get paths - validators_dir, graph_objs_dir, graph_objects_path = make_paths(outdir) + validators_dir, graph_objects_dir, graph_objs_path = make_paths(codedir) # Delete prior codegen output - if opath.exists(validators_dir): + if validators_dir.exists(): shutil.rmtree(validators_dir) - if opath.exists(graph_objs_dir): - shutil.rmtree(graph_objs_dir) + if graph_objects_dir.exists(): + shutil.rmtree(graph_objects_dir) # Load plotly schema - project_root = opath.dirname(outdir) - plot_schema_path = opath.join( - project_root, "codegen", "resources", "plot-schema.json" - ) - + project_root = codedir.parent + plot_schema_path = project_root / "resources" / "plot-schema.json" with open(plot_schema_path, "r") as f: plotly_schema = json.load(f) @@ -193,26 +176,26 @@ def perform_codegen(outdir, noformat=False): # Write out the JSON data for the validators os.makedirs(validators_dir, exist_ok=True) - write_validator_json(outdir, validator_params) + write_validator_json(codedir, validator_params) # Alls alls = {} # Write out datatypes for node in all_compound_nodes: - write_datatype_py(outdir, node) + write_datatype_py(codedir, node) # Deprecated - # These are deprecated legacy datatypes like graph_objs.Marker - write_deprecated_datatypes(outdir) + # These are deprecated legacy datatypes like graph_objects.Marker + write_deprecated_datatypes(codedir) - # Write figure class to graph_objs + # Write figure class to graph_objects data_validator = get_data_validator_instance(base_traces_node) layout_validator = layout_node.get_validator_instance() frame_validator = frame_node.get_validator_instance() write_figure_classes( - outdir, + codedir, base_traces_node, data_validator, layout_validator, @@ -239,10 +222,10 @@ def perform_codegen(outdir, noformat=False): f".{node.name_undercase}" ) - # Write plotly/graph_objs/graph_objs.py + # Write plotly/graph_objects/graph_objects.py # This is for backward compatibility. It just imports everything from - # graph_objs/__init__.py - write_graph_objs_graph_objs(outdir) + # graph_objects/__init__.py + write_graph_objects_graph_objects(codedir) # Add Figure and FigureWidget root_datatype_imports = datatype_rel_class_imports[()] @@ -258,7 +241,7 @@ def perform_codegen(outdir, noformat=False): import ipywidgets as _ipywidgets from packaging.version import Version as _Version if _Version(_ipywidgets.__version__) >= _Version("7.0.0"): - from ..graph_objs._figurewidget import FigureWidget + from ..graph_objects._figurewidget import FigureWidget else: raise ImportError() except Exception: @@ -272,7 +255,7 @@ def __getattr__(import_name): import ipywidgets from packaging.version import Version if Version(ipywidgets.__version__) >= Version("7.0.0"): - from ..graph_objs._figurewidget import FigureWidget + from ..graph_objects._figurewidget import FigureWidget return FigureWidget else: raise ImportError() @@ -287,12 +270,13 @@ def __getattr__(import_name): # __all__ for path_parts, class_names in alls.items(): if path_parts and class_names: - filepath = opath.join(outdir, "graph_objs", *path_parts, "__init__.py") + filepath = codedir / "graph_objects" + filepath = filepath.joinpath(*path_parts) / "__init__.py" with open(filepath, "at") as f: f.write(f"\n__all__ = {class_names}") # Output datatype __init__.py files - graph_objs_pkg = opath.join(outdir, "graph_objs") + graph_objects_pkg = codedir / "graph_objects" for path_parts in datatype_rel_class_imports: rel_classes = sorted(datatype_rel_class_imports[path_parts]) rel_modules = sorted(datatype_rel_module_imports.get(path_parts, [])) @@ -300,34 +284,29 @@ def __getattr__(import_name): init_extra = optional_figure_widget_import else: init_extra = "" - write_init_py(graph_objs_pkg, path_parts, rel_modules, rel_classes, init_extra) + write_init_py(graph_objects_pkg, path_parts, rel_modules, rel_classes, init_extra) - # Output graph_objects.py alias - graph_objects_rel_classes = [ - "..graph_objs." + rel_path.split(".")[-1] + # Output graph_objs.py alias + graph_objs_rel_classes = [ + "..graph_objects." + rel_path.split(".")[-1] for rel_path in datatype_rel_class_imports[()] ] - graph_objects_rel_modules = [ - "..graph_objs." + rel_module.split(".")[-1] + graph_objs_rel_modules = [ + "..graph_objects." + rel_module.split(".")[-1] for rel_module in datatype_rel_module_imports[()] ] - graph_objects_init_source = build_from_imports_py( - graph_objects_rel_modules, - graph_objects_rel_classes, + graph_objs_init_source = build_from_imports_py( + graph_objs_rel_modules, + graph_objs_rel_classes, init_extra=optional_figure_widget_import, ) - graph_objects_path = opath.join(outdir, "graph_objects", "__init__.py") - os.makedirs(opath.join(outdir, "graph_objects"), exist_ok=True) - with open(graph_objects_path, "wt") as f: + graph_objs_path = codedir / "graph_objs" + graph_objs_path.mkdir(parents=True, exist_ok=True) + graph_objs_path /= "__init__.py" + with open(graph_objs_path, "wt") as f: f.write("# ruff: noqa: F401\n") - f.write(graph_objects_init_source) - - # Run code formatter on output directories - if noformat: - print("skipping reformatting") - else: - reformat_code(outdir) + f.write(graph_objs_init_source) if __name__ == "__main__": diff --git a/codegen/compatibility.py b/bin/codegen/compatibility.py similarity index 82% rename from codegen/compatibility.py rename to bin/codegen/compatibility.py index 2b57685ff2e..43dd3b4b75c 100644 --- a/codegen/compatibility.py +++ b/bin/codegen/compatibility.py @@ -1,5 +1,4 @@ from io import StringIO -from os import path as opath from codegen.utils import write_source_py @@ -43,7 +42,7 @@ def build_deprecated_datatypes_py(): """ - Build datatype (graph_objs) class source code string for deprecated + Build datatype (graph_objects) class source code string for deprecated datatypes Returns @@ -58,7 +57,7 @@ def build_deprecated_datatypes_py(): buffer.write( r""" warnings.filterwarnings("default", - r"plotly\.graph_objs\.\w+ is deprecated", + r"plotly\.graph_objects\.\w+ is deprecated", DeprecationWarning) @@ -101,15 +100,15 @@ def build_deprecation_message(class_name, base_type, new): new: list of str List of replacements that users should use instead. Replacements may be: - - A package string relative to plotly.graph_objs. In this case the + - A package string relative to plotly.graph_objects. In this case the replacement class is assumed to be named `class_name`. e.g. `new` == ['layout`] and `class_name` == 'XAxis` corresponds - to the 'plotly.graph_objs.layout.XAxis' class + to the 'plotly.graph_objects.layout.XAxis' class - String containing the package and class. The string is identified as containing a class name if the final name in the package string begins with an uppercase letter. e.g. `new` == ['Scatter'] corresponds to the - ['plotly.graph_objs.Scatter'] class. + ['plotly.graph_objects.Scatter'] class. - The literal string 'etc.'. This string is not interpreted as a package or class and is displayed to the user as-is to indicate that the list of replacement classes is not complete. @@ -130,35 +129,35 @@ def build_deprecation_message(class_name, base_type, new): if not repl_is_class: repl_parts.append(class_name) - # Add plotly.graph_objs prefix - full_class_str = ".".join(["plotly", "graph_objs"] + repl_parts) + # Add plotly.graph_objects prefix + full_class_str = ".".join(["plotly", "graph_objects"] + repl_parts) replacements.append(full_class_str) replacemens_str = "\n - ".join(replacements) if base_type is list: return f"""\ -plotly.graph_objs.{class_name} is deprecated. +plotly.graph_objects.{class_name} is deprecated. Please replace it with a list or tuple of instances of the following types - {replacemens_str} """ else: return f"""\ -plotly.graph_objs.{class_name} is deprecated. +plotly.graph_objects.{class_name} is deprecated. Please replace it with one of the following more specific types - {replacemens_str} """ -def write_deprecated_datatypes(outdir): +def write_deprecated_datatypes(codedir): """ Build source code for deprecated datatype class definitions and write them to a file Parameters ---------- - outdir : - Root outdir in which the graph_objs package should reside + codedir : + Root directory in which the graph_objects package should reside Returns ------- @@ -166,34 +165,34 @@ def write_deprecated_datatypes(outdir): """ # Generate source code datatype_source = build_deprecated_datatypes_py() - filepath = opath.join(outdir, "graph_objs", "_deprecations.py") + filepath = codedir / "graph_objects" / "_deprecations.py" # Write file write_source_py(datatype_source, filepath) -def write_graph_objs_graph_objs(outdir): +def write_graph_objects_graph_objects(codedir): """ - Write the plotly/graph_objs/graph_objs.py file + Write the plotly/graph_objects/graph_objects.py file - This module just imports everything from the plotly.graph_objs package. + This module just imports everything from the plotly.graph_objects package. We write it for backward compatibility with legacy imports like: - from plotly.graph_objs import graph_objs + from plotly.graph_objects import graph_objects Parameters ---------- - outdir : str - Root outdir in which the graph_objs package should reside + codedir : str + Root directory in which the graph_objects package should reside Returns ------- None """ - filepath = opath.join(outdir, "graph_objs", "graph_objs.py") + filepath = codedir / "graph_objects" / "graph_objects.py" with open(filepath, "wt") as f: f.write( """\ -from plotly.graph_objs import * +from plotly.graph_objects import * """ ) diff --git a/codegen/datatypes.py b/bin/codegen/datatypes.py similarity index 94% rename from codegen/datatypes.py rename to bin/codegen/datatypes.py index 28b11d1fc59..6d147c1997c 100644 --- a/codegen/datatypes.py +++ b/bin/codegen/datatypes.py @@ -1,6 +1,5 @@ -import os.path as opath -import textwrap from io import StringIO +import textwrap from codegen.utils import CAVEAT, write_source_py @@ -54,7 +53,7 @@ def get_typing_type(plotly_type, array_ok=False): def build_datatype_py(node): """ - Build datatype (graph_objs) class source code string for a datatype + Build datatype (graph_objects) class source code string for a datatype PlotlyNode Parameters @@ -74,14 +73,14 @@ def build_datatype_py(node): # Handle template traces # # We want template trace/layout classes like - # plotly.graph_objs.layout.template.data.Scatter to map to the - # corresponding trace/layout class (e.g. plotly.graph_objs.Scatter). + # plotly.graph_objects.layout.template.data.Scatter to map to the + # corresponding trace/layout class (e.g. plotly.graph_objects.Scatter). # So rather than generate a class definition, we just import the # corresponding trace/layout class if node.parent_path_str == "layout.template.data": - return f"from plotly.graph_objs import {node.name_datatype_class}" + return f"from plotly.graph_objects import {node.name_datatype_class}" elif node.path_str == "layout.template.layout": - return "from plotly.graph_objs import Layout" + return "from plotly.graph_objects import Layout" # Extract node properties datatype_class = node.name_datatype_class @@ -176,13 +175,13 @@ def _subplot_re_match(self, prop): for subtype_node in subtype_nodes: if subtype_node.is_array_element: prop_type = ( - f"tuple[plotly.graph_objs{node.dotpath_str}." + f"tuple[plotly.graph_objects{node.dotpath_str}." + f"{subtype_node.name_datatype_class}]" ) elif subtype_node.is_compound: prop_type = ( - f"plotly.graph_objs{node.dotpath_str}." + f"plotly.graph_objects{node.dotpath_str}." + f"{subtype_node.name_datatype_class}" ) @@ -219,6 +218,9 @@ def _subplot_re_match(self, prop): else: property_docstring = property_description + # FIXME: replace '][' with ']\[' to avoid confusion with Markdown reference links + # property_docstring = property_docstring.replace("][", "]\\[") + # Write get property buffer.write( f'''\ @@ -289,7 +291,7 @@ def __init__(self""" # Constructor Docstring header = f"Construct a new {datatype_class} object" class_name = ( - f"plotly.graph_objs{node.parent_dotpath_str}.{node.name_datatype_class}" + f"plotly.graph_objects{node.parent_dotpath_str}.{node.name_datatype_class}" ) extras = [ @@ -580,12 +582,12 @@ def add_docstring( def write_datatype_py(outdir, node): """ - Build datatype (graph_objs) class source code and write to a file + Build datatype (graph_objects) class source code and write to a file Parameters ---------- outdir : - Root outdir in which the graph_objs package should reside + Root outdir in which the graph_objects package should reside node : The datatype node (node.is_datatype must evaluate to true) for which to build the datatype class @@ -595,14 +597,6 @@ def write_datatype_py(outdir, node): None """ - # Build file path - # filepath = opath.join(outdir, "graph_objs", *node.parent_path_parts, "__init__.py") - filepath = opath.join( - outdir, "graph_objs", *node.parent_path_parts, "_" + node.name_undercase + ".py" - ) - - # Generate source code + filepath = (outdir / "graph_objects").joinpath(*node.parent_path_parts) / f"_{node.name_undercase}.py" datatype_source = build_datatype_py(node) - - # Write file write_source_py(datatype_source, filepath, leading_newlines=2) diff --git a/codegen/figure.py b/bin/codegen/figure.py similarity index 98% rename from codegen/figure.py rename to bin/codegen/figure.py index a15d806937c..74e6d8df8f0 100644 --- a/codegen/figure.py +++ b/bin/codegen/figure.py @@ -1,5 +1,4 @@ from io import StringIO -from os import path as opath from codegen.datatypes import ( reindent_validator_description, @@ -256,7 +255,7 @@ def add_{trace_node.plotly_name}(self""" # Function body buffer.write( f""" - from plotly.graph_objs import {trace_node.name_datatype_class} + from plotly.graph_objects import {trace_node.name_datatype_class} new_trace = {trace_node.name_datatype_class}( """ ) @@ -672,7 +671,7 @@ def add_{method_prefix}{singular_name}(self""" # Function body buffer.write( f""" - from plotly.graph_objs import layout as _layout + from plotly.graph_objects import layout as _layout new_obj = _layout.{node.name_datatype_class}(arg, """ ) @@ -705,7 +704,7 @@ def add_{method_prefix}{singular_name}(self""" def write_figure_classes( - outdir, + codedir, trace_node, data_validator, layout_validator, @@ -715,13 +714,13 @@ def write_figure_classes( ): """ Construct source code for the Figure and FigureWidget classes and - write to graph_objs/_figure.py and graph_objs/_figurewidget.py + write to graph_objects/_figure.py and graph_objects/_figurewidget.py respectively Parameters ---------- - outdir : str - Root outdir in which the graph_objs package should reside + codedir : str + Root directory in which the graph_objects package should reside trace_node : PlotlyNode Root trace node (the node that is the parent of all of the individual trace nodes like bar, scatter, etc.) @@ -768,5 +767,5 @@ def write_figure_classes( ) # Format and write to file - filepath = opath.join(outdir, "graph_objs", f"_{fig_classname.lower()}.py") + filepath = codedir / "graph_objects" / f"_{fig_classname.lower()}.py" write_source_py(figure_source, filepath) diff --git a/bin/codegen/utils.py b/bin/codegen/utils.py new file mode 100644 index 00000000000..c002574ada7 --- /dev/null +++ b/bin/codegen/utils.py @@ -0,0 +1,1315 @@ +from collections import ChainMap +from importlib import import_module +from io import StringIO +import re +import textwrap +from typing import List + +CAVEAT = """ + +# --- THIS FILE IS AUTO-GENERATED --- +# Modifications will be overwitten the next time code generation run. + +""" + + +# Source code utilities +# ===================== +def write_source_py(py_source, filepath, leading_newlines=0): + """ + Format Python source code and write to a file, creating parent + directories as needed. + + Parameters + ---------- + py_source : str + String containing valid Python source code. If string is empty, + no file will be written. + filepath : str + Full path to the file to be written + Returns + ------- + None + """ + if py_source: + # Make dir if needed + filepath.parent.mkdir(exist_ok=True) + + # Write file + py_source = "\n" * leading_newlines + py_source + with open(filepath, "at") as f: + f.write(py_source) + + +def build_from_imports_py(rel_modules=(), rel_classes=(), init_extra=""): + """ + Build a string containing a series of `from X import Y` lines + + Parameters + ---------- + rel_modules: list of str + list of submodules to import, of the form .submodule + rel_classes: list of str + list of submodule classes/variables to import, of the form ._submodule.Foo + init_extra: str + Extra code snippet to append to end of __init__.py file + Returns + ------- + str + String containing a series of imports + """ + + rel_modules = list(rel_modules) + rel_classes = list(rel_classes) + + import_lines = [] + for rel in rel_classes + rel_modules: + rel_parts = rel.split(".") + parent_module = ".".join(rel_parts[:-1]) or "." + import_target = rel_parts[-1] + import_line = f"from {parent_module} import {import_target}" + import_lines.append(import_line) + + imports_str = "\n ".join(import_lines) + + result = f"""\ +import sys +from typing import TYPE_CHECKING +if TYPE_CHECKING: + {imports_str} +else: + from _plotly_utils.importers import relative_import + __all__, __getattr__, __dir__ = relative_import( + __name__, + {repr(rel_modules)}, + {repr(rel_classes)} + ) + +{init_extra} +""" + return result + + +def write_init_py(pkg_root, path_parts, rel_modules=(), rel_classes=(), init_extra=""): + """ + Build __init__.py source code and write to a file + + Parameters + ---------- + pkg_root : str + Root package in which the top-level an __init__.py file with empty + path_parts should reside + path_parts : tuple of str + Tuple of sub-packages under pkg_root where the __init__.py + file should be written + rel_modules: list of str + list of submodules to import, of the form .submodule + rel_classes: list of str + list of submodule classes/variables to import, of the form ._submodule.Foo + init_extra: str + Extra code snippet to append to end of __init__.py file + Returns + ------- + None + """ + # Generate source code + init_source = build_from_imports_py(rel_modules, rel_classes, init_extra) + + # Write file + filepath = pkg_root.joinpath(*path_parts) / "__init__.py" + write_source_py(init_source, filepath) + + +def format_description(desc): + # Remove surrounding *s from numbers + desc = re.sub(r"(^|[\s(,.:])\*([\d.]+)\*([\s),.:]|$)", r"\1\2\3", desc) + + # replace *true* with True + desc = desc.replace("*true*", "True") + desc = desc.replace("*false*", "False") + + # Replace *word* with "word" + desc = re.sub(r"(^|[\s(,.:])\*(\S+)\*([\s),.:]|$)", r'\1"\2"\3', desc) + + # Special case strings that don't satisfy regex above + other_strings = [ + "", + "Courier New", + "Droid Sans", + "Droid Serif", + "Droid Sans Mono", + "Gravitas One", + "Old Standard TT", + "Open Sans", + "PT Sans Narrow", + "Times New Roman", + "bottom plot", + "top plot", + ] + + for s in other_strings: + desc = desc.replace(f"*{s}*", f'"{s}"') + + # Replace {array} with list + desc = desc.replace("an {array}", "a list") + desc = desc.replace("{array}", "list") + + # Replace {arrays} with lists + desc = desc.replace("{arrays}", "lists") + + # replace {2D array} with 2D list + desc = desc.replace("{2D array}", "2D list") + + # replace {2D arrays} with 2D lists + desc = desc.replace("{2D arrays}", "2D lists") + + # FIXME: replace '][' with ']\[' to avoid confusion with Markdown reference links + # desc = desc.replace("][", r"]\\[") + + return desc + + +# Constants +# ========= +# Mapping from full property paths to custom validator classes +CUSTOM_VALIDATOR_DATATYPES = { + "layout.image.source": "_plotly_utils.basevalidators.ImageUriValidator", + "layout.template": "_plotly_utils.basevalidators.BaseTemplateValidator", + "frame.data": "plotly.validators.DataValidator", + "frame.layout": "plotly.validators.LayoutValidator", +} + + +# Mapping from property string (as found in plot-schema.json) to a custom +# class name. If not included here, names are converted to TitleCase and +# underscores are removed. +OBJECT_NAME_TO_CLASS_NAME = { + "angularaxis": "AngularAxis", + "colorbar": "ColorBar", + "error_x": "ErrorX", + "error_y": "ErrorY", + "error_z": "ErrorZ", + "histogram2d": "Histogram2d", + "histogram2dcontour": "Histogram2dContour", + "mesh3d": "Mesh3d", + "radialaxis": "RadialAxis", + "scatter3d": "Scatter3d", + "xaxis": "XAxis", + "xbins": "XBins", + "yaxis": "YAxis", + "ybins": "YBins", + "zaxis": "ZAxis", +} + +# Tuple of types to be considered dicts by PlotlyNode logic +dict_like = (dict, ChainMap) + + +# PlotlyNode classes +# ================== +class PlotlyNode: + """ + Base class that represents a node in the plot-schema.json file + """ + + # Constructor + def __init__(self, plotly_schema, node_path=(), parent=None): + """ + Superclass constructor for all node types + + Parameters + ---------- + plotly_schema : dict + JSON-parsed version of the default-schema.xml file + node_path : str or tuple + Path of from the 'root' node for the current trace type to the + particular node that this instance represents + parent : PlotlyNode + Reference to the node's parent + """ + # Save params + self.plotly_schema = plotly_schema + self._parent = parent + + # Process node path + if isinstance(node_path, str): + node_path = (node_path,) + self.node_path = node_path + + # Compute children + # Note the node_data is a property that must be computed by the + # subclass based on plotly_schema and node_path + if isinstance(self.node_data, dict_like): + childs_parent = ( + parent if self.node_path and self.node_path[-1] == "items" else self + ) + + self._children = [ + self.__class__( + self.plotly_schema, + node_path=self.node_path + (c,), + parent=childs_parent, + ) + for c in self.node_data + if c and c[0] != "_" + ] + + # Sort by plotly name + self._children = sorted(self._children, key=lambda node: node.plotly_name) + else: + self._children = [] + + # Magic methods + def __repr__(self): + return self.path_str + + # Abstract methods + @property + def node_data(self): + """ + Dictionary of the subtree of the plotly_schema that this node + represents + + Returns + ------- + dict + """ + raise NotImplementedError() + + @property + def description(self): + """ + Description of the node + + Returns + ------- + str or None + """ + raise NotImplementedError() + + @property + def name_base_datatype(self): + """ + Superclass to use when generating a datatype class for this node + + Returns + ------- + str + """ + raise NotImplementedError + + # Names + @property + def root_name(self): + """ + Name of the node with empty node_path + + Returns + ------- + str + """ + raise NotImplementedError() + + @property + def plotly_name(self): + """ + Name of the node. Either the base_name or the name directly out of + the plotly_schema + + Returns + ------- + str + """ + if len(self.node_path) == 0: + return self.root_name + else: + return self.node_path[-1] + + @property + def name_datatype_class(self): + """ + Name of the Python datatype class representing this node + + Returns + ------- + str + """ + if self.plotly_name in OBJECT_NAME_TO_CLASS_NAME: + return OBJECT_NAME_TO_CLASS_NAME[self.plotly_name] + else: + return self.plotly_name.title().replace("_", "") + + @property + def name_undercase(self): + """ + Name of node converted to undercase (all lowercase with underscores + separating words) + + Returns + ------- + str + """ + if not self.plotly_name: + # Empty plotly_name + return self.plotly_name + + # Lowercase leading char + name1 = self.plotly_name[0].lower() + self.plotly_name[1:] + + # Replace capital chars by underscore-lower + name2 = "".join([("" if not c.isupper() else "_") + c.lower() for c in name1]) + + return name2 + + @property + def name_property(self): + """ + Name of the Python property corresponding to this node. This is the + same as `name_undercase` for compound nodes, but an 's' is appended + to the name for array nodes + + Returns + ------- + str + """ + + return self.plotly_name + ( + "s" + if self.is_array_element + and + # Don't add 's' to layout.template.data.scatter etc. + not ( + self.parent + and self.parent.parent + and self.parent.parent.parent + and self.parent.parent.parent.name_property == "template" + ) + else "" + ) + + @property + def name_validator_class(self) -> str: + """ + Name of the Python validator class representing this node + + Returns + ------- + str + """ + return self.name_property.title() + "Validator" + + @property + def name_base_validator(self) -> str: + """ + Superclass to use when generating a validator class for this node + + Returns + ------- + str + """ + if self.path_str in CUSTOM_VALIDATOR_DATATYPES: + validator_base = CUSTOM_VALIDATOR_DATATYPES[self.path_str] + elif self.plotly_name.endswith("src") and self.datatype == "string": + validator_base = "_plotly_utils.basevalidators.SrcValidator" + elif self.plotly_name.endswith("dash") and self.datatype == "string": + validator_base = "_plotly_utils.basevalidators.DashValidator" + elif self.plotly_name == "title" and self.datatype == "compound": + validator_base = "_plotly_utils.basevalidators.TitleValidator" + else: + datatype_title_case = self.datatype.title().replace("_", "") + validator_base = ( + f"_plotly_utils.basevalidators.{datatype_title_case}Validator" + ) + + return validator_base + + # Validators + def get_validator_params(self): + """ + Get kwargs to pass to the constructor of this node's validator + superclass. + + Returns + ------- + dict + The keys are strings matching the names of the constructor + params of this node's validator superclass. The values are + repr-strings of the values to be passed to the constructor. + These values are ready to be used to code generate calls to the + constructor. The values should be evald before being passed to + the constructor directly. + + """ + params = { + "plotly_name": repr(self.name_property), + "parent_name": repr(self.parent_path_str), + } + + if self.is_compound: + params["data_class_str"] = repr(self.name_datatype_class) + params["data_docs"] = '"""\n"""' + else: + assert self.is_simple + + # Exclude general properties + excluded_props = ["valType", "description", "dflt"] + if self.datatype == "subplotid": + # Default is required for subplotid validator + excluded_props.remove("dflt") + + attr_nodes = [ + n for n in self.simple_attrs if n.plotly_name not in excluded_props + ] + + for node in attr_nodes: + params[node.name_undercase] = repr(node.node_data) + + # Add extra properties + if self.datatype == "color" and self.parent: + # Check for colorscale sibling. We use the presence of a + # colorscale sibling to determine whether numeric color + # values are permissible + colorscale_node_list = [ + node + for node in self.parent.child_datatypes + if node.datatype == "colorscale" + ] + if colorscale_node_list: + colorscale_path = colorscale_node_list[0].path_str + params["colorscale_path"] = repr(colorscale_path) + elif self.datatype == "literal": + params["val"] = self.node_data + + return params + + def get_validator_instance(self): + """ + Return a constructed validator for this node + + Returns + ------- + BaseValidator + """ + + # Evaluate validator params to convert repr strings into values + # e.g. '2' -> 2 + params = { + prop: eval(repr_val) + for prop, repr_val in self.get_validator_params().items() + } + + validator_parts = self.name_base_validator.split(".") + if validator_parts[0] != "_plotly_utils": + return None + else: + validator_class_str = validator_parts[-1] + validator_module = ".".join(validator_parts[:-1]) + + validator_class = getattr( + import_module(validator_module), validator_class_str + ) + + return validator_class(**params) + + # Datatypes + @property + def datatype(self) -> str: + """ + Datatype string for this node. One of 'compound_array', 'compound', + 'literal', or the value of the 'valType' attribute + + Returns + ------- + str + """ + if self.is_array_element: + return "compound_array" + elif self.is_compound: + return "compound" + elif self.is_simple: + return self.node_data.get("valType") + else: + return "literal" + + @property + def is_array_ok(self) -> bool: + """ + Return true if arrays of datatype are acceptable + + Returns + ------- + bool + """ + return self.node_data.get("arrayOk", False) + + @property + def is_compound(self) -> bool: + """ + Node has a compound (in contrast to simple) datatype. + Note: All array and array_element types are also considered compound + + Returns + ------- + bool + """ + return ( + isinstance(self.node_data, dict_like) + and not self.is_simple + and not self.is_mapped + and self.plotly_name not in ("items", "impliedEdits", "transforms") + ) + + @property + def is_literal(self) -> bool: + """ + Node has a particular literal value (e.g. 'foo', or 23) + + Returns + ------- + bool + """ + return isinstance(self.node_data, (str, int, float)) + + @property + def is_simple(self) -> bool: + """ + Node has a simple datatype (e.g. boolean, color, colorscale, etc.) + + Returns + ------- + bool + """ + return ( + isinstance(self.node_data, dict_like) + and "valType" in self.node_data + and self.plotly_name != "items" + ) + + @property + def is_array(self) -> bool: + """ + Node has an array datatype + + Returns + ------- + bool + """ + return ( + isinstance(self.node_data, dict_like) + and self.node_data.get("role", "") == "object" + and "items" in self.node_data + and self.name_property != "transforms" + ) + + @property + def is_array_element(self): + """ + Node has an array-element datatype + + Returns + ------- + bool + """ + if self.parent: + return self.parent.is_array + else: + return False + + @property + def is_datatype(self) -> bool: + """ + Node represents any kind of datatype + + Returns + ------- + bool + """ + return self.is_simple or self.is_compound or self.is_array or self.is_mapped + + @property + def is_mapped(self) -> bool: + """ + Node represents a mapping from a deprecated property to a + normal property + + Returns + ------- + bool + """ + return False + + # Node path + def tidy_path_part(self, p): + """ + Return a tidy version of raw path entry. This allows subclasses to + adjust the raw property names in the plotly_schema + + Parameters + ---------- + p : str + Path element string + + Returns + ------- + str + """ + return p + + @property + def path_parts(self): + """ + Tuple of strings locating this node in the plotly_schema + e.g. ('layout', 'images', 'opacity') + + Returns + ------- + tuple of str + """ + res = [self.root_name] if self.root_name else [] + for i, p in enumerate(self.node_path): + # Handle array datatypes + if p == "items" or ( + i < len(self.node_path) - 1 and self.node_path[i + 1] == "items" + ): + # e.g. [parcoords, dimensions, items, dimension] -> + # [parcoords, dimension] + pass + else: + res.append(self.tidy_path_part(p)) + return tuple(res) + + # Node path strings + @property + def path_str(self): + """ + String containing path_parts joined on periods + e.g. 'layout.images.opacity' + + Returns + ------- + str + """ + return ".".join(self.path_parts) + + @property + def dotpath_str(self): + """ + path_str prefixed by a period if path_str is not empty, otherwise empty + + Returns + ------- + str + """ + path_str = "" + for p in self.path_parts: + path_str += "." + p + return path_str + + @property + def parent_path_parts(self): + """ + Tuple of strings locating this node's parent in the plotly_schema + + Returns + ------- + tuple of str + """ + return self.path_parts[:-1] + + @property + def parent_path_str(self): + """ + String containing parent_path_parts joined on periods + + Returns + ------- + str + """ + return ".".join(self.path_parts[:-1]) + + @property + def parent_dotpath_str(self): + """ + parent_path_str prefixed by a period if parent_path_str is not empty, + otherwise empty + + Returns + ------- + str + """ + path_str = "" + for p in self.parent_path_parts: + path_str += "." + p + return path_str + + # Children + @property + def parent(self): + """ + Parent node + + Returns + ------- + PlotlyNode + """ + return self._parent + + @property + def children(self): + """ + List of all child nodes + + Returns + ------- + list of PlotlyNode + """ + return self._children + + @property + def simple_attrs(self): + """ + List of simple attribute child nodes + (only valid when is_simple == True) + + Returns + ------- + list of PlotlyNode + """ + if not self.is_simple: + raise ValueError( + f"Cannot get simple attributes of the simple object '{self.path_str}'" + ) + + return [ + n for n in self.children if n.plotly_name not in ["valType", "description"] + ] + + @property + def child_datatypes(self): + """ + List of all datatype child nodes + + Returns + ------- + list of PlotlyNode + """ + nodes = [] + for n in self.children: + if n.is_array: + # Add array element node + nodes.append(n.children[0].children[0]) + + # Add elementdefaults node. Require parent_path_parts not + # empty to avoid creating defaults classes for traces + if n.parent_path_parts and n.parent_path_parts != ( + "layout", + "template", + "data", + ): + nodes.append(ElementDefaultsNode(n, self.plotly_schema)) + elif n.is_compound and n.plotly_name == "title": + nodes.append(n) + + # Remap deprecated title properties + deprecated_data = n.parent.node_data.get("_deprecated", {}) + deprecated_title_prop_names = [ + p for p in deprecated_data if p.startswith("title") and p != "title" + ] + for prop_name in deprecated_title_prop_names: + mapped_prop_name = prop_name.replace("title", "") + + mapped_prop_node = [ + title_prop + for title_prop in n.child_datatypes + if title_prop.plotly_name == mapped_prop_name + ][0] + + prop_parent = n.parent + + legacy_node = MappedPropNode( + mapped_prop_node, prop_parent, prop_name, self.plotly_schema + ) + + nodes.append(legacy_node) + + elif n.is_datatype: + nodes.append(n) + + return nodes + + @property + def child_compound_datatypes(self): + """ + List of all compound datatype child nodes + + Returns + ------- + list of PlotlyNode + """ + return [n for n in self.child_datatypes if n.is_compound] + + @property + def child_simple_datatypes(self) -> List["PlotlyNode"]: + """ + List of all simple datatype child nodes + + Returns + ------- + list of PlotlyNode + """ + return [n for n in self.child_datatypes if n.is_simple] + + @property + def child_literals(self) -> List["PlotlyNode"]: + """ + List of all literal child nodes + + Returns + ------- + list of PlotlyNode + """ + return [n for n in self.children if n.is_literal] + + def has_child(self, name) -> bool: + """ + Check whether node has child of the specified name + """ + return bool([n for n in self.children if n.plotly_name == name]) + + def get_constructor_params_docstring(self, indent=12): + """ + Return a docstring-style string containing the names and + descriptions of all of the node's child datatypes + + Parameters + ---------- + indent : int + Leading indent of the string + + Returns + ------- + str + """ + assert self.is_compound + + buffer = StringIO() + + subtype_nodes = self.child_datatypes + for subtype_node in subtype_nodes: + raw_description = subtype_node.description + if raw_description: + subtype_description = raw_description + elif subtype_node.is_array_element: + if ( + self.name_datatype_class == "Data" + and self.parent + and self.parent.name_datatype_class == "Template" + ): + class_name = ( + f"plotly.graph_objects.{subtype_node.name_datatype_class}" + ) + else: + class_name = ( + f"plotly.graph_objects" + f"{subtype_node.parent_dotpath_str}." + f"{subtype_node.name_datatype_class}" + ) + subtype_description = ( + f"A tuple of :class:`{class_name}` instances or " + "dicts with compatible properties" + ) + elif subtype_node.is_compound: + if ( + subtype_node.name_datatype_class == "Layout" + and self.name_datatype_class == "Template" + ): + class_name = "plotly.graph_objects.Layout" + else: + class_name = ( + f"plotly.graph_objects" + f"{subtype_node.parent_dotpath_str}." + f"{subtype_node.name_datatype_class}" + ) + + subtype_description = ( + f":class:`{class_name}` instance or dict with compatible properties" + ) + else: + subtype_description = "" + + subtype_description = "\n".join( + textwrap.wrap( + subtype_description, + initial_indent=" " * (indent + 4), + subsequent_indent=" " * (indent + 4), + width=79 - (indent + 4), + ) + ) + + buffer.write("\n" + " " * indent + subtype_node.name_property) + buffer.write("\n" + subtype_description) + + return buffer.getvalue() + + # Static helpers + @staticmethod + def get_all_compound_datatype_nodes(plotly_schema, node_class): + """ + Build a list of the entire hierarchy of compound datatype nodes for + a given PlotlyNode subclass + + Parameters + ---------- + plotly_schema : dict + JSON-parsed version of the default-schema.xml file + node_class + PlotlyNode subclass + + Returns + ------- + list of PlotlyNode + """ + nodes = [] + nodes_to_process = [node_class(plotly_schema)] + + while nodes_to_process: + node = nodes_to_process.pop() + + if node.plotly_name and not node.is_array: + nodes.append(node) + + non_defaults_compound_children = [ + node + for node in node.child_compound_datatypes + if not isinstance(node, ElementDefaultsNode) + ] + + nodes_to_process.extend(non_defaults_compound_children) + + return nodes + + @staticmethod + def get_all_datatype_nodes(plotly_schema, node_class): + """ + Build a list of the entire hierarchy of datatype nodes for a given + PlotlyNode subclass + + Parameters + ---------- + plotly_schema : dict + JSON-parsed version of the default-schema.xml file + node_class + PlotlyNode subclass + + Returns + ------- + list of PlotlyNode + """ + nodes = [] + nodes_to_process = [node_class(plotly_schema)] + + while nodes_to_process: + node = nodes_to_process.pop() + + if node.plotly_name and not node.is_array: + nodes.append(node) + + nodes_to_process.extend(node.child_datatypes) + + return nodes + + +class TraceNode(PlotlyNode): + """ + Class representing datatypes in the trace hierarchy + """ + + # Constructor + def __init__(self, plotly_schema, node_path=(), parent=None): + super().__init__(plotly_schema, node_path, parent) + + @property + def name_base_datatype(self): + if len(self.node_path) <= 1: + return "BaseTraceType" + else: + return "BaseTraceHierarchyType" + + @property + def root_name(self): + return "" + + # Raw data + @property + def node_data(self) -> dict: + if not self.node_path: + node_data = self.plotly_schema["traces"] + else: + trace_name = self.node_path[0] + node_data = self.plotly_schema["traces"][trace_name]["attributes"] + for prop_name in self.node_path[1:]: + node_data = node_data[prop_name] + + return node_data + + # Description + @property + def description(self) -> str: + if len(self.node_path) == 0: + desc = "" + elif len(self.node_path) == 1: + # Get trace descriptions + trace_name = self.node_path[0] + desc = self.plotly_schema["traces"][trace_name]["meta"].get( + "description", "" + ) + else: + # Get datatype description + desc = self.node_data.get("description", "") + + if isinstance(desc, list): + desc = "".join(desc) + + return format_description(desc) + + +class LayoutNode(PlotlyNode): + """ + Class representing datatypes in the layout hierarchy + """ + + # Constructor + def __init__(self, plotly_schema, node_path=(), parent=None): + # Get main layout properties + layout = plotly_schema["layout"]["layoutAttributes"] + + # Get list of additional layout properties for each trace + trace_layouts = [ + plotly_schema["traces"][trace].get("layoutAttributes", {}) + for trace in plotly_schema["traces"] + if trace != "barpolar" + ] + + extra_polar_nodes = plotly_schema["traces"]["barpolar"].get( + "layoutAttributes", {} + ) + layout["polar"].update(extra_polar_nodes) + + # Chain together into layout_data + self.layout_data = ChainMap(layout, *trace_layouts) + + # Call superclass constructor + super().__init__(plotly_schema, node_path, parent) + + @property + def name_base_datatype(self): + if len(self.node_path) == 0: + return "BaseLayoutType" + else: + return "BaseLayoutHierarchyType" + + @property + def root_name(self): + return "layout" + + @property + def plotly_name(self) -> str: + if len(self.node_path) == 0: + return self.root_name + else: + return self.node_path[-1] + + # Description + @property + def description(self) -> str: + desc = self.node_data.get("description", "") + if isinstance(desc, list): + desc = "".join(desc) + return format_description(desc) + + # Raw data + @property + def node_data(self) -> dict: + node_data = self.layout_data + for prop_name in self.node_path: + node_data = node_data[prop_name] + + return node_data + + +class FrameNode(PlotlyNode): + """ + Class representing datatypes in the frames hierarchy + """ + + # Constructor + def __init__(self, plotly_schema, node_path=(), parent=None): + super().__init__(plotly_schema, node_path, parent) + + @property + def name_base_datatype(self): + return "BaseFrameHierarchyType" + + @property + def root_name(self): + return "" + + @property + def plotly_name(self) -> str: + if len(self.node_path) < 2: + return self.root_name + elif len(self.node_path) == 2: + return "frame" # override frames_entry + else: + return self.node_path[-1] + + def tidy_path_part(self, p): + return "frame" if p == "frames_entry" else p + + # Description + @property + def description(self) -> str: + desc = self.node_data.get("description", "") + if isinstance(desc, list): + desc = "".join(desc) + return format_description(desc) + + # Raw data + @property + def node_data(self) -> dict: + node_data = self.plotly_schema["frames"] + for prop_name in self.node_path: + node_data = node_data[prop_name] + + return node_data + + +class ElementDefaultsNode(PlotlyNode): + def __init__(self, array_node, plotly_schema): + """ + Create node that represents element defaults properties + (e.g. layout.annotationdefaults). Construct as a wrapper around the + corresponding array property node (e.g. layout.annotations) + + Parameters + ---------- + array_node: PlotlyNode + """ + super().__init__( + plotly_schema, node_path=array_node.node_path, parent=array_node.parent + ) + + assert array_node.is_array + self.array_node = array_node + self.element_node = array_node.children[0].children[0] + + @property + def node_data(self): + return {} + + @property + def description(self): + array_property_path = self.parent_path_str + "." + self.array_node.name_property + + if isinstance(self.array_node, TraceNode): + data_path = "data." + else: + data_path = "" + + defaults_property_path = ( + "layout.template." + + data_path + + self.parent_path_str + + "." + + self.plotly_name + ) + return f"""\ +When used in a template +(as {defaults_property_path}), +sets the default property values to use for elements +of {array_property_path}""" + + @property + def name_base_datatype(self): + return self.element_node.name_base_datatype + + @property + def root_name(self): + return self.array_node.root_name + + @property + def plotly_name(self): + return self.element_node.plotly_name + "defaults" + + @property + def name_datatype_class(self): + return self.element_node.name_datatype_class + + +class MappedPropNode(PlotlyNode): + def __init__(self, mapped_prop_node, parent, prop_name, plotly_schema): + """ + Create node that represents a legacy title property. + e.g. layout.title_font. These properties are now subproperties under + the sibling `title` property. e.g. layout.title.font. + + Parameters + ---------- + title_node: PlotlyNode + prop_name: str + The name of the propery (without the title prefix) + e.g. 'font' to represent the layout.title_font property. + """ + node_path = parent.node_path + (prop_name,) + super().__init__(plotly_schema, node_path=node_path, parent=parent) + + self.mapped_prop_node = mapped_prop_node + self.prop_name = prop_name + + @property + def node_data(self): + return {} + + @property + def description(self): + res = ( + f"""\ +Deprecated: Please use {self.mapped_prop_node.path_str} instead. +""" + + self.mapped_prop_node.description + ) + return res + + @property + def name_base_datatype(self): + return self.mapped_prop_node.description + + @property + def root_name(self): + return self.parent.root_name + + @property + def plotly_name(self): + return self.prop_name + + @property + def name_datatype_class(self): + return self.mapped_prop_node.name_datatype_class + + @property + def is_mapped(self): + return True + + @property + def datatype(self): + return self.mapped_prop_node.datatype + + def get_validator_instance(self): + return self.mapped_prop_node.get_validator_instance() + + @property + def relative_path(self): + return ( + self.mapped_prop_node.parent.plotly_name, + self.mapped_prop_node.plotly_name, + ) diff --git a/codegen/validators.py b/bin/codegen/validators.py similarity index 94% rename from codegen/validators.py rename to bin/codegen/validators.py index 4cef19fa29b..04ea65d2f8a 100644 --- a/codegen/validators.py +++ b/bin/codegen/validators.py @@ -1,4 +1,3 @@ -import os.path as opath import json import _plotly_utils.basevalidators @@ -54,7 +53,7 @@ def get_data_validator_params(base_trace_node: TraceNode, store: dict): } -def write_validator_json(outdir, params: dict): +def write_validator_json(codedir, params: dict): """ Write out a JSON serialization of the validator arguments for all validators (keyed by f"{parent_name}.{plotly_name}) @@ -64,8 +63,8 @@ def write_validator_json(outdir, params: dict): Parameters ---------- - outdir : str - Root outdir in which the validators package should reside + codedir : str + Root directory in which the validators package should reside params : dict Dictionary to store the JSON data for the validator Returns @@ -78,7 +77,7 @@ def write_validator_json(outdir, params: dict): raise ValueError("Expected params to be a dictionary") # Write file - filepath = opath.join(outdir, "validators", "_validators.json") + filepath = codedir / "validators" / "_validators.json" with open(filepath, "w") as f: f.write(json.dumps(params, indent=4)) diff --git a/bin/generate_code.py b/bin/generate_code.py new file mode 100644 index 00000000000..94fef3991cf --- /dev/null +++ b/bin/generate_code.py @@ -0,0 +1,27 @@ +"""Generate code.""" + +import argparse +from pathlib import Path + +import utils + + +def main(): + """Main driver.""" + + args = parse_args() + codedir = utils.select_code_directory(args) + utils.perform_codegen(codedir, noformat=args.noformat) + + +def parse_args(): + """Parse command-line arguments.""" + + parser = argparse.ArgumentParser() + parser.add_argument("--noformat", action="store_true", help="prevent reformatting") + parser.add_argument("--codedir", type=Path, help="code directory") + return parser.parse_args() + + +if __name__ == "__main__": + main() diff --git a/bin/generate_reference_pages.py b/bin/generate_reference_pages.py new file mode 100644 index 00000000000..cba3d633276 --- /dev/null +++ b/bin/generate_reference_pages.py @@ -0,0 +1,58 @@ +"""Generate the code reference pages and navigation.""" + +import os +from pathlib import Path + +import mkdocs_gen_files + + +# Saving Markdown files? +temp_dir = os.getenv("MKDOCS_TEMP_DIR", None) +if temp_dir is not None: + temp_dir = Path(temp_dir) + +# Set up the generation engine. +nav = mkdocs_gen_files.Nav() + +# Match each Python file. +for path in sorted(Path("plotly").rglob("*.py")): + # Documentation path. + module_path = path.relative_to(".").with_suffix("") + doc_path = path.relative_to(".").with_suffix(".md") + full_doc_path = Path("reference", doc_path) + + # Handle dunder special cases. + parts = tuple(module_path.parts) + if parts[-1] == "__init__": + parts = parts[:-1] + doc_path = doc_path.with_name("index.md") + full_doc_path = full_doc_path.with_name("index.md") + elif parts[-1] == "__main__": + continue + + # Save constructed data. + nav[parts] = doc_path.as_posix() + mkdocs_gen_files.set_edit_path(full_doc_path, path) + + # Save in-memory file. + with mkdocs_gen_files.open(full_doc_path, "w") as writer: + ident = ".".join(parts) + writer.write(f"# {ident}\n\n") + writer.write(f"::: {ident}") + + # Save to disk if requested. + if temp_dir is not None: + temp_path = temp_dir / doc_path + temp_path.parent.mkdir(exist_ok=True, parents=True) + with open(temp_path, "w") as writer: + ident = ".".join(parts) + writer.write(f"# {ident}\n\n") + writer.write(f"::: {ident}") + +# Generate navigation summary. +with mkdocs_gen_files.open("reference/SUMMARY.md", "w") as writer: + writer.writelines(nav.build_literate_nav()) +if temp_dir is not None: + temp_path = temp_dir / "SUMMARY.md" + with open(temp_path, "w") as writer: + writer.writelines(nav.build_literate_nav()) diff --git a/bin/run_markdown.py b/bin/run_markdown.py new file mode 100644 index 00000000000..d427c03c650 --- /dev/null +++ b/bin/run_markdown.py @@ -0,0 +1,255 @@ +#!/usr/bin/env python3 +""" +Process Markdown files with embedded Python code blocks, saving +the output and images. +""" + +import argparse +from contextlib import redirect_stdout, redirect_stderr +import io +from pathlib import Path +import plotly.graph_objects as go +import sys +import traceback + + +def main(): + args = _parse_args() + for filename in args.inputs: + _do_file(args, Path(filename)) + + +def _do_file(args, input_file): + """Process a single file.""" + + # Validate input file + if not input_file.exists(): + print(f"Error: '{input_file}' not found", file=sys.stderr) + sys.exit(1) + + # Determine output file path etc. + stem = input_file.stem + output_file = args.outdir / f"{input_file.stem}{input_file.suffix}" + if input_file.resolve() == output_file.resolve(): + print(f"Error: output would overwrite input '{input_file}'", file=sys.stderr) + sys.exit(1) + + # Read input + try: + with open(input_file, "r", encoding="utf-8") as f: + content = f.read() + except Exception as e: + print(f"Error reading input file: {e}", file=sys.stderr) + sys.exit(1) + + # Parse markdown and extract code blocks + _report(args.verbose > 0, f"Processing {input_file}...") + code_blocks = _parse_md(content) + _report(args.verbose > 1, f"- Found {len(code_blocks)} code blocks") + + # Execute code blocks and collect results + execution_results = [] + figure_counter = 0 + for i, block in enumerate(code_blocks): + _report(args.verbose > 1, f"- Executing block {i + 1}/{len(code_blocks)}") + figure_counter, result = _run_code(block["code"], args.outdir, stem, figure_counter) + execution_results.append(result) + _report(args.verbose > 0 and bool(result["error"]), f" - Warning: block {i + 1} had an error") + _report(args.verbose > 1 and bool(result["images"]), f" - Generated {len(result['images'])} image(s)") + + # Generate and save output + content = _generate_markdown(args, content, code_blocks, execution_results, args.outdir) + try: + with open(output_file, "w", encoding="utf-8") as f: + f.write(content) + _report(args.verbose > 1, f"- Output written to {output_file}") + _report(args.verbose > 1 and any(result["images"] for result in execution_results), f"- Images saved to {args.outdir}") + except Exception as e: + print(f"Error writing output file: {e}", file=sys.stderr) + sys.exit(1) + + +def _capture_plotly_show(fig, counter, result, output_dir, stem): + """Saves figures instead of displaying them.""" + # Save PNG + png_filename = f"{stem}_{counter}.png" + png_path = output_dir / png_filename + fig.write_image(png_path, width=800, height=600) + result["images"].append(png_filename) + + # Save HTML and get the content for embedding + html_filename = f"{stem}_{counter}.html" + html_path = output_dir / html_filename + fig.write_html(html_path, include_plotlyjs="cdn") + html_content = fig.to_html(include_plotlyjs="cdn", div_id=f"plotly-div-{counter}", full_html=False) + result["html_files"].append(html_filename) + result.setdefault("html_content", []).append(html_content) + + +def _generate_markdown(args, content, code_blocks, execution_results, output_dir): + """Generate the output markdown with embedded results.""" + lines = content.split("\n") + + # Sort code blocks by start line in reverse order for safe insertion + sorted_blocks = sorted( + enumerate(code_blocks), key=lambda x: x[1]["start_line"], reverse=True + ) + + # Process each code block and insert results + for block_idx, block in sorted_blocks: + result = execution_results[block_idx] + insert_lines = [] + + # Add output if there's stdout + if result["stdout"].strip(): + insert_lines.append("") + insert_lines.append("**Output:**") + insert_lines.append("```") + insert_lines.extend(result["stdout"].rstrip().split("\n")) + insert_lines.append("```") + + # Add error if there was one + if result["error"]: + insert_lines.append("") + insert_lines.append("**Error:**") + insert_lines.append("```") + insert_lines.extend(result["error"].rstrip().split("\n")) + insert_lines.append("```") + + # Add stderr if there's content + if result["stderr"].strip(): + insert_lines.append("") + insert_lines.append("**Warnings/Messages:**") + insert_lines.append("```") + insert_lines.extend(result["stderr"].rstrip().split("\n")) + insert_lines.append("```") + + # Add images + for image in result["images"]: + insert_lines.append("") + insert_lines.append(f"![Generated Plot](./{image})") + + # Embed HTML content for plotly figures + if args.inline: + for html_content in result.get("html_content", []): + insert_lines.append("") + insert_lines.append("**Interactive Plot:**") + insert_lines.append("") + insert_lines.extend(html_content.split("\n")) + + # Insert the results after the code block + if insert_lines: + # Insert after the closing ``` of the code block + insertion_point = block["end_line"] + 1 + lines[insertion_point:insertion_point] = insert_lines + + return "\n".join(lines) + + +def _parse_args(): + """Parse command-line arguments.""" + parser = argparse.ArgumentParser(description="Process Markdown files with code blocks") + parser.add_argument("inputs", nargs="+", help="Input .md files") + parser.add_argument("--inline", action="store_true", help="Inline HTML in .md") + parser.add_argument("--outdir", type=Path, help="Output directory") + parser.add_argument("--verbose", type=int, default=0, help="Integer verbosity level") + return parser.parse_args() + + +def _parse_md(content): + """Parse Markdown and extract Python code blocks.""" + lines = content.split("\n") + blocks = [] + current_block = None + in_code_block = False + in_region_block = False + + for i, line in enumerate(lines): + # Check for region start/end markers + if "