|
1 | | -# Copyright 2018 Google LLC |
| 1 | +# Copyright 2022 Google LLC |
2 | 2 | # |
3 | 3 | # Licensed under the Apache License, Version 2.0 (the "License"); |
4 | 4 | # you may not use this file except in compliance with the License. |
|
12 | 12 | # See the License for the specific language governing permissions and |
13 | 13 | # limitations under the License. |
14 | 14 |
|
15 | | -"""This script is used to synthesize generated parts of this library.""" |
| 15 | +import json |
| 16 | +from pathlib import Path |
| 17 | +import shutil |
16 | 18 |
|
17 | 19 | import synthtool as s |
18 | | -from synthtool import gcp |
| 20 | +import synthtool.gcp as gcp |
19 | 21 | from synthtool.languages import python |
20 | 22 |
|
21 | | -common = gcp.CommonTemplates() |
| 23 | +# ---------------------------------------------------------------------------- |
| 24 | +# Copy the generated client from the owl-bot staging directory |
| 25 | +# ---------------------------------------------------------------------------- |
22 | 26 |
|
23 | | -default_version = "v1" |
| 27 | +clean_up_generated_samples = True |
| 28 | + |
| 29 | +# Load the default version defined in .repo-metadata.json. |
| 30 | +default_version = json.load(open(".repo-metadata.json", "rt")).get("default_version") |
24 | 31 |
|
25 | 32 | for library in s.get_staging_dirs(default_version): |
26 | | - # Work around gapic generator bug https://github.com/googleapis/gapic-generator-python/issues/902 |
27 | | - s.replace( |
28 | | - library / f"google/cloud/bigquery_storage_{library.name}/types/arrow.py", |
29 | | - r""". |
30 | | - Attributes:""", |
31 | | - r""".\n |
32 | | - Attributes:""", |
33 | | - ) |
| 33 | + if clean_up_generated_samples: |
| 34 | + shutil.rmtree("samples/generated_samples", ignore_errors=True) |
| 35 | + clean_up_generated_samples = False |
34 | 36 |
|
35 | 37 | # We don't want the generated client to be accessible through |
36 | 38 | # "google.cloud.bigquery_storage", replace it with the hand written client that |
|
57 | 59 | "", |
58 | 60 | ) |
59 | 61 |
|
60 | | - # We want types and __version__ to be accessible through the "main" library |
61 | | - # entry point. |
62 | | - s.replace( |
63 | | - library / "google/cloud/bigquery_storage/__init__.py", |
64 | | - f"from google\\.cloud\\.bigquery_storage_{library.name}\\.types\\.arrow import ArrowRecordBatch", |
65 | | - ( |
66 | | - f"from google.cloud.bigquery_storage_{library.name} import types\n" |
67 | | - f"from google.cloud.bigquery_storage_{library.name} import __version__\n" |
68 | | - "\\g<0>" |
69 | | - ), |
70 | | - ) |
71 | 62 | s.replace( |
72 | 63 | library / "google/cloud/bigquery_storage/__init__.py", |
73 | 64 | r"""["']ArrowRecordBatch["']""", |
74 | 65 | ('"__version__",\n' ' "types",\n' " \\g<0>"), |
75 | 66 | ) |
76 | 67 |
|
77 | | - # We want to expose all types through "google.cloud.bigquery_storage.types", |
78 | | - # not just the types generated for the BQ Storage library. For example, we also |
79 | | - # want to include common proto types such as Timestamp. |
| 68 | + # We want types to be accessible through the "main" library |
80 | 69 | s.replace( |
81 | 70 | library / "google/cloud/bigquery_storage/__init__.py", |
82 | | - r"import types", |
83 | | - "import gapic_types as types", |
| 71 | + f"from google\\.cloud\\.bigquery_storage_{library.name}\\.types\\.arrow import ArrowRecordBatch", |
| 72 | + ( |
| 73 | + f"from google.cloud.bigquery_storage_{library.name} import gapic_types as types\n" |
| 74 | + "\\g<0>" |
| 75 | + ), |
84 | 76 | ) |
85 | 77 |
|
86 | 78 | # The DataFormat enum is not exposed in bigquery_storage_v1/types, add it there. |
|
95 | 87 | '"DataFormat",\n \\g<0>', |
96 | 88 | ) |
97 | 89 |
|
98 | | - # The append_rows method doesn't contain keyword arguments that build request |
99 | | - # objects, so flattened tests are not needed and break with TypeError. |
100 | | - s.replace( |
101 | | - library |
102 | | - / f"tests/unit/gapic/bigquery_storage_{library.name}*/test_big_query_write.py", |
103 | | - r"(@[a-z.()\n]*\n)?(async )?" |
104 | | - r"def test_append_rows_flattened[_a-z]*\(\):\n" |
105 | | - r"( {4}.*|\n)+", |
106 | | - "\n", |
107 | | - ) |
108 | | - |
109 | 90 | s.move( |
110 | | - library, |
| 91 | + [library], |
111 | 92 | excludes=[ |
112 | | - "bigquery-storage-*-py.tar.gz", |
113 | | - "docs/conf.py", |
114 | | - "docs/index.rst", |
| 93 | + "setup.py", |
115 | 94 | f"google/cloud/bigquery_storage_{library.name}/__init__.py", |
116 | 95 | # v1beta2 was first generated after the microgenerator migration. |
117 | 96 | "scripts/fixup_bigquery_storage_v1beta2_keywords.py", |
118 | | - "README.rst", |
119 | | - "nox*.py", |
120 | | - "setup.py", |
121 | | - "setup.cfg", |
| 97 | + "**/gapic_version.py", |
| 98 | + "docs/index.rst", |
| 99 | + "testing/constraints-3.7.txt", |
122 | 100 | ], |
123 | 101 | ) |
124 | | - |
125 | 102 | s.remove_staging_dirs() |
126 | 103 |
|
127 | 104 | # ---------------------------------------------------------------------------- |
128 | 105 | # Add templated files |
129 | 106 | # ---------------------------------------------------------------------------- |
| 107 | + |
130 | 108 | extras = ["fastavro", "pandas", "pyarrow"] |
131 | 109 | unit_test_extras = ["tests"] + extras |
132 | 110 |
|
133 | | -templated_files = common.py_library( |
| 111 | +templated_files = gcp.CommonTemplates().py_library( |
| 112 | + cov_level=98, |
134 | 113 | microgenerator=True, |
135 | | - samples=True, |
136 | 114 | unit_test_extras=unit_test_extras, |
137 | 115 | system_test_extras=extras, |
138 | 116 | system_test_external_dependencies=["google-cloud-bigquery"], |
139 | | - cov_level=98, |
| 117 | + versions=gcp.common.detect_versions(path="./google", default_first=True), |
140 | 118 | ) |
141 | 119 | s.move( |
142 | | - templated_files, excludes=[".coveragerc"] |
143 | | -) # microgenerator has a good .coveragerc file |
144 | | - |
145 | | -python.configure_previous_major_version_branches() |
146 | | - |
147 | | -# ---------------------------------------------------------------------------- |
148 | | -# Samples templates |
149 | | -# ---------------------------------------------------------------------------- |
| 120 | + templated_files, |
| 121 | + excludes=[".coveragerc", ".github/release-please.yml", "docs/index.rst"], |
| 122 | +) |
150 | 123 |
|
151 | 124 | python.py_samples(skip_readmes=True) |
152 | 125 |
|
153 | | -s.shell.run(["nox", "-s", "blacken"], hide_output=False) |
| 126 | +# run format session for all directories which have a noxfile |
| 127 | +for noxfile in Path(".").glob("**/noxfile.py"): |
| 128 | + s.shell.run(["nox", "-s", "blacken"], cwd=noxfile.parent, hide_output=False) |
0 commit comments