Merge pull request #1925 from tseaver/bigtable-v2-generate_v2_protos_… · googleapis/google-cloud-python@6d3ff2e · GitHub

googleapis / google-cloud-python Public

• Notifications You must be signed in to change notification settings
• Fork 1.8k
• Star 5.4k

• Code
• Issues 444
• Pull requests 99
• Discussions
• Actions
• Projects
• Security and quality
• Insights

Additional navigation options

Commit 6d3ff2e

Browse filesBrowse the repository at this point in the historyBrowse files

tseaver

authored

Merge pull request #1925 from tseaver/bigtable-v2-generate_v2_protos_redux

Regenerate from final Bigtable v2 v2 protos

2 parents 90cd2e7 + 24989d6 commit 6d3ff2e
Copy full SHA for 6d3ff2e

25 files changed

+1,454-750Lines changed: 1454 additions & 750 deletions

File tree

Expand file treeCollapse file tree

Open diff view settings

Filter options

• .gitignore
• Makefile.bigtable_v1
• Makefile.bigtable_v2
• Makefile.datastore
• gcloud/bigtable
  • _generated_v2
    • _bigtable.proto
    • _bigtable_instance_admin.proto
    • _bigtable_table_admin.proto
    • _common.proto
    • _data.proto
    • _instance.proto
    • _table.proto
    • bigtable_instance_admin_pb2.py
    • bigtable_pb2.py
    • bigtable_table_admin_pb2.py
    • operations_grpc_pb2.py
    • table_pb2.py
  • cluster.py
  • instance.py
  • table.py
  • test_cluster.py
  • test_instance.py
  • test_table.py
• scripts
  • make_datastore_grpc.py
  • make_operations_grpc.py
  • rewrite_imports.py

Expand file treeCollapse file tree

Open diff view settings

Collapse file

‎.gitignore‎

Copy file name to clipboardExpand all lines: .gitignore

+1Lines changed: 1 addition & 0 deletions

Original file line number	Diff line number	Diff line change
@@ -57,3 +57,4 @@ scripts/pylintrc_reduced
57	57	generated_python/
58	58	cloud-bigtable-client/
59	59	googleapis-pb/
	60	+ grpc_python_venv/

Collapse file

‎Makefile.bigtable_v1‎

Copy file name to clipboardExpand all lines: Makefile.bigtable_v1

+15-12Lines changed: 15 additions & 12 deletions

Original file line number	Diff line number	Diff line change
@@ -1,11 +1,11 @@
	1	+ GRPCIO_VIRTUALENV=$(shell pwd)/grpc_python_venv
1	2	GENERATED_DIR=$(shell pwd)/generated_python
2	3	GENERATED_SUBDIR=_generated
3	4	BIGTABLE_DIR=$(shell pwd)/gcloud/bigtable/$(GENERATED_SUBDIR)
4		- GRPC_PLUGIN=grpc_python_plugin
5		- PROTOC_CMD=protoc
6		- BIGTABLE_CHECKOUT_DIR=$(shell pwd)/cloud-bigtable-client
7		- BIGTABLE_PROTOS_DIR=$(BIGTABLE_CHECKOUT_DIR)/bigtable-protos/src/main/proto
	5	+ PROTOC_CMD=$(GRPCIO_VIRTUALENV)/bin/python -m grpc.tools.protoc
8	6	GOOGLEAPIS_PROTOS_DIR=$(shell pwd)/googleapis-pb
	7	+ BIGTABLE_CHECKOUT_DIR=$(shell pwd)/cloud-bigtable-client
	8	+ BIGTABLE_PROTOS_DIR=$(BIGTABLE_CHECKOUT_DIR)/bigtable-client-core-parent/bigtable-protos/src/main/proto
9	9
10	10	help:
11	11	@echo 'Makefile for gcloud-python Bigtable protos '
@@ -15,19 +15,22 @@ help:
15	15	@echo ' make clean Clean generated files '
16	16
17	17	generate:
	18	+ # Ensure we have a virtualenv w/ up-to-date grpcio/grpcio-tools
	19	+ [ -d $(GRPCIO_VIRTUALENV) ] || python2.7 -m virtualenv $(GRPCIO_VIRTUALENV)
	20	+ $(GRPCIO_VIRTUALENV)/bin/pip install --upgrade grpcio grpcio-tools
18	21	# Retrieve git repos that have our *.proto files.
19	22	[ -d $(BIGTABLE_CHECKOUT_DIR) ] || git clone https://github.com/GoogleCloudPlatform/cloud-bigtable-client --depth=1
20	23	cd $(BIGTABLE_CHECKOUT_DIR) && git pull origin master
21		- [ -d googleapis-pb ] || git clone https://github.com/google/googleapis googleapis-pb --depth=1
22		- cd googleapis-pb && git pull origin master
	24	+ [ -d $(GOOGLEAPIS_PROTOS_DIR) ] || git clone https://github.com/google/googleapis googleapis-pb --depth=1
	25	+ cd $(GOOGLEAPIS_PROTOS_DIR) && git pull origin master
23	26	# Make the directory where our *_pb2.py files will go.
24	27	mkdir -p $(GENERATED_DIR)
25	28	# Generate all *_pb2.py files that require gRPC.
26	29	$(PROTOC_CMD) \
27	30	--proto_path=$(BIGTABLE_PROTOS_DIR) \
	31	+ --proto_path=$(GOOGLEAPIS_PROTOS_DIR) \
28	32	--python_out=$(GENERATED_DIR) \
29		- --plugin=protoc-gen-grpc=$(GRPC_PLUGIN) \
30		- --grpc_out=$(GENERATED_DIR) \
	33	+ --grpc_python_out=$(GENERATED_DIR) \
31	34	$(BIGTABLE_PROTOS_DIR)/google/bigtable/v1/bigtable_service.proto \
32	35	$(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/cluster/v1/bigtable_cluster_service.proto \
33	36	$(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/table/v1/bigtable_table_service.proto
@@ -52,7 +55,7 @@ generate:
52	55	cp $(BIGTABLE_PROTOS_DIR)/google/bigtable/v1/*.proto $(BIGTABLE_DIR)
53	56	cp $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/cluster/v1/*.proto $(BIGTABLE_DIR)
54	57	cp $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/table/v1/*.proto $(BIGTABLE_DIR)
55		- cp $(BIGTABLE_PROTOS_DIR)/google/longrunning/operations.proto $(BIGTABLE_DIR)
	58	+ cp $(GOOGLEAPIS_PROTOS_DIR)/google/longrunning/operations.proto $(BIGTABLE_DIR)
56	59	# Rename all *.proto files in our library with an
57	60	# underscore and remove executable bit.
58	61	cd $(BIGTABLE_DIR) && \
@@ -63,8 +66,8 @@ generate:
63	66	# Separate the gRPC parts of the operations service from the
64	67	# non-gRPC parts so that the protos from `googleapis-common-protos`
65	68	# can be used without gRPC.
66		- PROTOC_CMD=$(PROTOC_CMD) GRPC_PLUGIN=$(GRPC_PLUGIN) \
67		- GENERATED_SUBDIR=$(GENERATED_SUBDIR) \
	69	+ GRPCIO_VIRTUALENV="$(GRPCIO_VIRTUALENV)" \
	70	+ GENERATED_SUBDIR=$(GENERATED_SUBDIR) \
68	71	python scripts/make_operations_grpc.py
69	72	# Rewrite the imports in the generated *_pb2.py files.
70	73	python scripts/rewrite_imports.py $(BIGTABLE_DIR)/*pb2.py
@@ -73,6 +76,6 @@ check_generate:
73	76	python scripts/check_generate.py
74	77
75	78	clean:
76		- rm -fr $(BIGTABLE_CHECKOUT_DIR) $(GOOGLEAPIS_PROTOS_DIR) $(GENERATED_DIR)
	79	+ rm -fr $(GRPCIO_VIRTUALENV) $(GOOGLEAPIS_PROTOS_DIR) $(GENERATED_DIR)
77	80
78	81	.PHONY: generate check_generate clean

Collapse file

‎Makefile.bigtable_v2‎

Copy file name to clipboard

+20-24Lines changed: 20 additions & 24 deletions

Original file line number	Diff line number	Diff line change
@@ -1,10 +1,8 @@
	1	+ GRPCIO_VIRTUALENV=$(shell pwd)/grpc_python_venv
1	2	GENERATED_DIR=$(shell pwd)/generated_python
2	3	GENERATED_SUBDIR=_generated_v2
3	4	BIGTABLE_DIR=$(shell pwd)/gcloud/bigtable/$(GENERATED_SUBDIR)
4		- GRPC_PLUGIN=grpc_python_plugin
5		- PROTOC_CMD=protoc
6		- BIGTABLE_CHECKOUT_DIR=$(shell pwd)/cloud-bigtable-client
7		- BIGTABLE_PROTOS_DIR=$(BIGTABLE_CHECKOUT_DIR)/bigtable-protos/src/main/proto
	5	+ PROTOC_CMD=$(GRPCIO_VIRTUALENV)/bin/python -m grpc.tools.protoc
8	6	GOOGLEAPIS_PROTOS_DIR=$(shell pwd)/googleapis-pb
9	7
10	8	help:
@@ -15,41 +13,39 @@ help:
15	13	@echo ' make clean Clean generated files '
16	14
17	15	generate:
	16	+ # Ensure we have a virtualenv w/ up-to-date grpcio/grpcio-tools
	17	+ [ -d $(GRPCIO_VIRTUALENV) ] || python2.7 -m virtualenv $(GRPCIO_VIRTUALENV)
	18	+ $(GRPCIO_VIRTUALENV)/bin/pip install --upgrade grpcio grpcio-tools
18	19	# Retrieve git repos that have our *.proto files.
19		- [ -d $(BIGTABLE_CHECKOUT_DIR) ] || git clone https://github.com/GoogleCloudPlatform/cloud-bigtable-client --depth=1
20		- cd $(BIGTABLE_CHECKOUT_DIR) && git pull origin master
21	20	[ -d googleapis-pb ] || git clone https://github.com/google/googleapis googleapis-pb --depth=1
22	21	cd googleapis-pb && git pull origin master
23	22	# Make the directory where our *_pb2.py files will go.
24	23	mkdir -p $(GENERATED_DIR)
25	24	# Generate all *_pb2.py files that require gRPC.
26	25	$(PROTOC_CMD) \
27		- --proto_path=$(BIGTABLE_PROTOS_DIR) \
	26	+ --proto_path=$(GOOGLEAPIS_PROTOS_DIR) \
28	27	--python_out=$(GENERATED_DIR) \
29		- --plugin=protoc-gen-grpc=$(GRPC_PLUGIN) \
30		- --grpc_out=$(GENERATED_DIR) \
31		- $(BIGTABLE_PROTOS_DIR)/google/bigtable/v2/bigtable.proto \
32		- $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/v2/bigtable_instance_admin.proto \
33		- $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/v2/bigtable_table_admin.proto
	28	+ --grpc_python_out=$(GENERATED_DIR) \
	29	+ $(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/v2/bigtable.proto \
	30	+ $(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/admin/v2/bigtable_instance_admin.proto \
	31	+ $(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/admin/v2/bigtable_table_admin.proto
34	32	# Generate all *_pb2.py files that do not require gRPC.
35	33	$(PROTOC_CMD) \
36		- --proto_path=$(BIGTABLE_PROTOS_DIR) \
37	34	--proto_path=$(GOOGLEAPIS_PROTOS_DIR) \
38	35	--python_out=$(GENERATED_DIR) \
39		- $(BIGTABLE_PROTOS_DIR)/google/bigtable/v2/data.proto \
40		- $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/v2/common.proto \
41		- $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/v2/instance.proto \
42		- $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/v2/table.proto \
	36	+ $(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/v2/data.proto \
	37	+ $(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/admin/v2/common.proto \
	38	+ $(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/admin/v2/instance.proto \
	39	+ $(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/admin/v2/table.proto \
43	40	# Move the newly generated *_pb2.py files into our library.
44	41	cp $(GENERATED_DIR)/google/bigtable/v2/* $(BIGTABLE_DIR)
45	42	cp $(GENERATED_DIR)/google/bigtable/admin/v2/* $(BIGTABLE_DIR)
46		- cp $(GENERATED_DIR)/google/bigtable/admin/v2/* $(BIGTABLE_DIR)
47	43	# Remove all existing *.proto files before we replace
48	44	rm -f $(BIGTABLE_DIR)/*.proto
49	45	# Copy over the *.proto files into our library.
50		- cp $(BIGTABLE_PROTOS_DIR)/google/bigtable/v2/*.proto $(BIGTABLE_DIR)
51		- cp $(BIGTABLE_PROTOS_DIR)/google/bigtable/admin/v2/*.proto $(BIGTABLE_DIR)
52		- cp $(BIGTABLE_PROTOS_DIR)/google/longrunning/operations.proto $(BIGTABLE_DIR)
	46	+ cp $(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/v2/*.proto $(BIGTABLE_DIR)
	47	+ cp $(GOOGLEAPIS_PROTOS_DIR)/google/bigtable/admin/v2/*.proto $(BIGTABLE_DIR)
	48	+ cp $(GOOGLEAPIS_PROTOS_DIR)/google/longrunning/operations.proto $(BIGTABLE_DIR)
53	49	# Rename all *.proto files in our library with an
54	50	# underscore and remove executable bit.
55	51	cd $(BIGTABLE_DIR) && \
@@ -60,8 +56,8 @@ generate:
60	56	# Separate the gRPC parts of the operations service from the
61	57	# non-gRPC parts so that the protos from `googleapis-common-protos`
62	58	# can be used without gRPC.
63		- PROTOC_CMD=$(PROTOC_CMD) GRPC_PLUGIN=$(GRPC_PLUGIN) \
64		- GENERATED_SUBDIR=$(GENERATED_SUBDIR) \
	59	+ GRPCIO_VIRTUALENV="$(GRPCIO_VIRTUALENV)" \
	60	+ GENERATED_SUBDIR=$(GENERATED_SUBDIR) \
65	61	python scripts/make_operations_grpc.py
66	62	# Rewrite the imports in the generated *_pb2.py files.
67	63	python scripts/rewrite_imports.py $(BIGTABLE_DIR)/*pb2.py
@@ -70,6 +66,6 @@ check_generate:
70	66	python scripts/check_generate.py
71	67
72	68	clean:
73		- rm -fr $(BIGTABLE_CHECKOUT_DIR) $(GOOGLEAPIS_PROTOS_DIR) $(GENERATED_DIR)
	69	+ rm -fr $(GRPCIO_VIRTUALENV) $(GOOGLEAPIS_PROTOS_DIR) $(GENERATED_DIR)
74	70
75	71	.PHONY: generate check_generate clean

Collapse file

‎Makefile.datastore‎

Copy file name to clipboardExpand all lines: Makefile.datastore

+7-3Lines changed: 7 additions & 3 deletions

Original file line number	Diff line number	Diff line change
@@ -1,7 +1,7 @@
	1	+ GRPCIO_VIRTUALENV=$(shell pwd)/grpc_python_venv
1	2	GENERATED_DIR=$(shell pwd)/generated_python
2	3	DATASTORE_DIR=$(shell pwd)/gcloud/datastore/_generated
3		- GRPC_PLUGIN=grpc_python_plugin
4		- PROTOC_CMD=protoc
	4	+ PROTOC_CMD=$(GRPCIO_VIRTUALENV)/bin/python -m grpc.tools.protoc
5	5	GOOGLEAPIS_PROTOS_DIR=$(shell pwd)/googleapis-pb
6	6
7	7	help:
@@ -12,6 +12,9 @@ help:
12	12	@echo ' make clean Clean generated files '
13	13
14	14	generate:
	15	+ # Ensure we have a virtualenv w/ up-to-date grpcio/grpcio-tools
	16	+ [ -d $(GRPCIO_VIRTUALENV) ] || python2.7 -m virtualenv $(GRPCIO_VIRTUALENV)
	17	+ $(GRPCIO_VIRTUALENV)/bin/pip install --upgrade grpcio grpcio-tools
15	18	# Retrieve git repos that have our *.proto files.
16	19	[ -d googleapis-pb ] || git clone https://github.com/google/googleapis googleapis-pb --depth=1
17	20	cd googleapis-pb && git pull origin master
@@ -39,7 +42,8 @@ generate:
39	42	done
40	43	# Separate the gRPC parts of the datastore service from the
41	44	# non-gRPC parts so that the protos can be used without gRPC.
42		- PROTOC_CMD=$(PROTOC_CMD) GRPC_PLUGIN=$(GRPC_PLUGIN) \
	45	+ GRPCIO_VIRTUALENV="$(GRPCIO_VIRTUALENV)" \
	46	+ GENERATED_SUBDIR=$(GENERATED_SUBDIR) \
43	47	python scripts/make_datastore_grpc.py
44	48	# Rewrite the imports in the generated *_pb2.py files.
45	49	python scripts/rewrite_imports.py $(DATASTORE_DIR)/*pb2.py

0 commit comments

Comments

0 (0)