FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

test(spanner): additional logging on UpdateDatabaseDdl() failures (#9… · QPC-github/google-cloud-cpp@dad8a9b · GitHub

/ google-cloud-cpp Public
forked from googleapis/google-cloud-cpp

Commit dad8a9b

Browse files
devbww
authored
test(spanner): additional logging on UpdateDatabaseDdl() failures (googleapis#9483)
When we see an UpdateDatabaseDdl() failure in the Spanner samples, fetch the database DDL, and list all completed and pending operations, so that we can use the RPC log to summarize the service state. Then abort the process so the database will not be dropped. Part of googleapis#4758. The extra code within the samples includes `//!` comments so that it will be omitted from the extracted snippets.
1 parent 60354fb commit dad8a9b

6 files changed

Lines changed: 133 additions & 0 deletions

File tree

‎google/cloud/spanner/CMakeLists.txt‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -270,6 +270,8 @@ function (spanner_client_define_tests)
270270
testing/cleanup_stale_instances.h
271271
testing/database_integration_test.cc
272272
testing/database_integration_test.h
273+
testing/debug_log.cc
274+
testing/debug_log.h
273275
testing/fake_clock.h
274276
testing/instance_location.cc
275277
testing/instance_location.h

‎google/cloud/spanner/samples/postgresql_samples.cc‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
#include "google/cloud/spanner/database.h"
1919
#include "google/cloud/spanner/instance.h"
2020
#include "google/cloud/spanner/numeric.h"
21+
#include "google/cloud/spanner/testing/debug_log.h" // TODO(#4758): remove
2122
#include "google/cloud/spanner/testing/pick_random_instance.h"
2223
#include "google/cloud/spanner/testing/random_database_name.h"
2324
#include "google/cloud/internal/getenv.h"
@@ -72,6 +73,8 @@ void AddColumn(google::cloud::spanner_admin::DatabaseAdminClient client,
7273
};
7374
auto metadata =
7475
client.UpdateDatabaseDdl(database.FullName(), statements).get();
76+
google::cloud::spanner_testing::LogUpdateDatabaseDdl( //! TODO(#4758)
77+
client, database, metadata.status()); //! TODO(#4758)
7578
if (!metadata) throw std::runtime_error(metadata.status().message());
7679
std::cout << "Column added.\nNew DDL:\n" << metadata->DebugString();
7780
}
@@ -249,6 +252,8 @@ void CaseSensitivity(
249252
};
250253
auto metadata =
251254
admin_client.UpdateDatabaseDdl(database.FullName(), statements).get();
255+
google::cloud::spanner_testing::LogUpdateDatabaseDdl( //! TODO(#4758)
256+
admin_client, database, metadata.status()); //! TODO(#4758)
252257
if (!metadata) throw std::runtime_error(metadata.status().message());
253258
std::cout << "Tables created.\nNew DDL:\n" << metadata->DebugString();
254259

@@ -486,6 +491,8 @@ void InterleavedTable(google::cloud::spanner_admin::DatabaseAdminClient client,
486491
};
487492
auto metadata =
488493
client.UpdateDatabaseDdl(database.FullName(), statements).get();
494+
google::cloud::spanner_testing::LogUpdateDatabaseDdl( //! TODO(#4758)
495+
client, database, metadata.status()); //! TODO(#4758)
489496
if (!metadata) throw std::runtime_error(metadata.status().message());
490497
std::cout << "Tables created.\nNew DDL:\n" << metadata->DebugString();
491498
}
@@ -504,6 +511,8 @@ void CreateStoringIndex(
504511
};
505512
auto metadata =
506513
client.UpdateDatabaseDdl(database.FullName(), statements).get();
514+
google::cloud::spanner_testing::LogUpdateDatabaseDdl( //! TODO(#4758)
515+
client, database, metadata.status()); //! TODO(#4758)
507516
if (!metadata) throw std::runtime_error(metadata.status().message());
508517
std::cout << "Index added.\nNew DDL:\n" << metadata->DebugString();
509518
}
@@ -526,6 +535,8 @@ void InformationSchema(
526535
};
527536
auto metadata =
528537
admin_client.UpdateDatabaseDdl(database.FullName(), statements).get();
538+
google::cloud::spanner_testing::LogUpdateDatabaseDdl( //! TODO(#4758)
539+
admin_client, database, metadata.status()); //! TODO(#4758)
529540
if (!metadata) throw std::runtime_error(metadata.status().message());
530541
std::cout << "Table created.\nNew DDL:\n" << metadata->DebugString();
531542

@@ -724,6 +735,8 @@ void CreateTables(google::cloud::spanner_admin::DatabaseAdminClient client,
724735
};
725736
auto metadata =
726737
client.UpdateDatabaseDdl(database.FullName(), statements).get();
738+
google::cloud::spanner_testing::LogUpdateDatabaseDdl( //! TODO(#4758)
739+
client, database, metadata.status()); //! TODO(#4758)
727740
if (!metadata) throw std::runtime_error(metadata.status().message());
728741
std::cout << "Tables created.\nNew DDL:\n" << metadata->DebugString();
729742
}
@@ -740,6 +753,8 @@ void DropTables(google::cloud::spanner_admin::DatabaseAdminClient client,
740753
};
741754
auto metadata =
742755
client.UpdateDatabaseDdl(database.FullName(), statements).get();
756+
google::cloud::spanner_testing::LogUpdateDatabaseDdl( //! TODO(#4758)
757+
client, database, metadata.status()); //! TODO(#4758)
743758
if (!metadata) throw std::runtime_error(metadata.status().message());
744759
std::cout << "Tables dropped.\nNew DDL:\n" << metadata->DebugString();
745760
}

‎google/cloud/spanner/samples/samples.cc‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
#include "google/cloud/spanner/connection_options.h"
2525
#include "google/cloud/spanner/create_instance_request_builder.h"
2626
#include "google/cloud/spanner/row.h"
27+
#include "google/cloud/spanner/testing/debug_log.h" // TODO(#4758): remove
2728
#include "google/cloud/spanner/testing/instance_location.h"
2829
#include "google/cloud/spanner/testing/pick_instance_config.h"
2930
#include "google/cloud/spanner/testing/pick_random_instance.h"
@@ -556,6 +557,8 @@ void UpdateDatabaseWithDefaultLeader(
556557
auto metadata =
557558
client.UpdateDatabaseDdl(database.FullName(), std::move(statements))
558559
.get();
560+
google::cloud::spanner_testing::LogUpdateDatabaseDdl( //! TODO(#4758)
561+
client, database, metadata.status()); //! TODO(#4758)
559562
if (!metadata) throw std::runtime_error(metadata.status().message());
560563
std::cout << "`default_leader` altered, new DDL metadata:\n"
561564
<< metadata->DebugString();
@@ -584,6 +587,8 @@ void CreateTableWithDatatypes(
584587
(allow_commit_timestamp=true)
585588
) PRIMARY KEY (VenueId))"""})
586589
.get();
590+
google::cloud::spanner_testing::LogUpdateDatabaseDdl( //! TODO(#4758)
591+
client, database, metadata.status()); //! TODO(#4758)
587592
if (!metadata) throw std::runtime_error(metadata.status().message());
588593
std::cout << "`Venues` table created, new DDL:\n" << metadata->DebugString();
589594
}
@@ -608,6 +613,8 @@ void CreateTableWithTimestamp(
608613
) PRIMARY KEY (SingerId, VenueId, EventDate),
609614
INTERLEAVE IN PARENT Singers ON DELETE CASCADE)"""})
610615
.get();
616+
google::cloud::spanner_testing::LogUpdateDatabaseDdl( //! TODO(#4758)
617+
client, database, metadata.status()); //! TODO(#4758)
611618
if (!metadata) throw std::runtime_error(metadata.status().message());
612619
std::cout << "`Performances` table created, new DDL:\n"
613620
<< metadata->DebugString();
@@ -626,6 +633,8 @@ void AddIndex(google::cloud::spanner_admin::DatabaseAdminClient client,
626633
database.FullName(),
627634
{"CREATE INDEX AlbumsByAlbumTitle ON Albums(AlbumTitle)"})
628635
.get();
636+
google::cloud::spanner_testing::LogUpdateDatabaseDdl( //! TODO(#4758)
637+
client, database, metadata.status()); //! TODO(#4758)
629638
if (!metadata) throw std::runtime_error(metadata.status().message());
630639
std::cout << "`AlbumsByAlbumTitle` Index successfully added, new DDL:\n"
631640
<< metadata->DebugString();
@@ -669,6 +678,8 @@ void AddColumn(google::cloud::spanner_admin::DatabaseAdminClient client,
669678
database.FullName(),
670679
{"ALTER TABLE Albums ADD COLUMN MarketingBudget INT64"})
671680
.get();
681+
google::cloud::spanner_testing::LogUpdateDatabaseDdl( //! TODO(#4758)
682+
client, database, metadata.status()); //! TODO(#4758)
672683
if (!metadata) throw std::runtime_error(metadata.status().message());
673684
std::cout << "Added MarketingBudget column\n";
674685
}
@@ -688,6 +699,8 @@ void AddTimestampColumn(
688699
{"ALTER TABLE Albums ADD COLUMN LastUpdateTime TIMESTAMP "
689700
"OPTIONS (allow_commit_timestamp=true)"})
690701
.get();
702+
google::cloud::spanner_testing::LogUpdateDatabaseDdl( //! TODO(#4758)
703+
client, database, metadata.status()); //! TODO(#4758)
691704
if (!metadata) throw std::runtime_error(metadata.status().message());
692705
std::cout << "Added LastUpdateTime column\n";
693706
}
@@ -705,6 +718,8 @@ void AddStoringIndex(google::cloud::spanner_admin::DatabaseAdminClient client,
705718
CREATE INDEX AlbumsByAlbumTitle2 ON Albums(AlbumTitle)
706719
STORING (MarketingBudget))"""})
707720
.get();
721+
google::cloud::spanner_testing::LogUpdateDatabaseDdl( //! TODO(#4758)
722+
client, database, metadata.status()); //! TODO(#4758)
708723
if (!metadata) throw std::runtime_error(metadata.status().message());
709724
std::cout << "`AlbumsByAlbumTitle2` Index successfully added, new DDL:\n"
710725
<< metadata->DebugString();
@@ -1981,6 +1996,8 @@ void AddJsonColumn(google::cloud::spanner_admin::DatabaseAdminClient client,
19811996
.UpdateDatabaseDdl(database.FullName(), {R"""(
19821997
ALTER TABLE Venues ADD COLUMN VenueDetails JSON)"""})
19831998
.get();
1999+
google::cloud::spanner_testing::LogUpdateDatabaseDdl( //! TODO(#4758)
2000+
client, database, metadata.status()); //! TODO(#4758)
19842001
if (!metadata) throw std::runtime_error(metadata.status().message());
19852002
std::cout << "`Venues` table altered, new DDL:\n" << metadata->DebugString();
19862003
}
@@ -2058,6 +2075,8 @@ void AddNumericColumn(google::cloud::spanner_admin::DatabaseAdminClient client,
20582075
.UpdateDatabaseDdl(database.FullName(), {R"""(
20592076
ALTER TABLE Venues ADD COLUMN Revenue NUMERIC)"""})
20602077
.get();
2078+
google::cloud::spanner_testing::LogUpdateDatabaseDdl( //! TODO(#4758)
2079+
client, database, metadata.status()); //! TODO(#4758)
20612080
if (!metadata) throw std::runtime_error(metadata.status().message());
20622081
std::cout << "`Venues` table altered, new DDL:\n" << metadata->DebugString();
20632082
}

‎google/cloud/spanner/spanner_client_testing.bzl‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ spanner_client_testing_hdrs = [
2020
"testing/cleanup_stale_databases.h",
2121
"testing/cleanup_stale_instances.h",
2222
"testing/database_integration_test.h",
23+
"testing/debug_log.h",
2324
"testing/fake_clock.h",
2425
"testing/instance_location.h",
2526
"testing/matchers.h",
@@ -39,6 +40,7 @@ spanner_client_testing_srcs = [
3940
"testing/cleanup_stale_databases.cc",
4041
"testing/cleanup_stale_instances.cc",
4142
"testing/database_integration_test.cc",
43+
"testing/debug_log.cc",
4244
"testing/instance_location.cc",
4345
"testing/pick_instance_config.cc",
4446
"testing/pick_random_instance.cc",

‎google/cloud/spanner/testing/debug_log.cc‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
// Copyright 2022 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#include "google/cloud/spanner/testing/debug_log.h"
16+
#include "google/cloud/log.h"
17+
18+
namespace google {
19+
namespace cloud {
20+
namespace spanner_testing {
21+
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN
22+
23+
void LogUpdateDatabaseDdl(
24+
google::cloud::spanner_admin::DatabaseAdminClient client,
25+
google::cloud::spanner::Database const& database,
26+
google::cloud::Status const& status) {
27+
if (status.ok()) return;
28+
29+
GCP_LOG(DEBUG) << std::string(26, '=') << " UpdateDatabaseDdl() "
30+
<< std::string(26, '=');
31+
32+
// Call GetDatabaseDdl() and ListDatabaseOperations() so that their
33+
// RPC traces can give us information about the state of the database.
34+
auto ddl = client.GetDatabaseDdl(database.FullName());
35+
google::spanner::admin::database::v1::ListDatabaseOperationsRequest request;
36+
request.set_parent(database.instance().FullName());
37+
request.set_filter("name:" + database.FullName() + "/");
38+
for (auto const& operation : client.ListDatabaseOperations(request)) {
39+
static_cast<void>(operation);
40+
}
41+
42+
GCP_LOG(DEBUG) << std::string(73, '=');
43+
44+
// Terminate the process abruptly (after flushing the client log), without
45+
// dropping the database. This means we'll have a chance to examine it and
46+
// its server-side logs until they are garbage-collected.
47+
GCP_LOG(FATAL) << "Terminating after UpdateDatabaseDdl() failure";
48+
}
49+
50+
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
51+
} // namespace spanner_testing
52+
} // namespace cloud
53+
} // namespace google

‎google/cloud/spanner/testing/debug_log.h‎

Lines changed: 42 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
// Copyright 2022 Google LLC
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// https://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
15+
#ifndef GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_SPANNER_TESTING_DEBUG_LOG_H
16+
#define GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_SPANNER_TESTING_DEBUG_LOG_H
17+
18+
#include "google/cloud/spanner/admin/database_admin_client.h"
19+
#include "google/cloud/spanner/database.h"
20+
#include "google/cloud/spanner/version.h"
21+
#include "google/cloud/status.h"
22+
23+
namespace google {
24+
namespace cloud {
25+
namespace spanner_testing {
26+
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_BEGIN
27+
28+
/**
29+
* Log everything we can after an UpdateDatabaseDdl() failure so that
30+
* we might have a chance to debug the apparent replays behind #4758.
31+
*/
32+
void LogUpdateDatabaseDdl(
33+
google::cloud::spanner_admin::DatabaseAdminClient client,
34+
google::cloud::spanner::Database const& database,
35+
google::cloud::Status const& status);
36+
37+
GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END
38+
} // namespace spanner_testing
39+
} // namespace cloud
40+
} // namespace google
41+
42+
#endif // GOOGLE_CLOUD_CPP_GOOGLE_CLOUD_SPANNER_TESTING_DEBUG_LOG_H

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL