FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

test(fix): do not skip backup ITs (#498) · benjaminp/google-cloud-java@548dfd5 · GitHub

/ google-cloud-java Public
forked from googleapis/google-cloud-java

Commit 548dfd5

Browse files
kolea2
authored
test(fix): do not skip backup ITs (googleapis#498)
* test(fix): do not skip backup ITs * formatting * create new cluster for test * formatting * skip for emulator profile * skip for emulator profile * use test env * instance name * lint
1 parent 63d3255 commit 548dfd5

1 file changed

Lines changed: 81 additions & 132 deletions

File tree

  • java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/it

‎java-bigtable/google-cloud-bigtable/src/test/java/com/google/cloud/bigtable/admin/v2/it/BigtableBackupIT.java‎

Lines changed: 81 additions & 132 deletions
Original file line numberDiff line numberDiff line change
@@ -17,27 +17,31 @@
1717

1818
import static com.google.common.truth.Truth.assertThat;
1919
import static com.google.common.truth.Truth.assertWithMessage;
20+
import static com.google.common.truth.TruthJUnit.assume;
2021
import static io.grpc.Status.Code.NOT_FOUND;
2122
import static org.junit.Assert.fail;
2223

2324
import com.google.api.core.ApiFuture;
2425
import com.google.api.core.ApiFutures;
2526
import com.google.api.gax.rpc.ApiException;
2627
import com.google.cloud.Policy;
28+
import com.google.cloud.bigtable.admin.v2.BigtableInstanceAdminClient;
2729
import com.google.cloud.bigtable.admin.v2.BigtableTableAdminClient;
28-
import com.google.cloud.bigtable.admin.v2.BigtableTableAdminSettings;
2930
import com.google.cloud.bigtable.admin.v2.models.Backup;
3031
import com.google.cloud.bigtable.admin.v2.models.CreateBackupRequest;
32+
import com.google.cloud.bigtable.admin.v2.models.CreateInstanceRequest;
3133
import com.google.cloud.bigtable.admin.v2.models.CreateTableRequest;
34+
import com.google.cloud.bigtable.admin.v2.models.Instance.Type;
3235
import com.google.cloud.bigtable.admin.v2.models.RestoreTableRequest;
3336
import com.google.cloud.bigtable.admin.v2.models.RestoredTableResult;
37+
import com.google.cloud.bigtable.admin.v2.models.StorageType;
3438
import com.google.cloud.bigtable.admin.v2.models.Table;
3539
import com.google.cloud.bigtable.admin.v2.models.UpdateBackupRequest;
3640
import com.google.cloud.bigtable.data.v2.BigtableDataClient;
37-
import com.google.cloud.bigtable.data.v2.BigtableDataSettings;
3841
import com.google.cloud.bigtable.data.v2.models.RowMutation;
39-
import com.google.common.base.Joiner;
40-
import com.google.common.base.MoreObjects;
42+
import com.google.cloud.bigtable.test_helpers.env.AbstractTestEnv;
43+
import com.google.cloud.bigtable.test_helpers.env.EmulatorEnv;
44+
import com.google.cloud.bigtable.test_helpers.env.TestEnvRule;
4145
import com.google.common.collect.Lists;
4246
import com.google.protobuf.Timestamp;
4347
import io.grpc.StatusRuntimeException;
@@ -56,25 +60,19 @@
5660

5761
@RunWith(JUnit4.class)
5862
public class BigtableBackupIT {
63+
@ClassRule public static TestEnvRule testEnvRule = new TestEnvRule();
64+
5965
private static final Logger LOGGER = Logger.getLogger(BigtableBackupIT.class.getName());
6066

61-
private static final String PROJECT_PROPERTY_NAME = "bigtable.project";
62-
private static final String INSTANCE_PROPERTY_NAME = "bigtable.instance";
63-
private static final String CLUSTER_PROPERTY_NAME = "bigtable.cluster";
64-
private static final String ADMIN_ENDPOINT_PROPERTY_NAME = "bigtable.adminendpoint";
65-
private static final String DATA_ENDPOINT_PROPERTY_NAME = "bigtable.dataendpoint";
66-
private static final String TABLE_SIZE_PROPERTY_NAME = "bigtable.tablesizekb";
6767
private static final int[] BACKOFF_DURATION = {2, 4, 8, 16, 32, 64, 128, 256, 512, 1024};
6868

6969
private static final String TEST_TABLE_SUFFIX = "test-table-for-backup-it";
7070
private static final String TEST_BACKUP_SUFFIX = "test-backup-for-backup-it";
7171

72-
private static final int DAYS_IN_SECONDS = 24 * 60 * 60;
73-
7472
private static BigtableTableAdminClient tableAdmin;
73+
private static BigtableInstanceAdminClient instanceAdmin;
7574
private static BigtableDataClient dataClient;
7675

77-
private static String targetProject;
7876
private static String targetInstance;
7977
private static String targetCluster;
8078
private static Table testTable;
@@ -83,90 +81,48 @@ public class BigtableBackupIT {
8381
@BeforeClass
8482
public static void createClient()
8583
throws IOException, InterruptedException, ExecutionException, TimeoutException {
86-
List<String> missingProperties = Lists.newArrayList();
87-
88-
targetProject = System.getProperty(PROJECT_PROPERTY_NAME);
89-
if (targetProject == null) {
90-
missingProperties.add(PROJECT_PROPERTY_NAME);
91-
}
92-
93-
targetInstance = System.getProperty(INSTANCE_PROPERTY_NAME);
94-
if (targetInstance == null) {
95-
missingProperties.add(INSTANCE_PROPERTY_NAME);
96-
}
84+
assume()
85+
.withMessage("BigtableInstanceAdminClient is not supported on Emulator")
86+
.that(testEnvRule.env())
87+
.isNotInstanceOf(EmulatorEnv.class);
9788

98-
targetCluster = System.getProperty(CLUSTER_PROPERTY_NAME);
99-
if (targetCluster == null) {
100-
missingProperties.add(CLUSTER_PROPERTY_NAME);
101-
}
89+
instanceAdmin = testEnvRule.env().getInstanceAdminClient();
10290

103-
String adminApiEndpoint = System.getProperty(ADMIN_ENDPOINT_PROPERTY_NAME);
104-
if (adminApiEndpoint == null) {
105-
adminApiEndpoint = "bigtableadmin.googleapis.com:443";
106-
}
91+
targetCluster = AbstractTestEnv.TEST_CLUSTER_PREFIX + Instant.now().getEpochSecond();
92+
targetInstance =
93+
AbstractTestEnv.TEST_INSTANCE_PREFIX + "backup-" + Instant.now().getEpochSecond();
10794

108-
int tableSize = MoreObjects.firstNonNull(Integer.getInteger(TABLE_SIZE_PROPERTY_NAME), 1);
109-
if (!missingProperties.isEmpty()) {
110-
LOGGER.warning("Missing properties: " + Joiner.on(",").join(missingProperties));
111-
return;
112-
}
95+
instanceAdmin.createInstance(
96+
CreateInstanceRequest.of(targetInstance)
97+
.addCluster(targetCluster, testEnvRule.env().getPrimaryZone(), 3, StorageType.SSD)
98+
.setDisplayName("backups-test-instance")
99+
.addLabel("state", "readytodelete")
100+
.setType(Type.PRODUCTION));
113101

114102
// Setup a prefix to avoid collisions between concurrent test runs
115103
prefix = String.format("020%d", System.currentTimeMillis());
116104

117-
BigtableTableAdminSettings.Builder settings =
118-
BigtableTableAdminSettings.newBuilder()
119-
.setInstanceId(targetInstance)
120-
.setProjectId(targetProject);
121-
settings.stubSettings().setEndpoint(adminApiEndpoint);
122-
tableAdmin = BigtableTableAdminClient.create(settings.build());
105+
tableAdmin = BigtableTableAdminClient.create(testEnvRule.env().getProjectId(), targetInstance);
123106

124107
testTable =
125108
tableAdmin.createTable(
126109
CreateTableRequest.of(generateId(TEST_TABLE_SUFFIX)).addFamily("cf1"));
127110

128111
// Populate test data.
129-
if (tableSize > 0) {
130-
String dataApiEndpoint = System.getProperty(DATA_ENDPOINT_PROPERTY_NAME);
131-
if (dataApiEndpoint == null) {
132-
dataApiEndpoint = "bigtable.googleapis.com:443";
133-
}
134-
BigtableDataSettings.Builder dataSettings =
135-
BigtableDataSettings.newBuilder()
136-
.setInstanceId(targetInstance)
137-
.setProjectId(targetProject);
138-
dataSettings.stubSettings().setEndpoint(dataApiEndpoint);
139-
dataClient = BigtableDataClient.create(dataSettings.build());
140-
byte[] rowBytes = new byte[1024];
141-
Random random = new Random();
142-
random.nextBytes(rowBytes);
143-
144-
List<ApiFuture<?>> futures = Lists.newArrayList();
145-
for (int i = 0; i < tableSize; i++) {
146-
ApiFuture<Void> future =
147-
dataClient.mutateRowAsync(
148-
RowMutation.create(testTable.getId(), "test-row-" + i)
149-
.setCell("cf1", "", rowBytes.toString()));
150-
futures.add(future);
151-
}
152-
ApiFutures.allAsList(futures).get(3, TimeUnit.MINUTES);
153-
}
154-
155-
// Cleanup old backups and tables, under normal circumstances this will do nothing
156-
String stalePrefix =
157-
String.format("020%d", System.currentTimeMillis() - TimeUnit.HOURS.toMillis(2));
158-
for (String backupId : tableAdmin.listBackups(targetCluster)) {
159-
if (backupId.endsWith(TEST_BACKUP_SUFFIX) && stalePrefix.compareTo(backupId) > 0) {
160-
LOGGER.info("Deleting stale backup: " + backupId);
161-
tableAdmin.deleteBackup(targetCluster, backupId);
162-
}
163-
}
164-
for (String tableId : tableAdmin.listTables()) {
165-
if (tableId.endsWith("TEST_TABLE_SUFFIX") && stalePrefix.compareTo(tableId) > 0) {
166-
LOGGER.info("Deleting stale backup: " + tableId);
167-
tableAdmin.deleteTable(tableId);
168-
}
112+
dataClient = BigtableDataClient.create(testEnvRule.env().getProjectId(), targetInstance);
113+
byte[] rowBytes = new byte[1024];
114+
Random random = new Random();
115+
random.nextBytes(rowBytes);
116+
117+
List<ApiFuture<?>> futures = Lists.newArrayList();
118+
for (int i = 0; i < 10; i++) {
119+
ApiFuture<Void> future =
120+
dataClient.mutateRowAsync(
121+
RowMutation.create(testTable.getId(), "test-row-" + i)
122+
.setCell("cf1", "", rowBytes.toString()));
123+
futures.add(future);
169124
}
125+
ApiFutures.allAsList(futures).get(3, TimeUnit.MINUTES);
170126
}
171127

172128
@AfterClass
@@ -179,6 +135,10 @@ public static void closeClient() {
179135
}
180136
}
181137

138+
if (targetInstance != null) {
139+
instanceAdmin.deleteInstance(targetInstance);
140+
}
141+
182142
if (tableAdmin != null) {
183143
tableAdmin.close();
184144
}
@@ -247,8 +207,8 @@ public void listBackupTest() throws InterruptedException {
247207
String backupId2 = generateId("list-2-" + TEST_BACKUP_SUFFIX);
248208

249209
try {
250-
createBackupAndWait(backupId1);
251-
createBackupAndWait(backupId2);
210+
tableAdmin.createBackup(createBackupRequest(backupId1));
211+
tableAdmin.createBackup(createBackupRequest(backupId2));
252212

253213
List<String> response = tableAdmin.listBackups(targetCluster);
254214
// Concurrent tests running may cause flakiness. Use containsAtLeast instead of
@@ -265,7 +225,7 @@ public void listBackupTest() throws InterruptedException {
265225
@Test
266226
public void updateBackupTest() throws InterruptedException {
267227
String backupId = generateId("update-" + TEST_BACKUP_SUFFIX);
268-
createBackupAndWait(backupId);
228+
tableAdmin.createBackup(createBackupRequest(backupId));
269229

270230
Instant expireTime = Instant.now().plus(Duration.ofDays(20));
271231
UpdateBackupRequest req =
@@ -282,7 +242,7 @@ public void updateBackupTest() throws InterruptedException {
282242
public void deleteBackupTest() throws InterruptedException {
283243
String backupId = generateId("delete-" + TEST_BACKUP_SUFFIX);
284244

285-
createBackupAndWait(backupId);
245+
tableAdmin.createBackup(createBackupRequest(backupId));
286246
tableAdmin.deleteBackup(targetCluster, backupId);
287247

288248
try {
@@ -307,7 +267,7 @@ public void deleteBackupTest() throws InterruptedException {
307267
public void restoreTableTest() throws InterruptedException, ExecutionException {
308268
String backupId = generateId("restore-" + TEST_BACKUP_SUFFIX);
309269
String tableId = generateId("restored-table");
310-
createBackupAndWait(backupId);
270+
tableAdmin.createBackup(createBackupRequest(backupId));
311271

312272
// Wait 2 minutes so that the RestoreTable API will trigger an optimize restored
313273
// table operation.
@@ -320,13 +280,15 @@ public void restoreTableTest() throws InterruptedException, ExecutionException {
320280
.that(result.getTable().getId())
321281
.isEqualTo(tableId);
322282

323-
// The assertion might be missing if the test is running against a HDD cluster or an
324-
// optimization is not necessary.
325-
assertWithMessage("Empty OptimizeRestoredTable token")
326-
.that(result.getOptimizeRestoredTableOperationToken())
327-
.isNotNull();
328-
tableAdmin.awaitOptimizeRestoredTable(result.getOptimizeRestoredTableOperationToken());
329-
tableAdmin.getTable(tableId);
283+
if (result.getOptimizeRestoredTableOperationToken() != null) {
284+
// The assertion might be missing if the test is running against a HDD cluster or an
285+
// optimization is not necessary.
286+
tableAdmin.awaitOptimizeRestoredTable(result.getOptimizeRestoredTableOperationToken());
287+
Table restoredTable = tableAdmin.getTable(tableId);
288+
assertWithMessage("Incorrect restored table id")
289+
.that(restoredTable.getId())
290+
.isEqualTo(tableId);
291+
}
330292
} finally {
331293
tableAdmin.deleteBackup(targetCluster, backupId);
332294
tableAdmin.deleteTable(tableId);
@@ -336,28 +298,33 @@ public void restoreTableTest() throws InterruptedException, ExecutionException {
336298
@Test
337299
public void backupIamTest() throws InterruptedException {
338300
String backupId = generateId("iam-" + TEST_BACKUP_SUFFIX);
339-
createBackupAndWait(backupId);
340301

341-
Policy policy = tableAdmin.getBackupIamPolicy(targetCluster, backupId);
342-
assertThat(policy).isNotNull();
343-
344-
Exception actualEx = null;
345302
try {
346-
assertThat(tableAdmin.setBackupIamPolicy(targetCluster, backupId, policy)).isNotNull();
347-
} catch (Exception iamException) {
348-
actualEx = iamException;
303+
tableAdmin.createBackup(createBackupRequest(backupId));
304+
305+
Policy policy = tableAdmin.getBackupIamPolicy(targetCluster, backupId);
306+
assertThat(policy).isNotNull();
307+
308+
Exception actualEx = null;
309+
try {
310+
assertThat(tableAdmin.setBackupIamPolicy(targetCluster, backupId, policy)).isNotNull();
311+
} catch (Exception iamException) {
312+
actualEx = iamException;
313+
}
314+
assertThat(actualEx).isNull();
315+
316+
List<String> permissions =
317+
tableAdmin.testBackupIamPermission(
318+
targetCluster,
319+
backupId,
320+
"bigtable.backups.get",
321+
"bigtable.backups.delete",
322+
"bigtable.backups.update",
323+
"bigtable.backups.restore");
324+
assertThat(permissions).hasSize(4);
325+
} finally {
326+
tableAdmin.deleteBackup(targetCluster, backupId);
349327
}
350-
assertThat(actualEx).isNull();
351-
352-
List<String> permissions =
353-
tableAdmin.testBackupIamPermission(
354-
targetCluster,
355-
backupId,
356-
"bigtable.backups.get",
357-
"bigtable.backups.delete",
358-
"bigtable.backups.update",
359-
"bigtable.backups.restore");
360-
assertThat(permissions).hasSize(4);
361328
}
362329

363330
private CreateBackupRequest createBackupRequest(String backupName) {
@@ -369,22 +336,4 @@ private CreateBackupRequest createBackupRequest(String backupName) {
369336
private static String generateId(String name) {
370337
return prefix + "-" + name;
371338
}
372-
373-
private void createBackupAndWait(String backupId) throws InterruptedException {
374-
tableAdmin.createBackup(createBackupRequest(backupId));
375-
for (int i = 0; i < BACKOFF_DURATION.length; i++) {
376-
try {
377-
Backup backup = tableAdmin.getBackup(targetCluster, backupId);
378-
if (backup.getState() == Backup.State.READY) {
379-
return;
380-
}
381-
} catch (ApiException ex) {
382-
LOGGER.info("Wait for " + BACKOFF_DURATION[i] + " seconds for creating backup " + backupId);
383-
}
384-
385-
Thread.sleep(BACKOFF_DURATION[i] * 1000);
386-
}
387-
388-
fail("Creating Backup Timeout");
389-
}
390339
}

0 commit comments

Comments
 (0)
Back | FazBrowse Home | New Git URL