FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

Fix interrupted KVM snapshot leaving VM on orphan overlay by sidshas03 · Pull Request #14697 · apache/cloudstack · GitHub

Repository navigation

Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension .java  (6) All 1 file type selected
Viewed files
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Unified
Split
Hide whitespace
Diff view
Unified
Split
Hide whitespace
33 changes: 31 additions & 2 deletions agent/src/main/java/com/cloud/agent/Agent.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -663,6 +663,35 @@ protected void reconnect(final Link link, String preferredMSHost, boolean forTra
logger.info("Connected to the host: {}", shell.getConnectedHost());
}

protected void waitForCommandsAndReconnect(final Link link) {
int inProgress = commandsInProgress.get();
if (inProgress > 0) {
logger.warn("Agent disconnected with {} command(s) still in progress. Waiting for them to complete before reconnecting.", inProgress);

long maxWaitMs = 300000;
long waitedMs = 0;
while (commandsInProgress.get() > 0 && waitedMs < maxWaitMs) {
try {
Thread.sleep(1000);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
break;
}
waitedMs += 1000;
}

if (commandsInProgress.get() > 0) {
logger.warn("Timed out waiting for {} command(s) to complete after {} ms. Proceeding with reconnect — "
+ "in-progress commands (e.g. snapshot merges) may not have completed.", commandsInProgress.get(), waitedMs);
} else {
logger.info("All in-progress commands completed. Proceeding with reconnect.");
}
}

logger.info("Reconnecting instead of exiting to avoid losing in-flight work (e.g. snapshot overlays).");
reconnect(link);
}

protected void closeAndTerminateLink(final Link link) {
if (link == null) {
return;
Expand Down Expand Up @@ -700,7 +729,7 @@ public void processStartupAnswer(final Answer answer, final Response response, f
reconnect(link);
return;
}
System.exit(1);
waitForCommandsAndReconnect(link);
}
if (!answerValid) {
logger.warn("Threw away a startup answer because we're reconnecting.");
Expand Down Expand Up @@ -772,7 +801,7 @@ protected void processRequest(final Request request, final Link link) {
reconnect(link);
return;
}
System.exit(1);
waitForCommandsAndReconnect(link);
return;
} else if (cmd instanceof MaintainCommand) {
logger.debug("Received maintainCommand, do not cancel current tasks");
Expand Down
41 changes: 41 additions & 0 deletions agent/src/test/java/com/cloud/agent/AgentTest.java
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,12 @@
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.any;
import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.eq;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
Expand Down Expand Up @@ -254,4 +256,43 @@ public void testStopAndCleanupConnectionValidConnectionWaitForStopWaitsForStartu
verify(mockConnection).cleanUp();
verify(mockBackoff, times(3)).waitBeforeRetry();
}

@Test
public void testWaitForCommandsAndReconnectNoCommandsInProgress() throws ConfigurationException {
Agent spyAgent = spy(new Agent(shell, 1, serverResource));
ReflectionTestUtils.setField(spyAgent, "logger", logger);
doNothing().when(spyAgent).reconnect(any(Link.class));

Link mockLink = mock(Link.class);
spyAgent.commandsInProgress.set(0);
spyAgent.waitForCommandsAndReconnect(mockLink);

verify(spyAgent).reconnect(mockLink);
verify(logger).info(eq("Reconnecting instead of exiting to avoid losing in-flight work (e.g. snapshot overlays)."));
}

@Test
public void testWaitForCommandsAndReconnectCommandsCompleteBeforeTimeout() throws ConfigurationException {
Agent spyAgent = spy(new Agent(shell, 1, serverResource));
ReflectionTestUtils.setField(spyAgent, "logger", logger);
doNothing().when(spyAgent).reconnect(any(Link.class));

Link mockLink = mock(Link.class);
spyAgent.commandsInProgress.set(1);

Thread decrementThread = new Thread(() -> {
try {
Thread.sleep(500);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
spyAgent.commandsInProgress.set(0);
});
decrementThread.start();

spyAgent.waitForCommandsAndReconnect(mockLink);

verify(spyAgent).reconnect(mockLink);
verify(logger).info(eq("All in-progress commands completed. Proceeding with reconnect."));
}
}
Loading

Back | FazBrowse Home | New Git URL