FazBrowse GitHub Viewer | Trending |
URL:
| Home
Tools: [Download Repo ZIP]   [Original HTTPS Page]

<fix>[encrypt]: recover iam2 attarbute data by MatheMatrix · Pull Request #1328 · zstackio/zstack · GitHub

/ zstack Public

<fix>[encrypt]: recover iam2 attarbute data #1328

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
MatheMatrix wants to merge 1 commit into
base: master
Choose a base branch
Could not load branches
Branch not found: {{ refName }}

Are you sure you want to change the base?

Some commits from the old base branch may be removed from the timeline, and old review comments may become outdated.
from
+278 −3
Conversation 8 Commits 1 Checks 0 Files changed 3
Open

<fix>[encrypt]: recover iam2 attarbute data #1328

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension .groovy  (1) .java  (1) .sql  (1) All 3 file types selected
Viewed files
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Unified
Split
Hide whitespace
Diff view
Unified
Split
Hide whitespace
70 changes: 70 additions & 0 deletions conf/db/upgrade/V4.5.3__schema.sql
Show comments
View file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
@@ -1,3 +1,73 @@
alter table LicenseHistoryVO modify COLUMN `userName` varchar(64) NOT NULL;

ALTER TABLE LicenseHistoryVO ADD COLUMN capacity int(10) NOT NULL;

CREATE TABLE IF NOT EXISTS `zstack`.`IAM2VirtualIDInformationVO` (
`uuid` varchar(32) NOT NULL UNIQUE,
`phone` varchar(255),
`mail` varchar(255),
PRIMARY KEY (`uuid`),
CONSTRAINT `fkIAM2VirtualIDInformationVOIAM2VirtualIDVO` FOREIGN KEY (`uuid`) REFERENCES `IAM2VirtualIDVO` (`uuid`) ON DELETE CASCADE
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

DELIMITER $$
CREATE PROCEDURE attributePhoneToInformation()
BEGIN
DECLARE vitualIdPhone VARCHAR(32);
DECLARE vitualIdUuid VARCHAR(32);
DECLARE done INT DEFAULT FALSE;
DECLARE phoneCursor CURSOR FOR SELECT virtualIDUuid, value from `zstack`.`IAM2VirtualIDAttributeVO` WHERE name = "phone";
DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = TRUE;

open phoneCursor;
read_loop: LOOP
FETCH phoneCursor INTO vitualIdUuid, vitualIdPhone;
IF done THEN
LEAVE read_loop;
END IF;

IF (select count(*) from IAM2VirtualIDInformationVO where uuid = vitualIdUuid) = 0 THEN
INSERT `zstack`.`IAM2VirtualIDInformationVO`(uuid, phone) values (vitualIdUuid, vitualIdPhone);
else
update `zstack`.`IAM2VirtualIDInformationVO` set phone = vitualIdPhone where uuid = vitualIdUuid;
END IF;

END LOOP;
close phoneCursor;
SELECT CURTIME();
END $$
DELIMITER ;

call attributePhoneToInformation();
DROP PROCEDURE IF EXISTS attributePhoneToInformation;

DELIMITER $$
CREATE PROCEDURE attributeMailToInformation()
BEGIN
DECLARE vitualIdMail VARCHAR(32);
DECLARE vitualIdUuid VARCHAR(32);
DECLARE done INT DEFAULT FALSE;
DECLARE mailCursor CURSOR FOR SELECT virtualIDUuid, value from `zstack`.`IAM2VirtualIDAttributeVO` WHERE name = "mail";
DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = TRUE;

open mailCursor;
read_loop: LOOP
FETCH mailCursor INTO vitualIdUuid, vitualIdMail;
IF done THEN
LEAVE read_loop;
END IF;

IF (select count(*) from IAM2VirtualIDInformationVO where uuid = vitualIdUuid) = 0 THEN
INSERT `zstack`.`IAM2VirtualIDInformationVO`(uuid, mail) values (vitualIdUuid, vitualIdMail);
ELSE
update `zstack`.`IAM2VirtualIDInformationVO` set mail = vitualIdMail where uuid = vitualIdUuid;
END IF;

END LOOP;
close mailCursor;
SELECT CURTIME();
END $$
DELIMITER ;

call attributeMailToInformation();
DROP PROCEDURE IF EXISTS attributeMailToInformation;

cr-gpt Bot Mar 3, 2023

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Choose a reason Spam Abuse Off Topic Outdated Duplicate Resolved Low Quality

with a brief code review:

  1. The patch will modify the table "LicenseHistoryVO" by adding a new column "capacity" which is of type int(10) and not null.
  2. It will also create a new table "IAM2VirtualIDInformationVO" with columns "uuid", "phone" and "mail". The "uuid" column is the primary key and have a foreign key reference to the "IAM2VirtualIDVO" table.
  3. Create two stored procedures called "attributePhoneToInformation" and "attributeMailToInformation". These procedures will loop through the "IAM2VirtualIDAttributeVO" table and populate the newly created "IAM2VirtualIDInformationVO" table with values from the "value" column.

In terms of potential bugs, I would recommend checking that all the data types match in the foreign key reference. Additionally, make sure that the "uuid" column is properly indexed for both tables.

Finally, it may be worth considering adding a timestamp column to the "IAM2VirtualIDInformationVO" table to keep track of when the rows were inserted or updated.

cr-gpt Bot Mar 3, 2023

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Choose a reason Spam Abuse Off Topic Outdated Duplicate Resolved Low Quality

the review:

  1. The code should be checked to make sure that it adheres to the database's best practices and coding standards.
  2. The foreign key should be checked to make sure that it is correctly referencing the primary key of the other table.
  3. The cursor should be checked to make sure it is properly handling errors and that all data is being handled safely.
  4. The stored procedures should be checked to ensure that they are properly constructed and that they are not introducing any potential security risks.
  5. The code should also be checked for optimization opportunities and for any unnecessary or redundant code.
122 changes: 119 additions & 3 deletions core/src/main/java/org/zstack/core/encrypt/EncryptFacadeImpl.java
Show comments
View file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@
package org.zstack.core.encrypt;

import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.transaction.annotation.Transactional;
import org.zstack.core.Platform;
import org.zstack.core.cloudbus.CloudBus;
import org.zstack.core.componentloader.PluginRegistry;
import org.zstack.core.config.GlobalConfig;
import org.zstack.core.config.GlobalConfigBeforeUpdateExtensionPoint;
import org.zstack.core.config.GlobalConfigUpdateExtensionPoint;
import org.zstack.core.config.*;
import org.zstack.core.convert.PasswordConverter;
import org.zstack.core.db.DatabaseFacade;
import org.zstack.core.db.SQLBatch;
Comment on lines 3 to 11

coderabbitai Bot Dec 3, 2023

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Choose a reason Spam Abuse Off Topic Outdated Duplicate Resolved Low Quality

使用通配符导入 (org.zstack.core.config.*) 可能会导致命名空间冲突，并且降低了代码的可读性。建议明确列出所需的类。
Expand Down Expand Up @@ -273,6 +272,123 @@ public void updateGlobalConfig(GlobalConfig oldConfig, GlobalConfig newConfig) {
}
}
});
}

protected void handleNewAddedEncryptEntity() {
if (PasswordEncryptType.None.toString().equals(EncryptGlobalConfig.ENABLE_PASSWORD_ENCRYPT.value())) {
return;
}

List<EncryptEntityMetadataVO> metadataVOList = Q.New(EncryptEntityMetadataVO.class)
.eq(EncryptEntityMetadataVO_.state, EncryptEntityState.NewAdded)
.list();

new SQLBatch() {
@Override
protected void scripts() {
for (EncryptEntityMetadataVO metadata : metadataVOList) {
// do encrypt
long count = SQL.New(String.format("select count(1) from %s", metadata.getEntityName()), Long.class).find();
metadata.setState(EncryptEntityState.Encrypting);
metadata = dbf.updateAndRefresh(metadata);
String className = metadata.getEntityName();
String fieldName = metadata.getColumnName();
sql(String.format("select uuid from %s", metadata.getEntityName()), String.class)
.limit(1000)
.paginate(count, (List<String> uuids) -> {
for (String uuid : uuids) {
String value = sql(String.format("select %s from %s where uuid = '%s'", fieldName, className, uuid)).find();

try {
// If part of the data has been encrypted, first decrypt all the data before encrypting
String decryptedString = decrypt(value);
String encryptedString = encrypt(decryptedString);

String sql = String.format("update %s set %s = :encrypted where uuid = :uuid", className, fieldName);

Query query = dbf.getEntityManager().createQuery(sql);
query.setParameter("encrypted", encryptedString);
query.setParameter("uuid", uuid);
query.executeUpdate();
} catch (Exception e) {
logger.debug(String.format("encrypt error because : %s", e.getMessage()));
}
}

});
metadata.setState(EncryptEntityState.Encrypted);
dbf.updateAndRefresh(metadata);
}
}
}.execute();
}

private void collectEncryptEntityMetadata() {
for (Field field : encryptedFields) {
List<String> classNames = new ArrayList<>();

if (field.getDeclaringClass().getAnnotation(Entity.class) != null && field.getDeclaringClass().getAnnotation(Table.class) != null) {
classNames.add(field.getDeclaringClass().getSimpleName());
} else {
classNames.addAll(BeanUtils.reflections.getSubTypesOf(field.getDeclaringClass()).stream()
.filter(aClass -> aClass.getAnnotation(Entity.class) != null && aClass.getAnnotation(Table.class) != null)
.map(Class::getSimpleName)
.collect(Collectors.toList()));
}

for (String className : classNames) {
createIfNotExists(className, field.getName());
}
}
}

private void createIfNotExists(String entity, String column) {
if (Q.New(EncryptEntityMetadataVO.class)
.eq(EncryptEntityMetadataVO_.entityName, entity)
.eq(EncryptEntityMetadataVO_.columnName, column)
.isExists()) {
return;
}

EncryptEntityMetadataVO metadataVO = new EncryptEntityMetadataVO();
metadataVO.setColumnName(column);
metadataVO.setEntityName(entity);
metadataVO.setState(EncryptEntityState.NewAdded);
dbf.persist(metadataVO);
}

public void updateEncryptDataStateIfExists(String entity, String column, EncryptEntityState state) {
String sql = String.format("update EncryptEntityMetadataVO set state = :state where columnName = :columnName and entityName = :entityName");
Query query = dbf.getEntityManager().createQuery(sql);
query.setParameter("state", state);
query.setParameter("entityName", entity);
query.setParameter("columnName", column);
query.executeUpdate();
}

@Transactional
public void removeConvertRecoverData() {
if (Q.New(EncryptEntityMetadataVO.class)
.isExists()) {
return;
}

if (PasswordEncryptType.None.toString().equals(EncryptGlobalConfig.ENABLE_PASSWORD_ENCRYPT.value())) {
return;
}

decryptAllPassword();
encryptAllPassword();
}

@Override
public boolean start() {
initEncryptDriver();
collectAllEncryptPassword();
installGlobalConfigUpdateHooks();
removeConvertRecoverData();
collectEncryptEntityMetadata();
handleNewAddedEncryptEntity();

return true;
}
Expand Down
89 changes: 89 additions & 0 deletions test/src/test/groovy/org/zstack/test/integration/kvm/host/HostPasswordEncryptCase.groovy
Show comments
View file
Open in desktop
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters. Learn more about bidirectional Unicode characters
Show hidden characters
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,69 @@ class HostPasswordEncryptCase extends SubCase {
assert password == "password"
}

BeanUtils.reflections.getFieldsAnnotatedWith(Convert.class).forEach({ ec ->
if (ec.getDeclaringClass().getAnnotation(Entity.class) == null || ec.getDeclaringClass().getAnnotation(Table.class) == null) {
return;
}
if (!ec.getAnnotation(Convert.class).converter().equals(PasswordConverter.class)) {
return;
}

String entityName = ec.getDeclaringClass().getSimpleName();
String columnName = ec.getName();
retryInSecs {
assert Q.New(EncryptEntityMetadataVO.class).select(EncryptEntityMetadataVO_.state).eq(EncryptEntityMetadataVO_.entityName, entityName).eq(EncryptEntityMetadataVO_.columnName, columnName).findValue() == EncryptEntityState.Encrypted
}
})

assert Q.New(EncryptEntityMetadataVO.class).select(EncryptEntityMetadataVO_.state).eq(EncryptEntityMetadataVO_.entityName, KVMHostVO.class.getSimpleName()).findValue() == EncryptEntityState.Encrypted

// test for unencrypted strings Decryption failures
SQL.New(EncryptEntityMetadataVO.class)
.eq(EncryptEntityMetadataVO_.entityName, KVMHostVO.class.getSimpleName())
.set(EncryptEntityMetadataVO_.state, EncryptEntityState.NewAdded)
.update()

((EncryptFacadeImpl) encryptFacade).handleNewAddedEncryptEntity()

assert Q.New(EncryptEntityMetadataVO.class).select(EncryptEntityMetadataVO_.state).eq(EncryptEntityMetadataVO_.entityName, KVMHostVO.class.getSimpleName()).findValue() == EncryptEntityState.Encrypted

retryInSecs {
password = Q.New(KVMHostVO.class).select(KVMHostVO_.password).eq(KVMHostVO_.uuid, host.uuid).findValue()

assert password == encryptFacade.encrypt("password")
}

// test handleNewAddedEncryptEntity again, result unchanged
SQL.New(EncryptEntityMetadataVO.class)
.eq(EncryptEntityMetadataVO_.entityName, KVMHostVO.class.getSimpleName())
.set(EncryptEntityMetadataVO_.state, EncryptEntityState.NewAdded)
.update()

((EncryptFacadeImpl) encryptFacade).handleNewAddedEncryptEntity()

assert Q.New(EncryptEntityMetadataVO.class).select(EncryptEntityMetadataVO_.state).eq(EncryptEntityMetadataVO_.entityName, KVMHostVO.class.getSimpleName()).findValue() == EncryptEntityState.Encrypted

retryInSecs {
password = Q.New(KVMHostVO.class).select(KVMHostVO_.password).eq(KVMHostVO_.uuid, host.uuid).findValue()

assert password == encryptFacade.encrypt("password")
}

List<Tuple> needToChangeHosts = Q.New(KVMHostVO.class).select(KVMHostVO_.password, KVMHostVO_.uuid).listTuple()
needToChangeHosts.forEach({ changeHost ->
SQL.New(KVMHostVO.class)
.eq(KVMHostVO_.uuid, changeHost.get(1, String.class))
.set(KVMHostVO_.password, encryptFacade.decrypt(changeHost.get(0, String.class)))
.update()
})

retryInSecs {
password = Q.New(KVMHostVO.class).select(KVMHostVO_.password).eq(KVMHostVO_.uuid, host.uuid).findValue()

assert password == "password"
}

updateGlobalConfig {
category = EncryptGlobalConfig.CATEGORY
name = EncryptGlobalConfig.ENABLE_PASSWORD_ENCRYPT.name
Expand All @@ -85,6 +148,32 @@ class HostPasswordEncryptCase extends SubCase {

assert password == "password"
}

BeanUtils.reflections.getFieldsAnnotatedWith(Convert.class).forEach({ ec ->
if (ec.getDeclaringClass().getAnnotation(Entity.class) == null || ec.getDeclaringClass().getAnnotation(Table.class) == null) {
return;
}
if (!ec.getAnnotation(Convert.class).converter().equals(PasswordConverter.class)) {
return;
}

String entityName = ec.getDeclaringClass().getSimpleName();
String columnName = ec.getName();
assert Q.New(EncryptEntityMetadataVO.class).select(EncryptEntityMetadataVO_.state).eq(EncryptEntityMetadataVO_.entityName, entityName).eq(EncryptEntityMetadataVO_.columnName, columnName).findValue() == EncryptEntityState.NewAdded

})

assert Q.New(EncryptEntityMetadataVO.class).select(EncryptEntityMetadataVO_.state).eq(EncryptEntityMetadataVO_.entityName, KVMHostVO.class.getSimpleName()).findValue() == EncryptEntityState.NewAdded

((EncryptFacadeImpl) encryptFacade).handleNewAddedEncryptEntity()

assert Q.New(EncryptEntityMetadataVO.class).select(EncryptEntityMetadataVO_.state).eq(EncryptEntityMetadataVO_.entityName, KVMHostVO.class.getSimpleName()).findValue() == EncryptEntityState.NewAdded

retryInSecs {
password = Q.New(KVMHostVO.class).select(KVMHostVO_.password).eq(KVMHostVO_.uuid, host.uuid).findValue()

assert password == "password"
}
}


Expand Down
Back | FazBrowse Home | New Git URL