Skip to content

Commit ebaf5a4

Browse files
vishesh92vishesh92
authored
Speedup resource count calculation (#8903)
* Speed up resource count calculation * Refactor resource count calculation * Start transaction for updateCountByDeltaForIds
1 parent 090c1e3 commit ebaf5a4

File tree

5 files changed

+145
-87
lines changed

5 files changed

+145
-87
lines changed

‎engine/schema/src/main/java/com/cloud/configuration/dao/ResourceCountDao.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -49,10 +49,15 @@ public interface ResourceCountDao extends GenericDao<ResourceCountVO, Long> {
4949

5050
ResourceCountVO findByOwnerAndTypeAndTag(long ownerId, ResourceOwnerType ownerType, ResourceType type, String tag);
5151

52+
List<ResourceCountVO> findByOwnersAndTypeAndTag(List<Long> ownerIdList, ResourceOwnerType ownerType,
53+
ResourceType type, String tag);
54+
5255
List<ResourceCountVO> listResourceCountByOwnerType(ResourceOwnerType ownerType);
5356

5457
Set<Long> listAllRowsToUpdate(long ownerId, ResourceOwnerType ownerType, ResourceType type, String tag);
5558

59+
boolean updateCountByDeltaForIds(List<Long> ids, boolean increment, long delta);
60+
5661
Set<Long> listRowsToUpdateForDomain(long domainId, ResourceType type, String tag);
5762

5863
long removeEntriesByOwner(long ownerId, ResourceOwnerType ownerType);
@@ -72,4 +77,6 @@ public interface ResourceCountDao extends GenericDao<ResourceCountVO, Long> {
7277
long countMemoryAllocatedToAccount(long accountId);
7378

7479
void removeResourceCountsForNonMatchingTags(Long ownerId, ResourceOwnerType ownerType, List<ResourceType> types, List<String> tags);
80+
81+
List<ResourceCountVO> lockRows(Set<Long> ids);
7582
}

‎engine/schema/src/main/java/com/cloud/configuration/dao/ResourceCountDaoImpl.java‎

Lines changed: 61 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import java.util.HashSet;
2525
import java.util.List;
2626
import java.util.Set;
27+
import java.util.stream.Collectors;
2728

2829
import javax.annotation.PostConstruct;
2930
import javax.inject.Inject;
@@ -56,27 +57,30 @@ public class ResourceCountDaoImpl extends GenericDaoBase<ResourceCountVO, Long>
5657
private final SearchBuilder<ResourceCountVO> TypeSearch;
5758
private final SearchBuilder<ResourceCountVO> TypeNullTagSearch;
5859
private final SearchBuilder<ResourceCountVO> NonMatchingTagsSearch;
59-
6060
private final SearchBuilder<ResourceCountVO> AccountSearch;
6161
private final SearchBuilder<ResourceCountVO> DomainSearch;
62+
private final SearchBuilder<ResourceCountVO> IdsSearch;
6263

6364
@Inject
6465
private DomainDao _domainDao;
6566
@Inject
6667
private AccountDao _accountDao;
6768

69+
protected static final String INCREMENT_COUNT_BY_IDS_SQL = "UPDATE `cloud`.`resource_count` SET `count` = `count` + ? WHERE `id` IN (?)";
70+
protected static final String DECREMENT_COUNT_BY_IDS_SQL = "UPDATE `cloud`.`resource_count` SET `count` = `count` - ? WHERE `id` IN (?)";
71+
6872
public ResourceCountDaoImpl() {
6973
TypeSearch = createSearchBuilder();
7074
TypeSearch.and("type", TypeSearch.entity().getType(), SearchCriteria.Op.EQ);
71-
TypeSearch.and("accountId", TypeSearch.entity().getAccountId(), SearchCriteria.Op.EQ);
72-
TypeSearch.and("domainId", TypeSearch.entity().getDomainId(), SearchCriteria.Op.EQ);
75+
TypeSearch.and("accountId", TypeSearch.entity().getAccountId(), SearchCriteria.Op.IN);
76+
TypeSearch.and("domainId", TypeSearch.entity().getDomainId(), SearchCriteria.Op.IN);
7377
TypeSearch.and("tag", TypeSearch.entity().getTag(), SearchCriteria.Op.EQ);
7478
TypeSearch.done();
7579

7680
TypeNullTagSearch = createSearchBuilder();
7781
TypeNullTagSearch.and("type", TypeNullTagSearch.entity().getType(), SearchCriteria.Op.EQ);
78-
TypeNullTagSearch.and("accountId", TypeNullTagSearch.entity().getAccountId(), SearchCriteria.Op.EQ);
79-
TypeNullTagSearch.and("domainId", TypeNullTagSearch.entity().getDomainId(), SearchCriteria.Op.EQ);
82+
TypeNullTagSearch.and("accountId", TypeNullTagSearch.entity().getAccountId(), SearchCriteria.Op.IN);
83+
TypeNullTagSearch.and("domainId", TypeNullTagSearch.entity().getDomainId(), SearchCriteria.Op.IN);
8084
TypeNullTagSearch.and("tag", TypeNullTagSearch.entity().getTag(), SearchCriteria.Op.NULL);
8185
TypeNullTagSearch.done();
8286

@@ -90,6 +94,10 @@ public ResourceCountDaoImpl() {
9094

9195
AccountSearch = createSearchBuilder();
9296
DomainSearch = createSearchBuilder();
97+
98+
IdsSearch = createSearchBuilder();
99+
IdsSearch.and("id", IdsSearch.entity().getId(), SearchCriteria.Op.IN);
100+
IdsSearch.done();
93101
}
94102

95103
@PostConstruct
@@ -109,20 +117,33 @@ protected void configure() {
109117

110118
@Override
111119
public ResourceCountVO findByOwnerAndTypeAndTag(long ownerId, ResourceOwnerType ownerType, ResourceType type, String tag) {
120+
List<ResourceCountVO> resourceCounts = findByOwnersAndTypeAndTag(List.of(ownerId), ownerType, type, tag);
121+
if (CollectionUtils.isNotEmpty(resourceCounts)) {
122+
return resourceCounts.get(0);
123+
} else {
124+
return null;
125+
}
126+
}
127+
128+
@Override
129+
public List<ResourceCountVO> findByOwnersAndTypeAndTag(List<Long> ownerIdList, ResourceOwnerType ownerType, ResourceType type, String tag) {
130+
if (CollectionUtils.isEmpty(ownerIdList)) {
131+
return new ArrayList<>();
132+
}
112133
SearchCriteria<ResourceCountVO> sc = tag != null ? TypeSearch.create() : TypeNullTagSearch.create();
113134
sc.setParameters("type", type);
114135
if (tag != null) {
115136
sc.setParameters("tag", tag);
116137
}
117138

118139
if (ownerType == ResourceOwnerType.Account) {
119-
sc.setParameters("accountId", ownerId);
120-
return findOneIncludingRemovedBy(sc);
140+
sc.setParameters("accountId", ownerIdList.toArray());
141+
return listIncludingRemovedBy(sc);
121142
} else if (ownerType == ResourceOwnerType.Domain) {
122-
sc.setParameters("domainId", ownerId);
123-
return findOneIncludingRemovedBy(sc);
143+
sc.setParameters("domainId", ownerIdList.toArray());
144+
return listIncludingRemovedBy(sc);
124145
} else {
125-
return null;
146+
return new ArrayList<>();
126147
}
127148
}
128149

@@ -154,6 +175,26 @@ public boolean updateById(long id, boolean increment, long delta) {
154175
return update(resourceCountVO.getId(), resourceCountVO);
155176
}
156177

178+
@Override
179+
public boolean updateCountByDeltaForIds(List<Long> ids, boolean increment, long delta) {
180+
if (CollectionUtils.isEmpty(ids)) {
181+
return false;
182+
}
183+
String updateSql = increment ? INCREMENT_COUNT_BY_IDS_SQL : DECREMENT_COUNT_BY_IDS_SQL;
184+
185+
String poolIdsInStr = ids.stream().map(String::valueOf).collect(Collectors.joining(",", "(", ")"));
186+
String sql = updateSql.replace("(?)", poolIdsInStr);
187+
188+
final TransactionLegacy txn = TransactionLegacy.currentTxn();
189+
try(PreparedStatement pstmt = txn.prepareStatement(sql);) {
190+
pstmt.setLong(1, delta);
191+
pstmt.executeUpdate();
192+
return true;
193+
} catch (SQLException e) {
194+
throw new CloudRuntimeException(e);
195+
}
196+
}
197+
157198
@Override
158199
public Set<Long> listRowsToUpdateForDomain(long domainId, ResourceType type, String tag) {
159200
Set<Long> rowIds = new HashSet<Long>();
@@ -345,4 +386,14 @@ public void removeResourceCountsForNonMatchingTags(Long ownerId, ResourceOwnerTy
345386
}
346387
remove(sc);
347388
}
389+
390+
@Override
391+
public List<ResourceCountVO> lockRows(Set<Long> ids) {
392+
if (CollectionUtils.isEmpty(ids)) {
393+
return new ArrayList<>();
394+
}
395+
SearchCriteria<ResourceCountVO> sc = IdsSearch.create();
396+
sc.setParameters("id", ids.toArray());
397+
return lockRows(sc, null, true);
398+
}
348399
}

‎engine/schema/src/main/java/org/apache/cloudstack/reservation/dao/ReservationDao.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -31,4 +31,5 @@ public interface ReservationDao extends GenericDao<ReservationVO, Long> {
3131
void setResourceId(Resource.ResourceType type, Long resourceId);
3232
List<Long> getResourceIds(long accountId, Resource.ResourceType type);
3333
List<ReservationVO> getReservationsForAccount(long accountId, Resource.ResourceType type, String tag);
34+
void removeByIds(List<Long> reservationIds);
3435
}

‎engine/schema/src/main/java/org/apache/cloudstack/reservation/dao/ReservationDaoImpl.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import com.cloud.utils.db.SearchBuilder;
3030
import com.cloud.utils.db.SearchCriteria;
3131
import org.apache.cloudstack.user.ResourceReservation;
32+
import org.apache.commons.collections.CollectionUtils;
3233
import org.apache.logging.log4j.LogManager;
3334
import org.apache.logging.log4j.Logger;
3435

@@ -40,13 +41,15 @@ public class ReservationDaoImpl extends GenericDaoBase<ReservationVO, Long> impl
4041
private static final String RESOURCE_ID = "resourceId";
4142
private static final String ACCOUNT_ID = "accountId";
4243
private static final String DOMAIN_ID = "domainId";
44+
private static final String IDS = "ids";
4345
private final SearchBuilder<ReservationVO> listResourceByAccountAndTypeSearch;
4446
private final SearchBuilder<ReservationVO> listAccountAndTypeSearch;
4547
private final SearchBuilder<ReservationVO> listAccountAndTypeAndNoTagSearch;
4648

4749
private final SearchBuilder<ReservationVO> listDomainAndTypeSearch;
4850
private final SearchBuilder<ReservationVO> listDomainAndTypeAndNoTagSearch;
4951
private final SearchBuilder<ReservationVO> listResourceByAccountAndTypeAndNoTagSearch;
52+
private final SearchBuilder<ReservationVO> listIdsSearch;
5053

5154
public ReservationDaoImpl() {
5255

@@ -87,6 +90,10 @@ public ReservationDaoImpl() {
8790
listDomainAndTypeAndNoTagSearch.and(RESOURCE_TYPE, listDomainAndTypeAndNoTagSearch.entity().getResourceType(), SearchCriteria.Op.EQ);
8891
listDomainAndTypeAndNoTagSearch.and(RESOURCE_TAG, listDomainAndTypeAndNoTagSearch.entity().getTag(), SearchCriteria.Op.NULL);
8992
listDomainAndTypeAndNoTagSearch.done();
93+
94+
listIdsSearch = createSearchBuilder();
95+
listIdsSearch.and(IDS, listIdsSearch.entity().getId(), SearchCriteria.Op.IN);
96+
listIdsSearch.done();
9097
}
9198

9299
@Override
@@ -161,4 +168,13 @@ public List<ReservationVO> getReservationsForAccount(long accountId, Resource.Re
161168
}
162169
return listBy(sc);
163170
}
171+
172+
@Override
173+
public void removeByIds(List<Long> reservationIds) {
174+
if (CollectionUtils.isNotEmpty(reservationIds)) {
175+
SearchCriteria<ReservationVO> sc = listIdsSearch.create();
176+
sc.setParameters(IDS, reservationIds.toArray());
177+
remove(sc);
178+
}
179+
}
164180
}

0 commit comments

Comments
 (0)