Skip to content

Commit

Permalink
Some improvements over performance and usability
Browse files Browse the repository at this point in the history 
feat: Issue list performance improvements (OD-2042)
feat: Add criteria to query project by id (OD-2041)
  • Loading branch information
@robinshine
robinshine committed Aug 28, 2024
1 parent 45e38e9 commit d35cf26
Show file tree
Hide file tree
Showing 137 changed files with 429 additions and 814 deletions.
4 changes: 2 additions & 2 deletions pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
<version>1.2.3</version>
</parent>
<artifactId>server</artifactId>
<version>11.0.7</version>
<version>11.0.8</version>
<packaging>pom</packaging>
<build>
<finalName>${project.groupId}.${project.artifactId}-${project.version}</finalName>
Expand Down Expand Up @@ -637,7 +637,7 @@
</repositories>
<properties>
<commons.version>3.0.3</commons.version>
<agent.version>2.1.6</agent.version>
<agent.version>2.1.7</agent.version>
<slf4j.version>2.0.9</slf4j.version>
<logback.version>1.4.14</logback.version>
<antlr.version>4.7.2</antlr.version>
Expand Down
2 changes: 1 addition & 1 deletion server-core/pom.xml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
<parent>
<groupId>io.onedev</groupId>
<artifactId>server</artifactId>
<version>11.0.7</version>
<version>11.0.8</version>
</parent>
<build>
<plugins>
Expand Down
4 changes: 4 additions & 0 deletions server-core/src/main/java/io/onedev/server/commandhandler/Upgrade.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -940,6 +940,10 @@ protected void updateProgramFiles(File upgradeDir, int oldAppDataVersion) {
wrapperConf += "\r\nwrapper.disable_console_input=TRUE";

wrapperConf = wrapperConf.replaceAll("\r\n(\r\n)+\r\n", "\r\n\r\n");
wrapperConf = wrapperConf.replaceAll("\n(\n)+\n", "\n\n");
wrapperConf = wrapperConf.replace(
"wrapperConfwrapper.java.additional.30=--add-modules=java.se",
"wrapper.java.additional.30=--add-modules=java.se");

var lines = Splitter.on('\n').trimResults().splitToList(wrapperConf);
if (lines.stream().noneMatch(it -> it.contains("-XX:MaxRAMPercentage"))) {
Expand Down
125 changes: 79 additions & 46 deletions server-core/src/main/java/io/onedev/server/data/migration/DataMigrator.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,8 @@
import io.onedev.server.model.support.TimeGroups;
import io.onedev.server.ssh.SshKeyUtils;
import io.onedev.server.util.CryptoUtils;
import io.onedev.server.util.DateUtils;
import io.onedev.server.util.Pair;
import io.onedev.server.util.date.Day;
import io.onedev.server.util.date.Month;
import io.onedev.server.util.date.Week;
import io.onedev.server.util.patternset.PatternSet;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.tuple.Triple;
Expand All @@ -41,11 +39,9 @@
import java.security.GeneralSecurityException;
import java.security.PublicKey;
import java.text.MessageFormat;
import java.time.Instant;
import java.time.OffsetDateTime;
import java.time.ZoneId;
import java.time.ZonedDateTime;
import java.time.*;
import java.time.format.DateTimeFormatter;
import java.time.temporal.TemporalAdjusters;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
Expand All @@ -62,7 +58,7 @@ public class DataMigrator {
private static final Logger logger = LoggerFactory.getLogger(DataMigrator.class);

private static final DateTimeFormatter dateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSSSSSSSS'Z'");

private void migrate5(File dataDir, Stack<Integer> versions) {
for (File file : dataDir.listFiles()) {
if (file.getName().startsWith("Configs.xml")) {
Expand Down Expand Up @@ -6701,7 +6697,7 @@ private void migrate171(File dataDir, Stack<Integer> versions) {
throw new RuntimeException(e);
}
}

Map<Long, List<IssueStateHistory>> histories = new HashMap<>();
var id = 1L;
for (File file : dataDir.listFiles()) {
Expand All @@ -6710,7 +6706,7 @@ private void migrate171(File dataDir, Stack<Integer> versions) {
for (Element element : dom.getRootElement().elements()) {
Element dataElement = element.element("data");
var className = dataElement.attributeValue("class");
if (className.contains("IssueStateChangeData")
if (className.contains("IssueStateChangeData")
|| className.contains("IssueBatchUpdateData")) {
var oldState = dataElement.elementText("oldState").trim();
var newState = dataElement.elementText("newState").trim();
Expand All @@ -6722,7 +6718,7 @@ private void migrate171(File dataDir, Stack<Integer> versions) {
history.setTimeGroups(TimeGroups.of(history.getDate()));
history.setIssue(new Issue());
history.getIssue().setId(Long.parseLong(element.elementText("issue").trim()));
histories.computeIfAbsent(history.getIssue().getId(), it->new ArrayList<>()).add(history);
histories.computeIfAbsent(history.getIssue().getId(), it -> new ArrayList<>()).add(history);
}
}
}
Expand All @@ -6738,9 +6734,9 @@ private void migrate171(File dataDir, Stack<Integer> versions) {
if (finishDateElement != null) {
var finishDate = parseDate(finishDateElement.getText().trim());
var finishTimeGroupsElement = element.addElement("finishTimeGroups");
finishTimeGroupsElement.addElement("day").setText(String.valueOf(new Day(finishDate).getValue()));
finishTimeGroupsElement.addElement("week").setText(String.valueOf(new Week(finishDate).getValue()));
finishTimeGroupsElement.addElement("month").setText(String.valueOf(new Month(finishDate).getValue()));
finishTimeGroupsElement.addElement("day").setText(String.valueOf(DateUtils.toLocalDate(finishDate).toEpochDay()));
finishTimeGroupsElement.addElement("week").setText(String.valueOf(DateUtils.toLocalDate(finishDate).with(TemporalAdjusters.previousOrSame(DayOfWeek.MONDAY)).toEpochDay()));
finishTimeGroupsElement.addElement("month").setText(String.valueOf(DateUtils.toLocalDate(finishDate).with(TemporalAdjusters.firstDayOfMonth()).toEpochDay()));
if (runningDateElement != null)
element.addElement("runningDuration").setText(String.valueOf(finishDate.getTime() - parseDate(runningDateElement.getText().trim()).getTime()));
}
Expand All @@ -6753,43 +6749,43 @@ private void migrate171(File dataDir, Stack<Integer> versions) {
for (Element element : dom.getRootElement().elements()) {
var submitDate = parseDate(element.elementText("submitDate").trim());
var submitTimeGroupsElement = element.addElement("submitTimeGroups");
submitTimeGroupsElement.addElement("day").setText(String.valueOf(new Day(submitDate).getValue()));
submitTimeGroupsElement.addElement("week").setText(String.valueOf(new Week(submitDate).getValue()));
submitTimeGroupsElement.addElement("month").setText(String.valueOf(new Month(submitDate).getValue()));
submitTimeGroupsElement.addElement("day").setText(String.valueOf(DateUtils.toLocalDate(submitDate).toEpochDay()));
submitTimeGroupsElement.addElement("week").setText(String.valueOf(DateUtils.toLocalDate(submitDate).with(TemporalAdjusters.previousOrSame(DayOfWeek.MONDAY)).toEpochDay()));
submitTimeGroupsElement.addElement("month").setText(String.valueOf(DateUtils.toLocalDate(submitDate).with(TemporalAdjusters.firstDayOfMonth()).toEpochDay()));

if (!element.element("status").equals("OPEN")) {
var closeDate = parseDate(element.element("lastActivity").elementText("date").trim());
element.addElement("closeDate").addAttribute("class", "sql-timestamp").setText(formatDate(closeDate));
element.addElement("duration").setText(String.valueOf(closeDate.getTime() - submitDate.getTime()));
var closeTimeGroupsElement = element.addElement("closeTimeGroups");
closeTimeGroupsElement.addElement("day").setText(String.valueOf(new Day(closeDate).getValue()));
closeTimeGroupsElement.addElement("week").setText(String.valueOf(new Week(closeDate).getValue()));
closeTimeGroupsElement.addElement("month").setText(String.valueOf(new Month(closeDate).getValue()));
closeTimeGroupsElement.addElement("day").setText(String.valueOf(DateUtils.toLocalDate(closeDate).toEpochDay()));
closeTimeGroupsElement.addElement("week").setText(String.valueOf(DateUtils.toLocalDate(closeDate).with(TemporalAdjusters.previousOrSame(DayOfWeek.MONDAY)).toEpochDay()));
closeTimeGroupsElement.addElement("month").setText(String.valueOf(DateUtils.toLocalDate(closeDate).with(TemporalAdjusters.firstDayOfMonth()).toEpochDay()));
}
}
dom.writeToFile(file, false);
}
}

VersionedXmlDoc historiesDom;
File historiesDomFile = new File(dataDir, "IssueStateHistorys.xml");
historiesDom = new VersionedXmlDoc();
Element listElement = historiesDom.addElement("list");
for (var value: histories.values()) {

for (var value : histories.values()) {
value.sort(comparing(IssueStateHistory::getDate));
for(int i=0; i<value.size()-1; i++) {
for (int i = 0; i < value.size() - 1; i++) {
var history = value.get(i);
var nextHistory = value.get(i+1);
var nextHistory = value.get(i + 1);
history.setDuration(nextHistory.getDate().getTime() - history.getDate().getTime());
migrate171_createHistoryElement(listElement, history);
}
if (!value.isEmpty())
migrate171_createHistoryElement(listElement, value.get(value.size()-1));
migrate171_createHistoryElement(listElement, value.get(value.size() - 1));
}
historiesDom.writeToFile(historiesDomFile, true);
}

private static Date parseDate(String dateString) {
if (dateString.endsWith("Z")) {
Instant instant = Instant.parse(dateString);
Expand All @@ -6799,12 +6795,12 @@ private static Date parseDate(String dateString) {
return Date.from(offsetDateTime.toInstant());
}
}

private static String formatDate(Date date) {
ZonedDateTime zdt = ZonedDateTime.ofInstant(date.toInstant(), ZoneId.of("UTC"));
return zdt.format(dateTimeFormatter);
}

private Element migrate171_createHistoryElement(Element parentElement, IssueStateHistory history) {
var historyElement = parentElement.addElement("io.onedev.server.model.IssueStateHistory");
historyElement.addAttribute("revision", "0.0");
Expand All @@ -6813,12 +6809,12 @@ private Element migrate171_createHistoryElement(Element parentElement, IssueStat
historyElement.addElement("state").setText(history.getState());
historyElement.addElement("date").addAttribute("class", "sql-timestamp").setText(formatDate(history.getDate()));
var timeGroupsElement = historyElement.addElement("timeGroups");
timeGroupsElement.addElement("day").setText(String.valueOf(new Day(history.getDate()).getValue()));
timeGroupsElement.addElement("week").setText(String.valueOf(new Week(history.getDate()).getValue()));
timeGroupsElement.addElement("month").setText(String.valueOf(new Month(history.getDate()).getValue()));
timeGroupsElement.addElement("day").setText(String.valueOf(DateUtils.toLocalDate(history.getDate()).toEpochDay()));
timeGroupsElement.addElement("week").setText(String.valueOf(DateUtils.toLocalDate(history.getDate()).with(TemporalAdjusters.previousOrSame(DayOfWeek.MONDAY)).toEpochDay()));
timeGroupsElement.addElement("month").setText(String.valueOf(DateUtils.toLocalDate(history.getDate()).with(TemporalAdjusters.firstDayOfMonth()).toEpochDay()));
if (history.getDuration() != null)
historyElement.addElement("duration").setText(history.getDuration().toString());
return historyElement;
return historyElement;
}

private void migrate172(File dataDir, Stack<Integer> versions) {
Expand Down Expand Up @@ -6847,7 +6843,7 @@ private void migrate173(File dataDir, Stack<Integer> versions) {
} catch (IOException e) {
throw new RuntimeException(e);
}

for (File file : dataDir.listFiles()) {
if (file.getName().startsWith("Settings.xml")) {
VersionedXmlDoc dom = VersionedXmlDoc.fromFile(file);
Expand Down Expand Up @@ -6877,7 +6873,7 @@ private void migrate174(File dataDir, Stack<Integer> versions) {
if (key.equals("ISSUE")) {
Element valueElement = element.element("value");
if (valueElement != null) {
for (var stateSpecElement: valueElement.element("stateSpecs").elements()) {
for (var stateSpecElement : valueElement.element("stateSpecs").elements()) {
initialState = stateSpecElement.elementText("name").trim();
break;
}
Expand All @@ -6886,12 +6882,12 @@ private void migrate174(File dataDir, Stack<Integer> versions) {
}
} else if (file.getName().startsWith("IssueStateHistorys.xml")) {
FileUtils.deleteFile(file);
}
}
}

Map<Long, List<IssueStateHistory>> histories = new HashMap<>();
var id = 1L;

for (File file : dataDir.listFiles()) {
if (file.getName().startsWith("Issues.xml")) {
if (initialState != null) {
Expand Down Expand Up @@ -6924,7 +6920,7 @@ private void migrate174(File dataDir, Stack<Integer> versions) {
history.setTimeGroups(TimeGroups.of(history.getDate()));
history.setIssue(new Issue());
history.getIssue().setId(Long.parseLong(element.elementText("issue").trim()));
histories.computeIfAbsent(history.getIssue().getId(), it->new ArrayList<>()).add(history);
histories.computeIfAbsent(history.getIssue().getId(), it -> new ArrayList<>()).add(history);
}
}
}
Expand All @@ -6936,16 +6932,16 @@ private void migrate174(File dataDir, Stack<Integer> versions) {
historiesDom = new VersionedXmlDoc();
Element listElement = historiesDom.addElement("list");

for (var value: histories.values()) {
for (var value : histories.values()) {
value.sort(comparing(IssueStateHistory::getDate));
for(int i=0; i<value.size()-1; i++) {
for (int i = 0; i < value.size() - 1; i++) {
var history = value.get(i);
var nextHistory = value.get(i+1);
var nextHistory = value.get(i + 1);
history.setDuration(nextHistory.getDate().getTime() - history.getDate().getTime());
migrate171_createHistoryElement(listElement, history);
}
if (!value.isEmpty())
migrate171_createHistoryElement(listElement, value.get(value.size()-1));
migrate171_createHistoryElement(listElement, value.get(value.size() - 1));
}
historiesDom.writeToFile(historiesDomFile, true);
}
Expand Down Expand Up @@ -6983,7 +6979,7 @@ private void migrate175(File dataDir, Stack<Integer> versions) {
} else {
transitionSpecElement.setName("io.onedev.server.model.support.issue.transitionspec.IssueStateTransitedSpec");
}
for (var childElement: triggerElement.elements()) {
for (var childElement : triggerElement.elements()) {
if (!childElement.getName().equals("buttonLabel")) {
childElement.detach();
transitionSpecElement.add(childElement);
Expand All @@ -7003,5 +6999,42 @@ private void migrate175(File dataDir, Stack<Integer> versions) {
}
}
}

private void migrate176_timeGroups(Element timeGroupsElement, String dateString) {
var localDate = DateUtils.toLocalDate(parseDate(dateString));
timeGroupsElement.clearContent();
timeGroupsElement.addElement("day").setText(String.valueOf(localDate.toEpochDay()));
timeGroupsElement.addElement("week").setText(String.valueOf(localDate.with(TemporalAdjusters.previousOrSame(DayOfWeek.MONDAY)).toEpochDay()));
timeGroupsElement.addElement("month").setText(String.valueOf(localDate.with(TemporalAdjusters.firstDayOfMonth()).toEpochDay()));
}

private void migrate176(File dataDir, Stack<Integer> versions) {
for (File file : dataDir.listFiles()) {
if (file.getName().startsWith("Builds.xml")) {
VersionedXmlDoc dom = VersionedXmlDoc.fromFile(file);
for (Element element : dom.getRootElement().elements()) {
var finishDate = element.elementText("finishDate");
if (finishDate != null)
migrate176_timeGroups(element.element("finishTimeGroups"), finishDate.trim());
}
dom.writeToFile(file, false);
} else if (file.getName().startsWith("PullRequests.xml")) {
VersionedXmlDoc dom = VersionedXmlDoc.fromFile(file);
for (Element element : dom.getRootElement().elements()) {
migrate176_timeGroups(element.element("submitTimeGroups"), element.elementText("submitDate").trim());
var closeDate = element.elementText("closeDate");
if (closeDate != null)
migrate176_timeGroups(element.element("closeTimeGroups"), closeDate.trim());
}
dom.writeToFile(file, false);
} else if (file.getName().startsWith("IssueStateHistorys.xml")) {
VersionedXmlDoc dom = VersionedXmlDoc.fromFile(file);
for (Element element : dom.getRootElement().elements()) {
migrate176_timeGroups(element.element("timeGroups"), element.elementText("date").trim());
}
dom.writeToFile(file, false);
}
}
}

}
}
2 changes: 1 addition & 1 deletion server-core/src/main/java/io/onedev/server/entitymanager/IssueManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ List<javax.persistence.criteria.Order> buildOrders(List<EntitySort> sorts, Crite
From<Issue, Issue> issue);

List<Issue> query(@Nullable ProjectScope projectScope, EntityQuery<Issue> issueQuery,
boolean loadFieldsAndLinks, int firstResult, int maxResults);
boolean loadExtraInfo, int firstResult, int maxResults);

int count(@Nullable ProjectScope projectScope, @Nullable Criteria<Issue> issueCriteria);

Expand Down
4 changes: 4 additions & 0 deletions server-core/src/main/java/io/onedev/server/entitymanager/IssueScheduleManager.java
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package io.onedev.server.entitymanager;

import java.util.Collection;
import java.util.List;

import io.onedev.server.model.Issue;
import io.onedev.server.model.IssueSchedule;
Expand All @@ -12,4 +13,7 @@ public interface IssueScheduleManager extends EntityManager<IssueSchedule> {
void syncIterations(Issue issue, Collection<Iteration> iterations);

void create(IssueSchedule schedule);

void populateSchedules(Collection<Issue> issues);

}
11 changes: 8 additions & 3 deletions server-core/src/main/java/io/onedev/server/entitymanager/impl/DefaultIssueManager.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,8 @@ public class DefaultIssueManager extends BaseEntityManager<Issue> implements Iss

private final IssueLinkManager linkManager;

private final IssueScheduleManager scheduleManager;

private final ClusterManager clusterManager;

private final IssueTouchManager touchManager;
Expand All @@ -107,7 +109,8 @@ public DefaultIssueManager(Dao dao, IssueFieldManager fieldManager, TransactionM
SettingManager settingManager, ListenerRegistry listenerRegistry,
ProjectManager projectManager, UserManager userManager, ClusterManager clusterManager,
RoleManager roleManager, LinkSpecManager linkSpecManager, IssueLinkManager linkManager,
IssueAuthorizationManager authorizationManager, IssueTouchManager touchManager) {
IssueAuthorizationManager authorizationManager, IssueTouchManager touchManager,
IssueScheduleManager scheduleManager) {
super(dao);
this.fieldManager = fieldManager;
this.queryPersonalizationManager = queryPersonalizationManager;
Expand All @@ -122,6 +125,7 @@ public DefaultIssueManager(Dao dao, IssueFieldManager fieldManager, TransactionM
this.authorizationManager = authorizationManager;
this.clusterManager = clusterManager;
this.touchManager = touchManager;
this.scheduleManager = scheduleManager;

numberGenerator = new SequenceGenerator(Issue.class, clusterManager, dao);
}
Expand Down Expand Up @@ -243,7 +247,7 @@ private GlobalIssueSetting getIssueSetting() {
@Sessional
@Override
public List<Issue> query(@Nullable ProjectScope projectScope, EntityQuery<Issue> issueQuery,
boolean loadFieldsAndLinks, int firstResult, int maxResults) {
boolean loadExtraInfo, int firstResult, int maxResults) {
CriteriaBuilder builder = getSession().getCriteriaBuilder();
CriteriaQuery<Issue> criteriaQuery = builder.createQuery(Issue.class);
Root<Issue> root = criteriaQuery.from(Issue.class);
Expand All @@ -255,9 +259,10 @@ public List<Issue> query(@Nullable ProjectScope projectScope, EntityQuery<Issue>
query.setFirstResult(firstResult);
query.setMaxResults(maxResults);
List<Issue> issues = query.getResultList();
if (loadFieldsAndLinks && !issues.isEmpty()) {
if (loadExtraInfo && !issues.isEmpty()) {
fieldManager.populateFields(issues);
linkManager.populateLinks(issues);
scheduleManager.populateSchedules(issues);
}
return issues;
}
Expand Down
Loading

0 comments on commit d35cf26

Please sign in to comment.