Stop listing all files POC

Author: kirill-knize-sonarsource

backend/core/src/main/java/org/sonarsource/sonarlint/core/fs/FileExclusionService.java

@@ -98,7 +98,6 @@ public FileExclusionService(ConfigurationRepository configRepo, StorageService s
   }
 
   public boolean computeIfExcluded(URI fileUri, SonarLintCancelMonitor cancelMonitor) {
-    LOG.debug("Computing file exclusion for uri '{}'", fileUri);
     var clientFile = clientFileSystemService.getClientFile(fileUri);
     if (clientFile == null) {
       LOG.debug("Unable to find client file for uri {}", fileUri);
@@ -140,7 +139,6 @@ public boolean computeIfExcluded(URI fileUri, SonarLintCancelMonitor cancelMonit
     }
     var type = clientFile.isTest() ? InputFile.Type.TEST : InputFile.Type.MAIN;
     var result = !exclusionFilters.accept(serverPath.toString(), type);
-    LOG.debug("File exclusion for uri '{}' is {}", fileUri, result);
     return result;
   }
 

backend/core/src/main/java/org/sonarsource/sonarlint/core/sync/FindingsSynchronizationService.java

@@ -20,13 +20,15 @@
 package org.sonarsource.sonarlint.core.sync;
 
 import java.nio.file.Path;
+import java.time.Instant;
 import java.util.LinkedList;
 import java.util.Set;
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.ExecutorService;
 import java.util.stream.Collectors;
 import org.sonarsource.sonarlint.core.branch.SonarProjectBranchTrackingService;
 import org.sonarsource.sonarlint.core.commons.Binding;
+import org.sonarsource.sonarlint.core.commons.log.SonarLintLogger;
 import org.sonarsource.sonarlint.core.commons.progress.SonarLintCancelMonitor;
 import org.sonarsource.sonarlint.core.commons.util.FailSafeExecutors;
 import org.sonarsource.sonarlint.core.file.FilePathTranslation;
@@ -36,6 +38,7 @@
 import org.sonarsource.sonarlint.core.rpc.protocol.backend.initialize.InitializeParams;
 
 public class FindingsSynchronizationService {
+  private static final SonarLintLogger LOG = SonarLintLogger.get();
   private static final int FETCH_ALL_ISSUES_THRESHOLD = 10;
   private final ConfigurationRepository configurationRepository;
   private final SonarProjectBranchTrackingService branchTrackingService;
@@ -58,6 +61,7 @@ public FindingsSynchronizationService(ConfigurationRepository configurationRepos
   }
 
   public void refreshServerFindings(String configurationScopeId, Set<Path> pathsToRefresh) {
+    LOG.debug("Refreshing server findings for configuration scope: {}", configurationScopeId);
     var effectiveBindingOpt = configurationRepository.getEffectiveBinding(configurationScopeId);
     var activeBranchOpt = branchTrackingService.awaitEffectiveSonarProjectBranch(configurationScopeId);
     var translationOpt = pathTranslationService.getOrComputePathTranslation(configurationScopeId);
@@ -75,9 +79,12 @@ public void refreshServerFindings(String configurationScopeId, Set<Path> pathsTo
 
   private void refreshServerIssues(SonarLintCancelMonitor cancelMonitor, Binding binding, String activeBranch,
     Set<Path> pathsInvolved, FilePathTranslation translation) {
+    LOG.debug("Refreshing server issues for binding: {}, active branch: {}", binding, activeBranch);
     var serverFileRelativePaths = pathsInvolved.stream().map(translation::ideToServerPath).collect(Collectors.toSet());
     var downloadAllIssuesAtOnce = serverFileRelativePaths.size() > FETCH_ALL_ISSUES_THRESHOLD;
     var fetchTasks = new LinkedList<CompletableFuture<?>>();
+    LOG.debug("Fetching issues");
+    var now = Instant.now();
     if (downloadAllIssuesAtOnce) {
       fetchTasks.add(CompletableFuture.runAsync(() -> issueSynchronizationService.fetchProjectIssues(binding, activeBranch, cancelMonitor), issueUpdaterExecutorService));
     } else {
@@ -87,6 +94,7 @@ private void refreshServerIssues(SonarLintCancelMonitor cancelMonitor, Binding b
         .toList());
     }
     CompletableFuture.allOf(fetchTasks.toArray(new CompletableFuture[0])).join();
+    LOG.debug("Fetching issues took {} ms", Instant.now().toEpochMilli() - now.toEpochMilli());
   }
 
   private void refreshServerSecurityHotspots(SonarLintCancelMonitor cancelMonitor, Binding binding, String activeBranch,

backend/core/src/main/java/org/sonarsource/sonarlint/core/sync/IssueSynchronizationService.java

@@ -61,11 +61,13 @@ public void syncServerIssuesForProject(ServerApi serverApi, String connectionId,
   }
 
   public void fetchProjectIssues(Binding binding, String activeBranch, SonarLintCancelMonitor cancelMonitor) {
+    LOG.debug("Start downloading issues from SonarQube - fetchProjectIssues");
     sonarQubeClientManager.withActiveClient(binding.connectionId(), serverApi ->
       downloadServerIssuesForProject(binding.connectionId(), serverApi, binding.sonarProjectKey(), activeBranch, cancelMonitor));
   }
 
   private void downloadServerIssuesForProject(String connectionId, ServerApi serverApi, String projectKey, String branchName, SonarLintCancelMonitor cancelMonitor) {
+    LOG.info("[SYNC] Synchronizing issues for project '{}' on branch '{}'", projectKey, branchName);
     var storage = storageService.connection(connectionId);
     var enabledLanguagesToSync = languageSupportRepository.getEnabledLanguagesInConnectedMode().stream().filter(SonarLanguage::shouldSyncInConnectedMode)
       .collect(Collectors.toCollection(LinkedHashSet::new));

backend/core/src/main/java/org/sonarsource/sonarlint/core/tracking/TrackingService.java

@@ -150,6 +150,7 @@ public void onAnalysisFailed(AnalysisFailedEvent event) {
 
   @EventListener
   public void onAnalysisFinished(AnalysisFinishedEvent event) {
+    LOG.debug("Analysis finished: {}", event);
     var analysisId = event.getAnalysisId();
     var matchingSession = matchingSessionByAnalysisId.remove(analysisId);
     if (matchingSession == null) {

backend/server-connection/src/main/java/org/sonarsource/sonarlint/core/serverconnection/ServerIssueUpdater.java

@@ -20,6 +20,7 @@
 package org.sonarsource.sonarlint.core.serverconnection;
 
 import java.nio.file.Path;
+import java.time.Instant;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Set;
@@ -29,6 +30,7 @@
 import org.sonarsource.sonarlint.core.serverapi.ServerApi;
 import org.sonarsource.sonarlint.core.serverconnection.issues.ServerIssue;
 import org.sonarsource.sonarlint.core.serverconnection.issues.ServerTaintIssue;
+import org.sonarsource.sonarlint.core.serverconnection.storage.ProjectServerIssueStore;
 import org.sonarsource.sonarlint.core.serverconnection.storage.UpdateSummary;
 
 import static java.util.stream.Collectors.toSet;
@@ -50,8 +52,16 @@ public ServerIssueUpdater(ConnectionStorage storage, IssueDownloader issueDownlo
 
   public void update(ServerApi serverApi, String projectKey, String branchName, SonarLintCancelMonitor cancelMonitor) {
     if (serverApi.isSonarCloud()) {
+      LOG.debug("Start downloading issues from SonarQube Cloud");
+      var start = Instant.now();
       var issues = issueDownloader.downloadFromBatch(serverApi, projectKey, branchName, cancelMonitor);
-      storage.project(projectKey).findings().replaceAllIssuesOfBranch(branchName, issues);
+      var finishedDownload = Instant.now();
+      LOG.debug("Finished downloading {} issues from SonarQube Cloud in {} ms", issues.size(), finishedDownload.toEpochMilli() - start.toEpochMilli());
+      var issueStore = storage.project(projectKey).findings();
+      LOG.debug("Issue store type: {}", issueStore.getClass().getSimpleName());
+      issueStore.replaceAllIssuesOfBranch(branchName, issues);
+      var finishedWritingToStorage = Instant.now();
+      LOG.debug("Finished updating {} issues in storage in {} ms", issues.size(), finishedWritingToStorage.toEpochMilli() - finishedDownload.toEpochMilli());
     } else {
       sync(serverApi, projectKey, branchName, issueDownloader.getEnabledLanguages(), cancelMonitor);
     }
@@ -61,10 +71,13 @@ public void sync(ServerApi serverApi, String projectKey, String branchName, Set<
     var lastSync = storage.project(projectKey).findings().getLastIssueSyncTimestamp(branchName);
 
     lastSync = computeLastSync(enabledLanguages, lastSync, storage.project(projectKey).findings().getLastIssueEnabledLanguages(branchName));
-
+    Instant start = Instant.now();
     var result = issueDownloader.downloadFromPull(serverApi, projectKey, branchName, lastSync, cancelMonitor);
+    var downloadTime = Instant.now();
+    LOG.debug("Downloaded {} issues took {}", result.getChangedIssues().size(), downloadTime.toEpochMilli() - start.toEpochMilli());
     storage.project(projectKey).findings().mergeIssues(branchName, result.getChangedIssues(), result.getClosedIssueKeys(),
       result.getQueryTimestamp(), enabledLanguages);
+    LOG.debug("Finished updating {} issues in storage in {} ms", result.getChangedIssues().size(), Instant.now().toEpochMilli() - downloadTime.toEpochMilli());
   }
 
   public UpdateSummary<ServerTaintIssue> syncTaints(ServerApi serverApi, String projectKey, String branchName, Set<SonarLanguage> enabledLanguages,

backend/server-connection/src/main/java/org/sonarsource/sonarlint/core/serverconnection/storage/ServerFindingRepository.java

@@ -20,6 +20,7 @@
 package org.sonarsource.sonarlint.core.serverconnection.storage;
 
 import java.nio.file.Path;
+import java.time.Duration;
 import java.time.Instant;
 import java.time.LocalDateTime;
 import java.time.ZoneId;
@@ -34,6 +35,7 @@
 import org.jooq.TableField;
 import org.sonarsource.sonarlint.core.commons.HotspotReviewStatus;
 import org.sonarsource.sonarlint.core.commons.api.SonarLanguage;
+import org.sonarsource.sonarlint.core.commons.log.SonarLintLogger;
 import org.sonarsource.sonarlint.core.commons.storage.SonarLintDatabase;
 import org.sonarsource.sonarlint.core.commons.storage.model.Tables;
 import org.sonarsource.sonarlint.core.commons.storage.model.tables.records.ServerBranchesRecord;
@@ -48,7 +50,7 @@
 import static org.sonarsource.sonarlint.core.commons.storage.model.Tables.SERVER_DEPENDENCY_RISKS;
 
 public class ServerFindingRepository implements ProjectServerIssueStore {
-
+  private static final SonarLintLogger LOG = SonarLintLogger.get();
   private final EntityMapper mapper = new EntityMapper();
   private final SonarLintDatabase database;
   private final String connectionId;
@@ -215,7 +217,7 @@ public Optional<Instant> getLastIssueSyncTimestamp(String branchName) {
         .and(SERVER_BRANCHES.CONNECTION_ID.eq(connectionId))
         .and(SERVER_BRANCHES.SONAR_PROJECT_KEY.eq(sonarProjectKey)))
       .fetchOne();
-    if (rec == null) {
+    if (rec == null || rec.value1() == null) {
       return Optional.empty();
     }
     var ldt = rec.value1();
@@ -245,7 +247,7 @@ public Optional<Instant> getLastTaintSyncTimestamp(String branchName) {
         .and(SERVER_BRANCHES.CONNECTION_ID.eq(connectionId))
         .and(SERVER_BRANCHES.SONAR_PROJECT_KEY.eq(sonarProjectKey)))
       .fetchOne();
-    if (rec == null) {
+    if (rec == null || rec.value1() == null) {
       return Optional.empty();
     }
     var ldt = rec.value1();
@@ -260,7 +262,7 @@ public Optional<Instant> getLastHotspotSyncTimestamp(String branchName) {
         .and(SERVER_BRANCHES.CONNECTION_ID.eq(connectionId))
         .and(SERVER_BRANCHES.SONAR_PROJECT_KEY.eq(sonarProjectKey)))
       .fetchOne();
-    if (rec == null) {
+    if (rec == null || rec.get(0, LocalDateTime.class) == null) {
       return Optional.empty();
     }
     var ldt = rec.get(0, LocalDateTime.class);
@@ -563,9 +565,12 @@ private void batchMergeHotspots(String branchName, String connectionId, String s
   }
 
   private void batchMergeIssues(String branchName, String connectionId, String sonarProjectKey, Configuration trx, Collection<ServerIssue<?>> issues) {
+    LOG.debug("Batch merging {} issues for branch '{}' and project '{}'", issues.size(), branchName, sonarProjectKey);
+    var start = Instant.now();
     trx.dsl().batchMerge(issues.stream()
         .map(issue -> mapper.serverIssueToRecord(issue, branchName, connectionId, sonarProjectKey)).toList())
       .execute();
+    LOG.debug("Batch merge completed in {} ms", Duration.between(start, Instant.now()).toMillis());
   }
 
   private void batchMergeTaints(String branchName, String connectionId, String sonarProjectKey, Configuration trx, Collection<ServerTaintIssue> taints) {

backend/server-connection/src/main/java/org/sonarsource/sonarlint/core/serverconnection/storage/XodusServerIssueStore.java

@@ -484,6 +484,8 @@ public boolean wasEverUpdated() {
 
   @Override
   public void replaceAllIssuesOfBranch(String branchName, List<ServerIssue<?>> issues) {
+    LOG.debug("Replacing all issues of branch {} with {} issues", branchName, issues.size());
+    var start = Instant.now();
     var issuesByFile = issues.stream().collect(Collectors.groupingBy(ServerIssue::getFilePath));
     timed(wroteMessage(issues.size(), ISSUES), () -> entityStore.executeInTransaction(txn -> {
       var branch = getOrCreateBranch(branchName, txn);
@@ -501,6 +503,7 @@ public void replaceAllIssuesOfBranch(String branchName, List<ServerIssue<?>> iss
         txn.flush();
       });
     }));
+    LOG.debug("Finished replacing all issues of branch {} in {} ms", branchName, Duration.between(start, Instant.now()).toMillis());
   }
 
   @Override