Add pagination for search based fetchers

CHANGELOG.md

@@ -21,6 +21,7 @@ Note that this project **does not** adhere to [Semantic Versioning](https://semv
 - We added the field `monthfiled` to the default list of fields to resolve BibTeX-Strings for [#13375](https://github.com/JabRef/jabref/issues/13375)
 - We added a new ID based fetcher for [EuropePMC](https://europepmc.org/). [#13389](https://github.com/JabRef/jabref/pull/13389)
 - We added an initial [cite as you write](https://retorque.re/zotero-better-bibtex/citing/cayw/) endpoint. [#13187](https://github.com/JabRef/jabref/issues/13187)
+- We added pagination support for the web search entries dialog, improving navigation for large search results. [#5507](https://github.com/JabRef/jabref/issues/5507)
 - We added "copy preview as markdown" feature. [#12552](https://github.com/JabRef/jabref/issues/12552)
 - In case no citation relation information can be fetched, we show the data providers reason. [#13549](https://github.com/JabRef/jabref/pull/13549)
 

jabgui/src/main/java/org/jabref/gui/importer/ImportEntriesDialog.java

@@ -6,6 +6,7 @@
 
 import javafx.beans.binding.Bindings;
 import javafx.beans.binding.BooleanBinding;
+import javafx.collections.ListChangeListener;
 import javafx.css.PseudoClass;
 import javafx.fxml.FXML;
 import javafx.geometry.Insets;
@@ -35,7 +36,9 @@
 import org.jabref.gui.util.BaseDialog;
 import org.jabref.gui.util.NoSelectionModel;
 import org.jabref.gui.util.ViewModelListCellFactory;
+import org.jabref.logic.importer.PagedSearchBasedFetcher;
 import org.jabref.logic.importer.ParserResult;
+import org.jabref.logic.importer.SearchBasedFetcher;
 import org.jabref.logic.l10n.Localization;
 import org.jabref.logic.shared.DatabaseLocation;
 import org.jabref.logic.util.BackgroundTask;
@@ -53,7 +56,8 @@
 import org.fxmisc.richtext.CodeArea;
 
 public class ImportEntriesDialog extends BaseDialog<Boolean> {
-
+    @FXML private HBox paginationBox;
+    @FXML private Label pageNumberLabel;
     @FXML private CheckListView<BibEntry> entriesListView;
     @FXML private ComboBox<BibDatabaseContext> libraryListView;
     @FXML private ButtonType importButton;
@@ -68,6 +72,8 @@ public class ImportEntriesDialog extends BaseDialog<Boolean> {
     private final BackgroundTask<ParserResult> task;
     private final BibDatabaseContext database;
     private ImportEntriesViewModel viewModel;
+    private final SearchBasedFetcher searchBasedFetcher;
+    private final String query;
 
     @Inject private TaskExecutor taskExecutor;
     @Inject private DialogService dialogService;
@@ -84,12 +90,23 @@ public class ImportEntriesDialog extends BaseDialog<Boolean> {
      * @param task     the task executed for parsing the selected files(s).
      */
     public ImportEntriesDialog(BibDatabaseContext database, BackgroundTask<ParserResult> task) {
+        this(database, task, null, null);
+    }
+
+    public ImportEntriesDialog(BibDatabaseContext database, BackgroundTask<ParserResult> task, SearchBasedFetcher fetcher, String query) {
         this.database = database;
         this.task = task;
+        this.searchBasedFetcher = fetcher;
+        this.query = query;
+
         ViewLoader.view(this)
                   .load()
                   .setAsDialogPane(this);
 
+        boolean showPagination = (searchBasedFetcher != null) && (query != null);
+        paginationBox.setVisible(showPagination);
+        paginationBox.setManaged(showPagination);
+
         BooleanBinding booleanBind = Bindings.isEmpty(entriesListView.getCheckModel().getCheckedItems());
         Button btn = (Button) this.getDialogPane().lookupButton(importButton);
         btn.disableProperty().bind(booleanBind);
@@ -98,7 +115,7 @@ public ImportEntriesDialog(BibDatabaseContext database, BackgroundTask<ParserRes
 
         setResultConverter(button -> {
             if (button == importButton) {
-                viewModel.importEntries(entriesListView.getCheckModel().getCheckedItems(), downloadLinkedOnlineFiles.isSelected());
+                viewModel.importEntries(viewModel.getCheckedEntries().stream().toList(), downloadLinkedOnlineFiles.isSelected());
             } else {
                 dialogService.notify(Localization.lang("Import canceled"));
             }
@@ -109,11 +126,25 @@ public ImportEntriesDialog(BibDatabaseContext database, BackgroundTask<ParserRes
 
     @FXML
     private void initialize() {
-        viewModel = new ImportEntriesViewModel(task, taskExecutor, database, dialogService, undoManager, preferences, stateManager, entryTypesManager, fileUpdateMonitor);
+        viewModel = new ImportEntriesViewModel(task, taskExecutor, database, dialogService, undoManager, preferences, stateManager, entryTypesManager, fileUpdateMonitor, searchBasedFetcher, query);
         Label placeholder = new Label();
         placeholder.textProperty().bind(viewModel.messageProperty());
         entriesListView.setPlaceholder(placeholder);
         entriesListView.setItems(viewModel.getEntries());
+        entriesListView.getCheckModel().getCheckedItems().addListener((ListChangeListener<BibEntry>) change -> {
+            while (change.next()) {
+                if (change.wasAdded()) {
+                    for (BibEntry entry : change.getAddedSubList()) {
+                        viewModel.getCheckedEntries().add(entry);
+                    }
+                }
+                if (change.wasRemoved()) {
+                    for (BibEntry entry : change.getRemoved()) {
+                        viewModel.getCheckedEntries().remove(entry);
+                    }
+                }
+            }
+        });
 
         libraryListView.setEditable(false);
         libraryListView.getItems().addAll(stateManager.getOpenDatabases());
@@ -180,14 +211,39 @@ private void initialize() {
                 .withPseudoClass(entrySelected, entriesListView::getItemBooleanProperty)
                 .install(entriesListView);
 
-        selectedItems.textProperty().bind(Bindings.size(entriesListView.getCheckModel().getCheckedItems()).asString());
-        totalItems.textProperty().bind(Bindings.size(entriesListView.getItems()).asString());
+        selectedItems.textProperty().bind(Bindings.size(viewModel.getCheckedEntries()).asString());
+        totalItems.textProperty().bind(Bindings.size(viewModel.getAllEntries()).asString());
         entriesListView.setSelectionModel(new NoSelectionModel<>());
         initBibTeX();
+        if (searchBasedFetcher != null) {
+            updatePageUI();
+        }
+    }
+
+    private void updatePageUI() {
+        pageNumberLabel.textProperty().bind(Bindings.createStringBinding(() -> {
+            int totalPages = viewModel.totalPagesProperty().get();
+            int currentPage = viewModel.currentPageProperty().get() + 1;
+            if (searchBasedFetcher instanceof PagedSearchBasedFetcher && currentPage == totalPages) {
+                return Localization.lang("Fetching...");
+            }
+            if (totalPages != 0) {
+                return currentPage + " " + Localization.lang("of") + " " + totalPages;
+            }
+            return "";
+        }, viewModel.currentPageProperty(), viewModel.totalPagesProperty()));
+
+        viewModel.getAllEntries().addListener((ListChangeListener<BibEntry>) change -> {
+            while (change.next()) {
+                if (change.wasAdded() || change.wasRemoved()) {
+                    viewModel.updateTotalPages();
+                }
+            }
+        });
     }
 
     private void displayBibTeX(BibEntry entry, String bibTeX) {
-        if (entriesListView.getCheckModel().isChecked(entry)) {
+        if (viewModel.getCheckedEntries().contains(entry)) {
             bibTeXData.clear();
             bibTeXData.appendText(bibTeX);
             bibTeXData.moveTo(0);
@@ -208,14 +264,18 @@ private void initBibTeX() {
     }
 
     public void unselectAll() {
-        entriesListView.getCheckModel().clearChecks();
+        viewModel.getCheckedEntries().clear();
+        for (int i = 0; i < entriesListView.getItems().size(); i++) {
+            entriesListView.getCheckModel().clearCheck(i);
+        }
     }
 
     public void selectAllNewEntries() {
         unselectAll();
-        for (BibEntry entry : entriesListView.getItems()) {
+        for (BibEntry entry : viewModel.getAllEntries()) {
             if (!viewModel.hasDuplicate(entry)) {
                 entriesListView.getCheckModel().check(entry);
+                viewModel.getCheckedEntries().add(entry);
                 displayBibTeX(entry, viewModel.getSourceString(entry));
             }
         }
@@ -224,5 +284,26 @@ public void selectAllNewEntries() {
     public void selectAllEntries() {
         unselectAll();
         entriesListView.getCheckModel().checkAll();
+        viewModel.getCheckedEntries().addAll(viewModel.getAllEntries());
+    }
+
+    @FXML
+    private void onPrevPage() {
+        viewModel.prevPage();
+        restoreCheckedEntries();
+    }
+
+    @FXML
+    private void onNextPage() {
+        viewModel.nextPage();
+        restoreCheckedEntries();
+    }
+
+    private void restoreCheckedEntries() {
+        for (BibEntry entry : viewModel.getEntries()) {
+            if (viewModel.getCheckedEntries().contains(entry)) {
+                entriesListView.getCheckModel().check(entry);
+            }
+        }
     }
 }

jabgui/src/main/java/org/jabref/gui/importer/ImportEntriesViewModel.java

@@ -2,17 +2,23 @@
 
 import java.io.IOException;
 import java.io.StringWriter;
+import java.util.ArrayList;
 import java.util.List;
 import java.util.Optional;
 
 import javax.swing.undo.UndoManager;
 
+import javafx.beans.property.BooleanProperty;
+import javafx.beans.property.IntegerProperty;
 import javafx.beans.property.ObjectProperty;
+import javafx.beans.property.SimpleBooleanProperty;
+import javafx.beans.property.SimpleIntegerProperty;
 import javafx.beans.property.SimpleObjectProperty;
 import javafx.beans.property.SimpleStringProperty;
 import javafx.beans.property.StringProperty;
 import javafx.collections.FXCollections;
 import javafx.collections.ObservableList;
+import javafx.collections.ObservableSet;
 
 import org.jabref.gui.AbstractViewModel;
 import org.jabref.gui.DialogService;
@@ -24,7 +30,9 @@
 import org.jabref.logic.database.DatabaseMerger;
 import org.jabref.logic.database.DuplicateCheck;
 import org.jabref.logic.exporter.BibWriter;
+import org.jabref.logic.importer.PagedSearchBasedFetcher;
 import org.jabref.logic.importer.ParserResult;
+import org.jabref.logic.importer.SearchBasedFetcher;
 import org.jabref.logic.l10n.Localization;
 import org.jabref.logic.os.OS;
 import org.jabref.logic.util.BackgroundTask;
@@ -40,6 +48,8 @@
 public class ImportEntriesViewModel extends AbstractViewModel {
 
     private static final Logger LOGGER = LoggerFactory.getLogger(ImportEntriesViewModel.class);
+    private static final int PAGE_SIZE = 20;
+    private int CURRENT_PAGE = 0;
 
     private final StringProperty message;
     private final TaskExecutor taskExecutor;
@@ -54,6 +64,15 @@ public class ImportEntriesViewModel extends AbstractViewModel {
     private final BibEntryTypesManager entryTypesManager;
     private final ObjectProperty<BibDatabaseContext> selectedDb;
 
+    private final IntegerProperty currentPageProperty = new SimpleIntegerProperty(0);
+    private final IntegerProperty totalPagesProperty = new SimpleIntegerProperty(0);
+    private final BooleanProperty loading = new SimpleBooleanProperty(false);
+    private final ObservableList<BibEntry> pagedEntries = FXCollections.observableArrayList();
+    private final ObservableSet<BibEntry> checkedEntries = FXCollections.observableSet();
+    private final ObservableList<BibEntry> allEntries = FXCollections.observableArrayList();
+    private final SearchBasedFetcher fetcher;
+    private final String query;
+
     /**
      * @param databaseContext the database to import into
      * @param task            the task executed for parsing the selected files(s).
@@ -66,7 +85,9 @@ public ImportEntriesViewModel(BackgroundTask<ParserResult> task,
                                   GuiPreferences preferences,
                                   StateManager stateManager,
                                   BibEntryTypesManager entryTypesManager,
-                                  FileUpdateMonitor fileUpdateMonitor) {
+                                  FileUpdateMonitor fileUpdateMonitor,
+                                  SearchBasedFetcher fetcher,
+                                  String query) {
         this.taskExecutor = taskExecutor;
         this.databaseContext = databaseContext;
         this.dialogService = dialogService;
@@ -79,12 +100,17 @@ public ImportEntriesViewModel(BackgroundTask<ParserResult> task,
         this.message = new SimpleStringProperty();
         this.message.bind(task.messageProperty());
         this.selectedDb = new SimpleObjectProperty<>();
+        this.fetcher = fetcher;
+        this.query = query;
 
         task.onSuccess(parserResult -> {
             // store the complete parser result (to import groups, ... later on)
             this.parserResult = parserResult;
             // fill in the list for the user, where one can select the entries to import
             entries.addAll(parserResult.getDatabase().getEntries());
+            loadAllEntries(entries);
+            updatePagedEntries();
+            updateTotalPages();
             if (entries.isEmpty()) {
                 task.updateMessage(Localization.lang("No entries corresponding to given query"));
             }
@@ -110,8 +136,25 @@ public BibDatabaseContext getSelectedDb() {
         return selectedDb.get();
     }
 
+    public BooleanProperty loadingProperty() {
+        return loading;
+    }
+
     public ObservableList<BibEntry> getEntries() {
-        return entries;
+        return pagedEntries;
+    }
+
+    public ObservableSet<BibEntry> getCheckedEntries() {
+        return checkedEntries;
+    }
+
+    public ObservableList<BibEntry> getAllEntries() {
+        return allEntries;
+    }
+
+    public void loadAllEntries(List<BibEntry> entries) {
+        allEntries.addAll(entries);
+        this.CURRENT_PAGE = 0;
     }
 
     public boolean hasDuplicate(BibEntry entry) {
@@ -177,4 +220,86 @@ private Optional<BibEntry> findInternalDuplicate(BibEntry entry) {
         }
         return Optional.empty();
     }
+
+    public void prevPage() {
+        if (hasPrevPage()) {
+            currentPageProperty.set(currentPageProperty.get() - 1);
+            CURRENT_PAGE--;
+            updatePagedEntries();
+            updateTotalPages();
+        }
+    }
+
+    public void nextPage() {
+        if (hasNextPage()) {
+            currentPageProperty.set(currentPageProperty.get() + 1);
+            CURRENT_PAGE++;
+            updatePagedEntries();
+            updateTotalPages();
+        }
+    }
+
+    public boolean hasNextPage() {
+        return (CURRENT_PAGE + 1) * PAGE_SIZE < allEntries.size();
+    }
+
+    public boolean hasPrevPage() {
+        return CURRENT_PAGE > 0;
+    }
+
+    public IntegerProperty currentPageProperty() {
+        return currentPageProperty;
+    }
+
+    public IntegerProperty totalPagesProperty() {
+        return totalPagesProperty;
+    }
+
+    public void updateTotalPages() {
+        int total = (int) Math.ceil((double) allEntries.size() / PAGE_SIZE);
+        totalPagesProperty.set(total);
+    }
+
+    private void updatePagedEntries() {
+        if (fetcher == null && query == null) {
+            // For entries other than web search, show all entries at once
+            pagedEntries.setAll(allEntries);
+            return;
+        }
+        if (!loadingProperty().get() && fetcher instanceof PagedSearchBasedFetcher pagedFetcher && (CURRENT_PAGE + 1) * PAGE_SIZE >= allEntries.size()) {
+            loading.set(true);
+            fetchMoreEntries(pagedFetcher);
+        }
+
+        int fromIdx = CURRENT_PAGE * PAGE_SIZE;
+        int toIdx = Math.min(fromIdx + PAGE_SIZE, allEntries.size());
+        pagedEntries.setAll(allEntries.subList(fromIdx, toIdx));
+    }
+
+    private void fetchMoreEntries(PagedSearchBasedFetcher pagedFetcher) {
+        BackgroundTask<ArrayList<BibEntry>> fetchTask = BackgroundTask
+                .wrap(() -> {
+                    LOGGER.info("Fetching entries from {} for page {}", fetcher.getName(), CURRENT_PAGE + 2);
+                    return new ArrayList<>(pagedFetcher.performSearchPaged(query, CURRENT_PAGE + 1).getContent());
+                })
+                .onSuccess(newEntries -> {
+                    if (newEntries != null && !newEntries.isEmpty()) {
+                        allEntries.addAll(newEntries);
+                        updateTotalPages();
+                    } else {
+                        LOGGER.warn("No new entries fetched from {} for page {}", fetcher.getName(), CURRENT_PAGE + 2);
+                    }
+                    loading.set(false);
+                })
+                .onFailure(exception -> {
+                    loading.set(false);
+                    dialogService.showErrorDialogAndWait(
+                            Localization.lang("Error fetching entries"),
+                            Localization.lang("An error occurred while fetching entries from %0: %1",
+                                    fetcher.getName(), exception.getMessage())
+                    );
+                });
+
+        fetchTask.executeWith(taskExecutor);
+    }
 }