Initial implementation of global search (#651)

* initial implementation of global search

* use tokio::sync::mpsc::unbounded_channel instead of Arc, Mutex, Waker poll_fn

* use tokio_stream::wrappers::UnboundedReceiverStream to collect all search matches

* regex_prompt: unified callback; refactor

* global search doc

Cargo.lock

@@ -41,9 +41,17 @@ version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "90682c8d613ad3373e66de8c6411e0ae2ab2571e879d2efbf73558cc66f21279"
 dependencies = [
+ "lazy_static",
  "memchr",
+ "regex-automata",
 ]
 
+[[package]]
+name = "bytecount"
+version = "0.6.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "72feb31ffc86498dacdbd0fcebb56138e7177a8cc5cea4516031d15ae85a742e"
+
 [[package]]
 name = "bytes"
 version = "1.0.1"
@@ -174,6 +182,15 @@ dependencies = [
  "cfg-if",
 ]
 
+[[package]]
+name = "encoding_rs_io"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1cc3c5651fb62ab8aa3103998dade57efdd028544bd300516baa31840c252a83"
+dependencies = [
+ "encoding_rs",
+]
+
 [[package]]
 name = "error-code"
 version = "2.3.0"
@@ -300,6 +317,45 @@ dependencies = [
  "regex",
 ]
 
+[[package]]
+name = "grep-matcher"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6d27563c33062cd33003b166ade2bb4fd82db1fd6a86db764dfdad132d46c1cc"
+dependencies = [
+ "memchr",
+]
+
+[[package]]
+name = "grep-regex"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "121553c9768c363839b92fc2d7cdbbad44a3b70e8d6e7b1b72b05c977527bd06"
+dependencies = [
+ "aho-corasick",
+ "bstr",
+ "grep-matcher",
+ "log",
+ "regex",
+ "regex-syntax",
+ "thread_local",
+]
+
+[[package]]
+name = "grep-searcher"
+version = "0.1.8"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7fbdbde90ba52adc240d2deef7b6ad1f99f53142d074b771fe9b7bede6c4c23d"
+dependencies = [
+ "bstr",
+ "bytecount",
+ "encoding_rs",
+ "encoding_rs_io",
+ "grep-matcher",
+ "log",
+ "memmap2",
+]
+
 [[package]]
 name = "helix-core"
 version = "0.4.1"
@@ -361,6 +417,8 @@ dependencies = [
  "fern",
  "futures-util",
  "fuzzy-matcher",
+ "grep-regex",
+ "grep-searcher",
  "helix-core",
  "helix-lsp",
  "helix-tui",
@@ -375,6 +433,7 @@ dependencies = [
  "signal-hook",
  "signal-hook-tokio",
  "tokio",
+ "tokio-stream",
  "toml",
 ]
 
@@ -552,6 +611,15 @@ version = "2.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
 
+[[package]]
+name = "memmap2"
+version = "0.3.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "00b6c2ebff6180198788f5db08d7ce3bc1d0b617176678831a7510825973e357"
+dependencies = [
+ "libc",
+]
+
 [[package]]
 name = "mio"
 version = "0.7.13"
@@ -753,6 +821,12 @@ dependencies = [
  "regex-syntax",
 ]
 
+[[package]]
+name = "regex-automata"
+version = "0.1.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132"
+
 [[package]]
 name = "regex-syntax"
 version = "0.6.25"

book/src/keymap.md

@@ -206,8 +206,10 @@ This layer is a kludge of mappings, mostly pickers.
 | `y`     | Join and yank selections to clipboard                                 | `yank_joined_to_clipboard`          |
 | `Y`     | Yank main selection to clipboard                                      | `yank_main_selection_to_clipboard`  |
 | `R`     | Replace selections by clipboard contents                              | `replace_selections_with_clipboard` |
+| `/`     | Global search in workspace folder                                     | `global_search`                     |
 
-
+> NOTE: Global search display results in a fuzzy picker, use `space + '` to bring it back up after opening a file.
+ 
 #### Unimpaired
 
 Mappings in the style of [vim-unimpaired](https://github.com/tpope/vim-unimpaired).

helix-term/Cargo.toml

@@ -55,5 +55,10 @@ toml = "0.5"
 serde_json = "1.0"
 serde = { version = "1.0", features = ["derive"] }
 
+# ripgrep for global search
+grep-regex = "0.1.9"
+grep-searcher = "0.1.8"
+tokio-stream = "0.1.7"
+
 [target.'cfg(not(windows))'.dependencies]  # https://github.com/vorner/signal-hook/issues/100
 signal-hook-tokio = { version = "0.3", features = ["futures-v0_3"] }

helix-term/src/commands.rs

@@ -31,7 +31,7 @@ use crate::{
 };
 
 use crate::job::{self, Job, Jobs};
-use futures_util::FutureExt;
+use futures_util::{FutureExt, StreamExt};
 use std::num::NonZeroUsize;
 use std::{fmt, future::Future};
 
@@ -43,6 +43,11 @@ use std::{
 use once_cell::sync::Lazy;
 use serde::de::{self, Deserialize, Deserializer};
 
+use grep_regex::RegexMatcher;
+use grep_searcher::{sinks, BinaryDetection, SearcherBuilder};
+use ignore::{DirEntry, WalkBuilder, WalkState};
+use tokio_stream::wrappers::UnboundedReceiverStream;
+
 pub struct Context<'a> {
     pub register: Option<char>,
     pub count: Option<NonZeroUsize>,
@@ -209,6 +214,7 @@ impl Command {
         search_next, "Select next search match",
         extend_search_next, "Add next search match to selection",
         search_selection, "Use current selection as search pattern",
+        global_search, "Global Search in workspace folder",
         extend_line, "Select current line, if already selected, extend to next line",
         extend_to_line_bounds, "Extend selection to line bounds (line-wise selection)",
         delete_selection, "Delete selection",
@@ -1061,24 +1067,41 @@ fn select_all(cx: &mut Context) {
 
 fn select_regex(cx: &mut Context) {
     let reg = cx.register.unwrap_or('/');
-    let prompt = ui::regex_prompt(cx, "select:".into(), Some(reg), move |view, doc, regex| {
+    let prompt = ui::regex_prompt(
-        let text = doc.text().slice(..);
+        cx,
-        if let Some(selection) = selection::select_on_matches(text, doc.selection(view.id), &regex)
+        "select:".into(),
-        {
+        Some(reg),
-            doc.set_selection(view.id, selection);
+        move |view, doc, regex, event| {
-        }
+            if event != PromptEvent::Update {
-    });
+                return;
+            }
+            let text = doc.text().slice(..);
+            if let Some(selection) =
+                selection::select_on_matches(text, doc.selection(view.id), &regex)
+            {
+                doc.set_selection(view.id, selection);
+            }
+        },
+    );
 
     cx.push_layer(Box::new(prompt));
 }
 
 fn split_selection(cx: &mut Context) {
     let reg = cx.register.unwrap_or('/');
-    let prompt = ui::regex_prompt(cx, "split:".into(), Some(reg), move |view, doc, regex| {
+    let prompt = ui::regex_prompt(
-        let text = doc.text().slice(..);
+        cx,
-        let selection = selection::split_on_matches(text, doc.selection(view.id), &regex);
+        "split:".into(),
-        doc.set_selection(view.id, selection);
+        Some(reg),
-    });
+        move |view, doc, regex, event| {
+            if event != PromptEvent::Update {
+                return;
+            }
+            let text = doc.text().slice(..);
+            let selection = selection::split_on_matches(text, doc.selection(view.id), &regex);
+            doc.set_selection(view.id, selection);
+        },
+    );
 
     cx.push_layer(Box::new(prompt));
 }
@@ -1141,9 +1164,17 @@ fn search(cx: &mut Context) {
     // feed chunks into the regex yet
     let contents = doc.text().slice(..).to_string();
 
-    let prompt = ui::regex_prompt(cx, "search:".into(), Some(reg), move |view, doc, regex| {
+    let prompt = ui::regex_prompt(
-        search_impl(doc, view, &contents, &regex, false);
+        cx,
-    });
+        "search:".into(),
+        Some(reg),
+        move |view, doc, regex, event| {
+            if event != PromptEvent::Update {
+                return;
+            }
+            search_impl(doc, view, &contents, &regex, false);
+        },
+    );
 
     cx.push_layer(Box::new(prompt));
 }
@@ -1192,6 +1223,111 @@ fn search_selection(cx: &mut Context) {
     cx.editor.set_status(msg);
 }
 
+fn global_search(cx: &mut Context) {
+    let (all_matches_sx, all_matches_rx) =
+        tokio::sync::mpsc::unbounded_channel::<(usize, PathBuf)>();
+    let prompt = ui::regex_prompt(
+        cx,
+        "global search:".into(),
+        None,
+        move |_view, _doc, regex, event| {
+            if event != PromptEvent::Validate {
+                return;
+            }
+            if let Ok(matcher) = RegexMatcher::new_line_matcher(regex.as_str()) {
+                let searcher = SearcherBuilder::new()
+                    .binary_detection(BinaryDetection::quit(b'\x00'))
+                    .build();
+
+                let search_root = std::env::current_dir()
+                    .expect("Global search error: Failed to get current dir");
+                WalkBuilder::new(search_root).build_parallel().run(|| {
+                    let mut searcher_cl = searcher.clone();
+                    let matcher_cl = matcher.clone();
+                    let all_matches_sx_cl = all_matches_sx.clone();
+                    Box::new(move |dent: Result<DirEntry, ignore::Error>| -> WalkState {
+                        let dent = match dent {
+                            Ok(dent) => dent,
+                            Err(_) => return WalkState::Continue,
+                        };
+
+                        match dent.file_type() {
+                            Some(fi) => {
+                                if !fi.is_file() {
+                                    return WalkState::Continue;
+                                }
+                            }
+                            None => return WalkState::Continue,
+                        }
+
+                        let result_sink = sinks::UTF8(|line_num, _| {
+                            match all_matches_sx_cl
+                                .send((line_num as usize - 1, dent.path().to_path_buf()))
+                            {
+                                Ok(_) => Ok(true),
+                                Err(_) => Ok(false),
+                            }
+                        });
+                        let result = searcher_cl.search_path(&matcher_cl, dent.path(), result_sink);
+
+                        if let Err(err) = result {
+                            log::error!("Global search error: {}, {}", dent.path().display(), err);
+                        }
+                        WalkState::Continue
+                    })
+                });
+            } else {
+                // Otherwise do nothing
+                // log::warn!("Global Search Invalid Pattern")
+            }
+        },
+    );
+
+    cx.push_layer(Box::new(prompt));
+
+    let show_picker = async move {
+        let all_matches: Vec<(usize, PathBuf)> =
+            UnboundedReceiverStream::new(all_matches_rx).collect().await;
+        let call: job::Callback =
+            Box::new(move |editor: &mut Editor, compositor: &mut Compositor| {
+                if all_matches.is_empty() {
+                    editor.set_status("No matches found".to_string());
+                    return;
+                }
+                let picker = FilePicker::new(
+                    all_matches,
+                    move |(_line_num, path)| path.to_str().unwrap().into(),
+                    move |editor: &mut Editor, (line_num, path), action| {
+                        match editor.open(path.into(), action) {
+                            Ok(_) => {}
+                            Err(e) => {
+                                editor.set_error(format!(
+                                    "Failed to open file '{}': {}",
+                                    path.display(),
+                                    e
+                                ));
+                                return;
+                            }
+                        }
+
+                        let line_num = *line_num;
+                        let (view, doc) = current!(editor);
+                        let text = doc.text();
+                        let start = text.line_to_char(line_num);
+                        let end = text.line_to_char((line_num + 1).min(text.len_lines()));
+
+                        doc.set_selection(view.id, Selection::single(start, end));
+                        align_view(doc, view, Align::Center);
+                    },
+                    |_editor, (line_num, path)| Some((path.clone(), Some((*line_num, *line_num)))),
+                );
+                compositor.push(Box::new(picker));
+            });
+        Ok(call)
+    };
+    cx.jobs.callback(show_picker);
+}
+
 fn extend_line(cx: &mut Context) {
     let count = cx.count();
     let (view, doc) = current!(cx.editor);
@@ -3847,13 +3983,21 @@ fn join_selections(cx: &mut Context) {
 fn keep_selections(cx: &mut Context) {
     // keep selections matching regex
     let reg = cx.register.unwrap_or('/');
-    let prompt = ui::regex_prompt(cx, "keep:".into(), Some(reg), move |view, doc, regex| {
+    let prompt = ui::regex_prompt(
-        let text = doc.text().slice(..);
+        cx,
+        "keep:".into(),
+        Some(reg),
+        move |view, doc, regex, event| {
+            if event != PromptEvent::Update {
+                return;
+            }
+            let text = doc.text().slice(..);
 
-        if let Some(selection) = selection::keep_matches(text, doc.selection(view.id), &regex) {
+            if let Some(selection) = selection::keep_matches(text, doc.selection(view.id), &regex) {
-            doc.set_selection(view.id, selection);
+                doc.set_selection(view.id, selection);
-        }
+            }
-    });
+        },
+    );
 
     cx.push_layer(Box::new(prompt));
 }

helix-term/src/keymap.rs

@@ -555,6 +555,7 @@ impl Default for Keymaps {
                 "P" => paste_clipboard_before,
                 "R" => replace_selections_with_clipboard,
                 "space" => keep_primary_selection,
+                "/" => global_search,
             },
             "z" => { "View"
                 "z" | "c" => align_view_center,

helix-term/src/ui/mod.rs

@@ -29,7 +29,7 @@ pub fn regex_prompt(
     cx: &mut crate::commands::Context,
     prompt: std::borrow::Cow<'static, str>,
     history_register: Option<char>,
-    fun: impl Fn(&mut View, &mut Document, Regex) + 'static,
+    fun: impl Fn(&mut View, &mut Document, Regex, PromptEvent) + 'static,
 ) -> Prompt {
     let (view, doc) = current!(cx.editor);
     let view_id = view.id;
@@ -47,6 +47,14 @@ pub fn regex_prompt(
                 }
                 PromptEvent::Validate => {
                     // TODO: push_jump to store selection just before jump
+
+                    match Regex::new(input) {
+                        Ok(regex) => {
+                            let (view, doc) = current!(cx.editor);
+                            fun(view, doc, regex, event);
+                        }
+                        Err(_err) => (), // TODO: mark command line as error
+                    }
                 }
                 PromptEvent::Update => {
                     // skip empty input, TODO: trigger default
@@ -70,7 +78,7 @@ pub fn regex_prompt(
                             // revert state to what it was before the last update
                             doc.set_selection(view.id, snapshot.clone());
 
-                            fun(view, doc, regex);
+                            fun(view, doc, regex, event);
 
                             view.ensure_cursor_in_view(doc, cx.editor.config.scrolloff);
                         }