Merge a23b0bc3a9 into 4281228da3

Co-authored-by: Michael Davis <mcarsondavis@gmail.com>

helix-core/src/comment.rs

@@ -4,43 +4,98 @@
 use smallvec::SmallVec;
 
 use crate::{
-    syntax::config::BlockCommentToken, Change, Range, Rope, RopeSlice, Selection, Tendril,
+    syntax::{self, config::BlockCommentToken},
-    Transaction,
+    Change, Range, Rope, RopeSlice, Syntax, Tendril,
 };
 use helix_stdx::rope::RopeSliceExt;
 use std::borrow::Cow;
 
 pub const DEFAULT_COMMENT_TOKEN: &str = "#";
 
-/// Returns the longest matching comment token of the given line (if it exists).
+/// Returns the longest matching line comment token of the given line (if it exists).
-pub fn get_comment_token<'a, S: AsRef<str>>(
+pub fn get_line_comment_token(
+    loader: &syntax::Loader,
+    syntax: Option<&Syntax>,
     text: RopeSlice,
-    tokens: &'a [S],
+    doc_default_tokens: Option<&[String]>,
     line_num: usize,
-) -> Option<&'a str> {
+) -> Option<String> {
     let line = text.line(line_num);
     let start = line.first_non_whitespace_char()?;
+    let start_char = text.line_to_char(line_num) + start;
 
-    tokens
+    let injected_line_comment_tokens =
-        .iter()
+        injected_tokens_for_range(loader, syntax, start_char as u32, start_char as u32)
-        .map(AsRef::as_ref)
+            .0
-        .filter(|token| line.slice(start..).starts_with(token))
+            .and_then(|tokens| {
-        .max_by_key(|token| token.len())
+                tokens
+                    .into_iter()
+                    .filter(|token| line.slice(start..).starts_with(token))
+                    .max_by_key(|token| token.len())
+            });
+
+    injected_line_comment_tokens.or_else(||
+        // no line comment tokens found for injection, use doc comments if exists
+        doc_default_tokens.and_then(|tokens| {
+            tokens
+                .iter()
+                .filter(|token| line.slice(start..).starts_with(token))
+                .max_by_key(|token| token.len())
+                .cloned()
+        }))
 }
 
-/// Given text, a comment token, and a set of line indices, returns the following:
+/// Get the injected line and block comment of the smallest
-/// - Whether the given lines should be considered commented
+/// injection around the range which fully includes `start..=end`.
+///  
+/// Injections that do not have any comment tokens are skipped.
+pub fn injected_tokens_for_range(
+    loader: &syntax::Loader,
+    syntax: Option<&Syntax>,
+    start: u32,
+    end: u32,
+) -> (Option<Vec<String>>, Option<Vec<BlockCommentToken>>) {
+    syntax
+        .and_then(|syntax| {
+            syntax
+                .layers_for_byte_range(start, end)
+                .collect::<Vec<_>>()
+                .into_iter()
+                .rev()
+                .find_map(|layer| {
+                    let lang_config = loader.language(syntax.layer(layer).language).config();
+
+                    let has_any_comment_tokens = lang_config.comment_tokens.is_some()
+                        || lang_config.block_comment_tokens.is_some();
+
+                    // if the language does not have any comment tokens, it does not make
+                    // any sense to consider it.
+                    //
+                    // This includes languages such as `comment`, `jsdoc` and `regex`.
+                    // These languages are injected and never found in files by themselves
+                    has_any_comment_tokens.then_some((
+                        lang_config.comment_tokens.clone(),
+                        lang_config.block_comment_tokens.clone(),
+                    ))
+                })
+        })
+        .unwrap_or_default()
+}
+
+/// Given `text`, a comment `token`, and a set of line indices `lines_to_modify`,
+/// Returns the following:
+/// 1. Whether the given lines should be considered commented
 ///     - If any of the lines are uncommented, all lines are considered as such.
-/// - The lines to change for toggling comments
+/// 2. The lines to change for toggling comments
 ///     - This is all provided lines excluding blanks lines.
-/// - The column of the comment tokens
+/// 3. The column of the comment tokens
 ///     - Column of existing tokens, if the lines are commented; column to place tokens at otherwise.
-/// - The margin to the right of the comment tokens
+/// 4. The margin to the right of the comment tokens
 ///     - Defaults to `1`. If any existing comment token is not followed by a space, changes to `0`.
 fn find_line_comment(
     token: &str,
     text: RopeSlice,
-    lines: impl IntoIterator<Item = usize>,
+    lines_to_modify: impl IntoIterator<Item = usize>,
 ) -> (bool, Vec<usize>, usize, usize) {
     let mut commented = true;
     let mut to_change = Vec::new();
@@ -48,7 +103,7 @@ fn find_line_comment(
     let mut margin = 1;
     let token_len = token.chars().count();
 
-    for line in lines {
+    for line in lines_to_modify {
         let line_slice = text.line(line);
         if let Some(pos) = line_slice.first_non_whitespace_char() {
             let len = line_slice.len_chars();
@@ -78,42 +133,55 @@ fn find_line_comment(
     (commented, to_change, min, margin)
 }
 
+/// Returns the edits required to toggle the comment `token` for the `range` in the `doc`
 #[must_use]
-pub fn toggle_line_comments(doc: &Rope, selection: &Selection, token: Option<&str>) -> Transaction {
+pub fn toggle_line_comments(doc: &Rope, range: &Range, token: Option<&str>) -> Vec<Change> {
     let text = doc.slice(..);
 
     let token = token.unwrap_or(DEFAULT_COMMENT_TOKEN);
+
+    // Add a space between the comment token and the line.
     let comment = Tendril::from(format!("{} ", token));
 
-    let mut lines: Vec<usize> = Vec::with_capacity(selection.len());
+    let line_count = text.len_lines();
 
-    let mut min_next_line = 0;
+    let start = text.char_to_line(range.from()).clamp(0, line_count);
-    for selection in selection {
+    let end = (text.char_to_line(range.to().saturating_sub(1)) + 1).min(line_count);
-        let (start, end) = selection.line_range(text);
-        let start = start.clamp(min_next_line, text.len_lines());
-        let end = (end + 1).min(text.len_lines());
 
-        lines.extend(start..end);
+    let lines_to_modify = start..end;
-        min_next_line = end;
-    }
 
-    let (commented, to_change, min, margin) = find_line_comment(token, text, lines);
+    let (
+        was_commented,
+        lines_to_modify,
+        column_to_place_comment_tokens_at,
+        comment_tokens_right_margin,
+    ) = find_line_comment(token, text, lines_to_modify);
 
-    let mut changes: Vec<Change> = Vec::with_capacity(to_change.len());
+    lines_to_modify
+        .into_iter()
+        .map(|line| {
+            let place_comment_tokens_at =
+                text.line_to_char(line) + column_to_place_comment_tokens_at;
 
-    for line in to_change {
+            if !was_commented {
-        let pos = text.line_to_char(line) + min;
+                // comment line
-
+                (
-        if !commented {
+                    place_comment_tokens_at,
-            // comment line
+                    place_comment_tokens_at,
-            changes.push((pos, pos, Some(comment.clone())));
+                    // insert the token
-        } else {
+                    Some(comment.clone()),
-            // uncomment line
+                )
-            changes.push((pos, pos + token.len() + margin, None));
+            } else {
-        }
+                // uncomment line
-    }
+                (
-
+                    place_comment_tokens_at,
-    Transaction::change(doc, changes.into_iter())
+                    place_comment_tokens_at + token.len() + comment_tokens_right_margin,
+                    // remove the token - replace range with nothing
+                    None,
+                )
+            }
+        })
+        .collect()
 }
 
 #[derive(Debug, PartialEq, Eq)]
@@ -142,11 +210,11 @@ pub enum CommentChange {
 pub fn find_block_comments(
     tokens: &[BlockCommentToken],
     text: RopeSlice,
-    selection: &Selection,
+    ranges: &[Range],
 ) -> (bool, Vec<CommentChange>) {
-    let mut commented = true;
+    let mut was_commented = true;
     let mut only_whitespace = true;
-    let mut comment_changes = Vec::with_capacity(selection.len());
+    let mut comment_changes = Vec::with_capacity(ranges.len());
     let default_tokens = tokens.first().cloned().unwrap_or_default();
     let mut start_token = default_tokens.start.clone();
     let mut end_token = default_tokens.end.clone();
@@ -160,7 +228,7 @@ pub fn find_block_comments(
             b.start.len().cmp(&a.start.len())
         }
     });
-    for range in selection {
+    for range in ranges {
         let selection_slice = range.slice(text);
         if let (Some(start_pos), Some(end_pos)) = (
             selection_slice.first_non_whitespace_char(),
@@ -199,7 +267,7 @@ pub fn find_block_comments(
                     start_token: default_tokens.start.clone(),
                     end_token: default_tokens.end.clone(),
                 });
-                commented = false;
+                was_commented = false;
             } else {
                 comment_changes.push(CommentChange::Commented {
                     range: *range,
@@ -218,23 +286,22 @@ pub fn find_block_comments(
         }
     }
     if only_whitespace {
-        commented = false;
+        was_commented = false;
     }
-    (commented, comment_changes)
+    (was_commented, comment_changes)
 }
 
 #[must_use]
 pub fn create_block_comment_transaction(
-    doc: &Rope,
+    ranges: &[Range],
-    selection: &Selection,
+    was_commented: bool,
-    commented: bool,
     comment_changes: Vec<CommentChange>,
-) -> (Transaction, SmallVec<[Range; 1]>) {
+) -> (Vec<Change>, SmallVec<[Range; 1]>) {
-    let mut changes: Vec<Change> = Vec::with_capacity(selection.len() * 2);
+    let mut changes: Vec<Change> = Vec::with_capacity(ranges.len() * 2);
-    let mut ranges: SmallVec<[Range; 1]> = SmallVec::with_capacity(selection.len());
+    let mut ranges: SmallVec<[Range; 1]> = SmallVec::with_capacity(ranges.len());
     let mut offs = 0;
     for change in comment_changes {
-        if commented {
+        if was_commented {
             if let CommentChange::Commented {
                 range,
                 start_pos,
@@ -246,16 +313,12 @@ pub fn create_block_comment_transaction(
             } = change
             {
                 let from = range.from();
-                changes.push((
+                let keep_from = from + start_pos + start_token.len() + start_margin as usize;
-                    from + start_pos,
+                changes.push((from + start_pos, keep_from, None));
-                    from + start_pos + start_token.len() + start_margin as usize,
+                let keep_until = from + end_pos - end_token.len() - end_margin as usize + 1;
-                    None,
+                changes.push((keep_until, from + end_pos + 1, None));
-                ));
+                // The range of characters keep_from..keep_until remain in the document
-                changes.push((
+                ranges.push(Range::new(keep_from, keep_until).with_direction(range.direction()));
-                    from + end_pos - end_token.len() - end_margin as usize + 1,
-                    from + end_pos + 1,
-                    None,
-                ));
             }
         } else {
             // uncommented so manually map ranges through changes
@@ -292,37 +355,92 @@ pub fn create_block_comment_transaction(
             }
         }
     }
-    (Transaction::change(doc, changes.into_iter()), ranges)
+    (changes, ranges)
 }
 
 #[must_use]
 pub fn toggle_block_comments(
     doc: &Rope,
-    selection: &Selection,
+    ranges: &[Range],
     tokens: &[BlockCommentToken],
-) -> Transaction {
+    selections: &mut SmallVec<[Range; 1]>,
+    added_chars: &mut usize,
+    removed_chars: &mut usize,
+) -> Vec<Change> {
     let text = doc.slice(..);
-    let (commented, comment_changes) = find_block_comments(tokens, text, selection);
+    let (was_commented, comment_changes) = find_block_comments(tokens, text, ranges);
-    let (mut transaction, ranges) =
+    let (changes, new_ranges) =
-        create_block_comment_transaction(doc, selection, commented, comment_changes);
+        create_block_comment_transaction(ranges, was_commented, comment_changes);
-    if !commented {
+
-        transaction = transaction.with_selection(Selection::new(ranges, selection.primary_index()));
+    if was_commented {
+        for (range, changes) in new_ranges.iter().zip(changes.chunks_exact(2)) {
+            // every 2 elements (from, to) in `changes` corresponds
+            // the `from` - `to` represents the range of text that will be deleted.
+            // to 1 element in `new_ranges`
+            //
+            // Left token:
+            //
+            // "<!-- "
+            //  ^ left_from
+            //       ^ left_to
+            //
+            // Right token:
+            //
+            // " -->"
+            //  ^ right_from
+            //      ^ right_to
+            let [(left_from, left_to, _), (right_from, right_to, _)] = changes else {
+                unreachable!()
+            };
+
+            *removed_chars += left_to - left_from;
+
+            // We slide the range to the left by the amount of characters
+            // we've deleted so far + the amount of chars deleted for
+            // the left comment token of the current iteration
+            selections.push(Range::new(
+                range.anchor + *added_chars - *removed_chars,
+                range.head + *added_chars - *removed_chars,
+            ));
+
+            *removed_chars += right_to - right_from;
+        }
+
+        changes
+    } else {
+        // we're never removing or
+        // creating ranges. Only shifting / increasing size
+        // of existing ranges to accomodate the newly added
+        // comment tokens.
+        //
+        // when we add comment tokens, we want to extend our selection to
+        // also include the added tokens.
+        for (range, old_range) in new_ranges.iter().zip(ranges) {
+            // Will not underflow because the new range must always be
+            // at least the same size as the old range, since we're
+            // adding comment token characters, never removing.
+            let range = Range::new(
+                range.anchor + *added_chars - *removed_chars,
+                range.head + *added_chars - *removed_chars,
+            );
+            selections.push(range);
+            *added_chars += range.len() - old_range.len();
+        }
+
+        changes
     }
-    transaction
 }
 
-pub fn split_lines_of_selection(text: RopeSlice, selection: &Selection) -> Selection {
+pub fn split_lines_of_range(text: RopeSlice, range: &Range) -> Vec<Range> {
-    let mut ranges = SmallVec::new();
+    let mut ranges = vec![];
-    for range in selection.ranges() {
+    let (line_start, line_end) = range.line_range(text.slice(..));
-        let (line_start, line_end) = range.line_range(text.slice(..));
+    let mut pos = text.line_to_char(line_start);
-        let mut pos = text.line_to_char(line_start);
+    for line in text.slice(pos..text.line_to_char(line_end + 1)).lines() {
-        for line in text.slice(pos..text.line_to_char(line_end + 1)).lines() {
+        let start = pos;
-            let start = pos;
+        pos += line.len_chars();
-            pos += line.len_chars();
+        ranges.push(Range::new(start, pos));
-            ranges.push(Range::new(start, pos));
-        }
     }
-    Selection::new(ranges, 0)
+    ranges
 }
 
 #[cfg(test)]
@@ -358,6 +476,8 @@ mod test {
 
     // TODO: account for uncommenting with uneven comment indentation
     mod toggle_line_comment {
+        use crate::Transaction;
+
         use super::*;
 
         #[test]
@@ -365,9 +485,11 @@ mod test {
             // four lines, two space indented, except for line 1 which is blank.
             let mut doc = Rope::from("  1\n\n  2\n  3");
             // select whole document
-            let selection = Selection::single(0, doc.len_chars() - 1);
+            let range = Range::new(0, doc.len_chars() - 1);
+
+            let changes = toggle_line_comments(&doc, &range, None);
+            let transaction = Transaction::change(&doc, changes.into_iter());
 
-            let transaction = toggle_line_comments(&doc, &selection, None);
             transaction.apply(&mut doc);
 
             assert_eq!(doc, "  # 1\n\n  # 2\n  # 3");
@@ -376,112 +498,108 @@ mod test {
         #[test]
         fn uncomment() {
             let mut doc = Rope::from("  # 1\n\n  # 2\n  # 3");
-            let mut selection = Selection::single(0, doc.len_chars() - 1);
+            let range = Range::new(0, doc.len_chars() - 1);
 
-            let transaction = toggle_line_comments(&doc, &selection, None);
+            let changes = toggle_line_comments(&doc, &range, None);
+            let transaction = Transaction::change(&doc, changes.into_iter());
             transaction.apply(&mut doc);
-            selection = selection.map(transaction.changes());
+            _ = range.map(transaction.changes());
 
             assert_eq!(doc, "  1\n\n  2\n  3");
-            assert!(selection.len() == 1); // to ignore the selection unused warning
         }
 
         #[test]
         fn uncomment_0_margin_comments() {
             let mut doc = Rope::from("  #1\n\n  #2\n  #3");
-            let mut selection = Selection::single(0, doc.len_chars() - 1);
+            let range = Range::new(0, doc.len_chars() - 1);
 
-            let transaction = toggle_line_comments(&doc, &selection, None);
+            let changes = toggle_line_comments(&doc, &range, None);
+            let transaction = Transaction::change(&doc, changes.into_iter());
             transaction.apply(&mut doc);
-            selection = selection.map(transaction.changes());
+            _ = range.map(transaction.changes());
 
             assert_eq!(doc, "  1\n\n  2\n  3");
-            assert!(selection.len() == 1); // to ignore the selection unused warning
         }
 
         #[test]
         fn uncomment_0_margin_comments_with_no_space() {
             let mut doc = Rope::from("#");
-            let mut selection = Selection::single(0, doc.len_chars() - 1);
+            let range = Range::new(0, doc.len_chars() - 1);
 
-            let transaction = toggle_line_comments(&doc, &selection, None);
+            let changes = toggle_line_comments(&doc, &range, None);
+            let transaction = Transaction::change(&doc, changes.into_iter());
+            transaction.apply(&mut doc);
+            _ = range.map(transaction.changes());
+            assert_eq!(doc, "");
+        }
+
+        #[test]
+        fn test_find_block_comments() {
+            // three lines 5 characters.
+            let mut doc = Rope::from("1\n2\n3");
+            // select whole document
+            let range = Range::new(0, doc.len_chars());
+
+            let text = doc.slice(..);
+
+            let res = find_block_comments(&[BlockCommentToken::default()], text, &[range]);
+
+            assert_eq!(
+                res,
+                (
+                    false,
+                    vec![CommentChange::Uncommented {
+                        range: Range::new(0, 5),
+                        start_pos: 0,
+                        end_pos: 4,
+                        start_token: "/*".to_string(),
+                        end_token: "*/".to_string(),
+                    }]
+                )
+            );
+
+            // comment
+            let changes = toggle_block_comments(
+                &doc,
+                &[range],
+                &[BlockCommentToken::default()],
+                &mut SmallVec::new(),
+                &mut 0,
+                &mut 0,
+            );
+            let transaction = Transaction::change(&doc, changes.into_iter());
+            transaction.apply(&mut doc);
+
+            assert_eq!(doc, "/* 1\n2\n3 */");
+
+            // uncomment
+            let range = Range::new(0, doc.len_chars());
+            let changes = toggle_block_comments(
+                &doc,
+                &[range],
+                &[BlockCommentToken::default()],
+                &mut SmallVec::new(),
+                &mut 0,
+                &mut 0,
+            );
+            let transaction = Transaction::change(&doc, changes.into_iter());
+            transaction.apply(&mut doc);
+            assert_eq!(doc, "1\n2\n3");
+
+            // don't panic when there is just a space in comment
+            doc = Rope::from("/* */");
+            let range = Range::new(0, doc.len_chars());
+            let changes = toggle_block_comments(
+                &doc,
+                &[range],
+                &[BlockCommentToken::default()],
+                &mut SmallVec::new(),
+                &mut 0,
+                &mut 0,
+            );
+            let transaction = Transaction::change(&doc, changes.into_iter());
             transaction.apply(&mut doc);
-            selection = selection.map(transaction.changes());
             assert_eq!(doc, "");
-            assert!(selection.len() == 1); // to ignore the selection unused warning
         }
     }
-
-    #[test]
-    fn test_find_block_comments() {
-        // three lines 5 characters.
-        let mut doc = Rope::from("1\n2\n3");
-        // select whole document
-        let selection = Selection::single(0, doc.len_chars());
-
-        let text = doc.slice(..);
-
-        let res = find_block_comments(&[BlockCommentToken::default()], text, &selection);
-
-        assert_eq!(
-            res,
-            (
-                false,
-                vec![CommentChange::Uncommented {
-                    range: Range::new(0, 5),
-                    start_pos: 0,
-                    end_pos: 4,
-                    start_token: "/*".to_string(),
-                    end_token: "*/".to_string(),
-                }]
-            )
-        );
-
-        // comment
-        let transaction = toggle_block_comments(&doc, &selection, &[BlockCommentToken::default()]);
-        transaction.apply(&mut doc);
-
-        assert_eq!(doc, "/* 1\n2\n3 */");
-
-        // uncomment
-        let selection = Selection::single(0, doc.len_chars());
-        let transaction = toggle_block_comments(&doc, &selection, &[BlockCommentToken::default()]);
-        transaction.apply(&mut doc);
-        assert_eq!(doc, "1\n2\n3");
-
-        // don't panic when there is just a space in comment
-        doc = Rope::from("/* */");
-        let selection = Selection::single(0, doc.len_chars());
-        let transaction = toggle_block_comments(&doc, &selection, &[BlockCommentToken::default()]);
-        transaction.apply(&mut doc);
-        assert_eq!(doc, "");
-    }
-
-    /// Test, if `get_comment_tokens` works, even if the content of the file includes chars, whose
-    /// byte size unequal the amount of chars
-    #[test]
-    fn test_get_comment_with_char_boundaries() {
-        let rope = Rope::from("··");
-        let tokens = ["//", "///"];
-
-        assert_eq!(
-            super::get_comment_token(rope.slice(..), tokens.as_slice(), 0),
-            None
-        );
-    }
-
-    /// Test for `get_comment_token`.
-    ///
-    /// Assuming the comment tokens are stored as `["///", "//"]`, `get_comment_token` should still
-    /// return `///` instead of `//` if the user is in a doc-comment section.
-    #[test]
-    fn test_use_longest_comment() {
-        let text = Rope::from("    /// amogus");
-        let tokens = ["///", "//"];
-
-        assert_eq!(
-            super::get_comment_token(text.slice(..), tokens.as_slice(), 0),
-            Some("///")
-        );
-    }
 }

helix-core/src/syntax.rs

@@ -513,6 +513,14 @@ impl Syntax {
         self.inner.layer_for_byte_range(start, end)
     }
 
+    pub fn layers_for_byte_range(
+        &self,
+        start: u32,
+        end: u32,
+    ) -> impl Iterator<Item = Layer> + use<'_> {
+        self.inner.layers_for_byte_range(start, end)
+    }
+
     pub fn root_language(&self) -> Language {
         self.layer(self.root_layer()).language
     }

helix-term/src/commands.rs

@@ -22,7 +22,8 @@ pub use typed::*;
 use helix_core::{
     char_idx_at_visual_offset,
     chars::char_is_word,
-    command_line, comment,
+    command_line,
+    comment::{self},
     doc_formatter::TextFormat,
     encoding, find_workspace,
     graphemes::{self, next_grapheme_boundary},
@@ -36,7 +37,10 @@ use helix_core::{
     regex::{self, Regex},
     search::{self, CharMatcher},
     selection, surround,
-    syntax::config::{BlockCommentToken, LanguageServerFeature},
+    syntax::{
+        self,
+        config::{BlockCommentToken, LanguageServerFeature},
+    },
     text_annotations::{Overlay, TextAnnotations},
     textobject,
     unicode::width::UnicodeWidthChar,
@@ -3653,13 +3657,14 @@ fn open(cx: &mut Context, open: Open, comment_continuation: CommentContinuation)
 
     let mut ranges = SmallVec::with_capacity(selection.len());
 
-    let continue_comment_tokens =
+    let continue_comments =
-        if comment_continuation == CommentContinuation::Enabled && config.continue_comments {
+        comment_continuation == CommentContinuation::Enabled && config.continue_comments;
-            doc.language_config()
+
-                .and_then(|config| config.comment_tokens.as_ref())
+    let doc_default_tokens = doc
-        } else {
+        .language_config()
-            None
+        .and_then(|config| config.comment_tokens.as_deref());
-        };
+
+    let syntax = doc.syntax();
 
     let mut transaction = Transaction::change_by_selection(contents, selection, |range| {
         // the line number, where the cursor is currently
@@ -3676,8 +3681,17 @@ fn open(cx: &mut Context, open: Open, comment_continuation: CommentContinuation)
 
         let above_next_new_line_num = next_new_line_num.saturating_sub(1);
 
-        let continue_comment_token = continue_comment_tokens
+        let continue_comment_token = continue_comments
-            .and_then(|tokens| comment::get_comment_token(text, tokens, curr_line_num));
+            .then(|| {
+                comment::get_line_comment_token(
+                    &loader,
+                    syntax,
+                    text,
+                    doc_default_tokens,
+                    curr_line_num,
+                )
+            })
+            .flatten();
 
         // Index to insert newlines after, as well as the char width
         // to use to compensate for those inserted newlines.
@@ -3711,7 +3725,7 @@ fn open(cx: &mut Context, open: Open, comment_continuation: CommentContinuation)
 
         if open == Open::Above && next_new_line_num == 0 {
             text.push_str(&indent);
-            if let Some(token) = continue_comment_token {
+            if let Some(ref token) = continue_comment_token {
                 text.push_str(token);
                 text.push(' ');
             }
@@ -3720,7 +3734,7 @@ fn open(cx: &mut Context, open: Open, comment_continuation: CommentContinuation)
             text.push_str(doc.line_ending.as_str());
             text.push_str(&indent);
 
-            if let Some(token) = continue_comment_token {
+            if let Some(ref token) = continue_comment_token {
                 text.push_str(token);
                 text.push(' ');
             }
@@ -4211,12 +4225,11 @@ pub mod insert {
         let mut global_offs = 0;
         let mut new_text = String::new();
 
-        let continue_comment_tokens = if config.continue_comments {
+        let doc_default_comment_token = doc
-            doc.language_config()
+            .language_config()
-                .and_then(|config| config.comment_tokens.as_ref())
+            .and_then(|config| config.comment_tokens.as_deref());
-        } else {
+
-            None
+        let syntax = doc.syntax();
-        };
 
         let mut last_pos = 0;
         let mut transaction = Transaction::change_by_selection(contents, selection, |range| {
@@ -4234,8 +4247,14 @@ pub mod insert {
             let current_line = text.char_to_line(pos);
             let line_start = text.line_to_char(current_line);
 
-            let continue_comment_token = continue_comment_tokens
+            let continue_comment_token = comment::get_line_comment_token(
-                .and_then(|tokens| comment::get_comment_token(text, tokens, current_line));
+                &cx.editor.syn_loader.load(),
+                syntax,
+                text,
+                doc_default_comment_token,
+                current_line,
+            )
+            .filter(|_| config.continue_comments);
 
             let (from, to, local_offs) = if let Some(idx) =
                 text.slice(line_start..pos).last_non_whitespace_char()
@@ -4271,7 +4290,7 @@ pub mod insert {
                     new_text.reserve_exact(line_ending.len() + indent.len() + token.len() + 1);
                     new_text.push_str(line_ending);
                     new_text.push_str(&indent);
-                    new_text.push_str(token);
+                    new_text.push_str(&token);
                     new_text.push(' ');
                     new_text.chars().count()
                 } else if on_auto_pair {
@@ -5167,122 +5186,239 @@ pub fn completion(cx: &mut Context) {
 }
 
 // comments
-type CommentTransactionFn = fn(
+
-    line_token: Option<&str>,
+/// Perform a `Transaction` to toggle comments
-    block_tokens: Option<&[BlockCommentToken]>,
+type CommentTransaction = fn(
-    doc: &Rope,
+    text: &Rope,
     selection: &Selection,
+    doc_line_token: Option<&str>,
+    doc_block_tokens: Option<&[BlockCommentToken]>,
+    syntax: Option<&Syntax>,
+    loader: &syntax::Loader,
 ) -> Transaction;
 
-fn toggle_comments_impl(cx: &mut Context, comment_transaction: CommentTransactionFn) {
+/// Commenting behavior, for each range in selection:
+///
+/// 1. Only line comment tokens -> line comment
+/// 2. Each line block commented -> uncomment all lines
+/// 3. Whole selection block commented -> uncomment selection
+/// 4. All lines not commented and block tokens -> comment uncommented lines
+/// 5. No comment tokens and not block commented -> line comment
+fn toggle_comments_impl(cx: &mut Context, comments_transaction: CommentTransaction) {
     let (view, doc) = current!(cx.editor);
-    let line_token: Option<&str> = doc
+    let syntax = doc.syntax();
+    let rope = doc.text();
+    let selection = doc.selection(view.id);
+
+    // The comment tokens to fallback to if no comment tokens are found for the injection layer.
+    let doc_line_token: Option<&str> = doc
         .language_config()
         .and_then(|lc| lc.comment_tokens.as_ref())
         .and_then(|tc| tc.first())
         .map(|tc| tc.as_str());
-    let block_tokens: Option<&[BlockCommentToken]> = doc
+    let doc_block_tokens: Option<&[BlockCommentToken]> = doc
         .language_config()
         .and_then(|lc| lc.block_comment_tokens.as_ref())
         .map(|tc| &tc[..]);
 
-    let transaction =
+    // Call the custom logic provided by the caller (the original functions).
-        comment_transaction(line_token, block_tokens, doc.text(), doc.selection(view.id));
+    let transaction = comments_transaction(
+        rope,
+        selection,
+        doc_line_token,
+        doc_block_tokens,
+        syntax,
+        &cx.editor.syn_loader.load(),
+    );
 
     doc.apply(&transaction, view.id);
     exit_select_mode(cx);
 }
 
-/// commenting behavior:
-/// 1. only line comment tokens -> line comment
-/// 2. each line block commented -> uncomment all lines
-/// 3. whole selection block commented -> uncomment selection
-/// 4. all lines not commented and block tokens -> comment uncommented lines
-/// 5. no comment tokens and not block commented -> line comment
 fn toggle_comments(cx: &mut Context) {
-    toggle_comments_impl(cx, |line_token, block_tokens, doc, selection| {
+    toggle_comments_impl(
-        let text = doc.slice(..);
+        cx,
+        |rope, selection, doc_line_token, doc_block_tokens, syntax, loader| {
+            Transaction::change(
+                rope,
+                selection.iter().flat_map(|range| {
+                    let (injected_line_tokens, injected_block_tokens) =
+                        comment::injected_tokens_for_range(
+                            loader,
+                            syntax,
+                            range.from() as u32,
+                            range.to() as u32,
+                        );
 
-        // only have line comment tokens
+                    let line_token = injected_line_tokens
-        if line_token.is_some() && block_tokens.is_none() {
+                        .as_ref()
-            return comment::toggle_line_comments(doc, selection, line_token);
+                        .and_then(|token| token.first())
-        }
+                        .map(|token| token.as_str())
+                        .or(doc_line_token);
 
-        let split_lines = comment::split_lines_of_selection(text, selection);
+                    let block_tokens = injected_block_tokens.as_deref().or(doc_block_tokens);
 
-        let default_block_tokens = &[BlockCommentToken::default()];
+                    if line_token.is_some() && block_tokens.is_none() {
-        let block_comment_tokens = block_tokens.unwrap_or(default_block_tokens);
+                        return comment::toggle_line_comments(rope, range, line_token);
+                    }
 
-        let (line_commented, line_comment_changes) =
+                    let split_lines = comment::split_lines_of_range(rope.slice(..), range);
-            comment::find_block_comments(block_comment_tokens, text, &split_lines);
 
-        // block commented by line would also be block commented so check this first
+                    let default_block_tokens = &[BlockCommentToken::default()];
-        if line_commented {
+                    let block_comment_tokens = block_tokens.unwrap_or(default_block_tokens);
-            return comment::create_block_comment_transaction(
+
-                doc,
+                    let (line_commented, line_comment_changes) = comment::find_block_comments(
-                &split_lines,
+                        block_comment_tokens,
-                line_commented,
+                        rope.slice(..),
-                line_comment_changes,
+                        &split_lines,
+                    );
+
+                    if line_commented {
+                        return comment::create_block_comment_transaction(
+                            &split_lines,
+                            line_commented,
+                            line_comment_changes,
+                        )
+                        .0;
+                    }
+
+                    let (block_commented, comment_changes) = comment::find_block_comments(
+                        block_comment_tokens,
+                        rope.slice(..),
+                        &[*range],
+                    );
+
+                    if block_commented {
+                        return comment::create_block_comment_transaction(
+                            &[*range],
+                            block_commented,
+                            comment_changes,
+                        )
+                        .0;
+                    }
+
+                    if line_token.is_none() && block_tokens.is_some() {
+                        return comment::create_block_comment_transaction(
+                            &split_lines,
+                            line_commented,
+                            line_comment_changes,
+                        )
+                        .0;
+                    }
+
+                    comment::toggle_line_comments(rope, range, line_token)
+                }),
             )
-            .0;
+        },
-        }
+    );
-
-        let (block_commented, comment_changes) =
-            comment::find_block_comments(block_comment_tokens, text, selection);
-
-        // check if selection has block comments
-        if block_commented {
-            return comment::create_block_comment_transaction(
-                doc,
-                selection,
-                block_commented,
-                comment_changes,
-            )
-            .0;
-        }
-
-        // not commented and only have block comment tokens
-        if line_token.is_none() && block_tokens.is_some() {
-            return comment::create_block_comment_transaction(
-                doc,
-                &split_lines,
-                line_commented,
-                line_comment_changes,
-            )
-            .0;
-        }
-
-        // not block commented at all and don't have any tokens
-        comment::toggle_line_comments(doc, selection, line_token)
-    })
 }
 
 fn toggle_line_comments(cx: &mut Context) {
-    toggle_comments_impl(cx, |line_token, block_tokens, doc, selection| {
+    toggle_comments_impl(
-        if line_token.is_none() && block_tokens.is_some() {
+        cx,
-            let default_block_tokens = &[BlockCommentToken::default()];
+        |rope, selection, doc_line_token, doc_block_tokens, syntax, loader| {
-            let block_comment_tokens = block_tokens.unwrap_or(default_block_tokens);
+            let mut selections = SmallVec::new();
-            comment::toggle_block_comments(
+            let mut added_chars = 0;
-                doc,
+            let mut removed_chars = 0;
-                &comment::split_lines_of_selection(doc.slice(..), selection),
+
-                block_comment_tokens,
+            let transaction = Transaction::change(
-            )
+                rope,
-        } else {
+                selection.iter().flat_map(|range| {
-            comment::toggle_line_comments(doc, selection, line_token)
+                    let (injected_line_tokens, injected_block_tokens) =
-        }
+                        comment::injected_tokens_for_range(
-    });
+                            loader,
+                            syntax,
+                            range.from() as u32,
+                            range.to() as u32,
+                        );
+
+                    let line_token = injected_line_tokens
+                        .as_ref()
+                        .and_then(|tokens| tokens.first())
+                        .map(|token| token.as_str())
+                        .or(doc_line_token);
+
+                    let block_tokens = injected_block_tokens.as_deref().or(doc_block_tokens);
+
+                    if line_token.is_none() && block_tokens.is_some() {
+                        let default_block_tokens = &[BlockCommentToken::default()];
+                        let block_comment_tokens = block_tokens.unwrap_or(default_block_tokens);
+                        let ranges = &comment::split_lines_of_range(rope.slice(..), range);
+                        comment::toggle_block_comments(
+                            rope,
+                            ranges,
+                            block_comment_tokens,
+                            &mut selections,
+                            &mut added_chars,
+                            &mut removed_chars,
+                        )
+                    } else {
+                        comment::toggle_line_comments(rope, range, line_token)
+                    }
+                }),
+            );
+
+            if selections.is_empty() {
+                transaction
+            } else {
+                transaction.with_selection(Selection::new(selections, selection.primary_index()))
+            }
+        },
+    );
 }
 
 fn toggle_block_comments(cx: &mut Context) {
-    toggle_comments_impl(cx, |line_token, block_tokens, doc, selection| {
+    toggle_comments_impl(
-        if line_token.is_some() && block_tokens.is_none() {
+        cx,
-            comment::toggle_line_comments(doc, selection, line_token)
+        |rope, selection, doc_line_token, doc_block_tokens, syntax, loader| {
-        } else {
+            let mut selections = SmallVec::new();
-            let default_block_tokens = &[BlockCommentToken::default()];
+            let mut added_chars = 0;
-            let block_comment_tokens = block_tokens.unwrap_or(default_block_tokens);
+            let mut removed_chars = 0;
-            comment::toggle_block_comments(doc, selection, block_comment_tokens)
+
-        }
+            let transaction = Transaction::change(
-    });
+                rope,
+                selection.iter().flat_map(|range| {
+                    let (injected_line_tokens, injected_block_tokens) =
+                        comment::injected_tokens_for_range(
+                            loader,
+                            syntax,
+                            range.from() as u32,
+                            range.to() as u32,
+                        );
+
+                    let line_token = injected_line_tokens
+                        .as_ref()
+                        .and_then(|token| token.first())
+                        .map(|token| token.as_str())
+                        .or(doc_line_token);
+
+                    let block_tokens = injected_block_tokens.as_deref().or(doc_block_tokens);
+
+                    if line_token.is_some() && block_tokens.is_none() {
+                        comment::toggle_line_comments(rope, range, line_token)
+                    } else {
+                        let default_block_tokens = &[BlockCommentToken::default()];
+                        let block_comment_tokens = block_tokens.unwrap_or(default_block_tokens);
+                        let ranges = vec![*range];
+                        comment::toggle_block_comments(
+                            rope,
+                            &ranges,
+                            block_comment_tokens,
+                            &mut selections,
+                            &mut added_chars,
+                            &mut removed_chars,
+                        )
+                    }
+                }),
+            );
+
+            if selections.is_empty() {
+                transaction
+            } else {
+                transaction.with_selection(Selection::new(selections, selection.primary_index()))
+            }
+        },
+    );
 }
 
 fn rotate_selections(cx: &mut Context, direction: Direction) {

helix-term/tests/integration.rs

@@ -19,6 +19,7 @@ mod test {
     mod auto_pairs;
     mod command_line;
     mod commands;
+    mod comments;
     mod languages;
     mod movement;
     mod splits;

helix-term/tests/test/commands.rs

@@ -653,49 +653,6 @@ async fn test_join_selections_space() -> anyhow::Result<()> {
     Ok(())
 }
 
-#[tokio::test(flavor = "multi_thread")]
-async fn test_join_selections_comment() -> anyhow::Result<()> {
-    test((
-        indoc! {"\
-            /// #[a|]#bc
-            /// def
-        "},
-        ":lang rust<ret>J",
-        indoc! {"\
-            /// #[a|]#bc def
-        "},
-    ))
-    .await?;
-
-    // Only join if the comment token matches the previous line.
-    test((
-        indoc! {"\
-            #[| // a
-            // b
-            /// c
-            /// d
-            e
-            /// f
-            // g]#
-        "},
-        ":lang rust<ret>J",
-        indoc! {"\
-            #[| // a b /// c d e f // g]#
-        "},
-    ))
-    .await?;
-
-    test((
-        "#[|\t// Join comments
-\t// with indent]#",
-        ":lang go<ret>J",
-        "#[|\t// Join comments with indent]#",
-    ))
-    .await?;
-
-    Ok(())
-}
-
 #[tokio::test(flavor = "multi_thread")]
 async fn test_read_file() -> anyhow::Result<()> {
     let mut file = tempfile::NamedTempFile::new()?;

helix-term/tests/test/comments.rs

@@ -0,0 +1,570 @@
+use super::*;
+
+mod simple {
+    use super::*;
+    #[tokio::test(flavor = "multi_thread")]
+    async fn uncomment_inner_injection() -> anyhow::Result<()> {
+        test((
+            indoc! {r#"\
+            <p>Comment toggle on this line should use the HTML comment token(s).</p>
+            <script type="text/javascript">
+              // Comment toggle #[|on this line s]#hould use the javascript comment token(s).
+              foo();
+            </script>
+        "#},
+            ":lang html<ret> c",
+            indoc! {r#"\
+            <p>Comment toggle on this line should use the HTML comment token(s).</p>
+            <script type="text/javascript">
+              Comment toggle #[|on this line s]#hould use the javascript comment token(s).
+              foo();
+            </script>
+        "#},
+        ))
+        .await?;
+
+        Ok(())
+    }
+    #[tokio::test(flavor = "multi_thread")]
+    async fn comment_inner_injection() -> anyhow::Result<()> {
+        test((
+            indoc! {r#"\
+            <p>Comment toggle on this line should use the HTML comment token(s).</p>
+            <script type="text/javascript">
+              Comment toggle #[|on this line s]#hould use the javascript comment token(s).
+              foo();
+            </script>
+        "#},
+            ":lang html<ret> c",
+            indoc! {r#"\
+            <p>Comment toggle on this line should use the HTML comment token(s).</p>
+            <script type="text/javascript">
+              // Comment toggle #[|on this line s]#hould use the javascript comment token(s).
+              foo();
+            </script>
+        "#},
+        ))
+        .await?;
+
+        Ok(())
+    }
+    #[tokio::test(flavor = "multi_thread")]
+    async fn block_comment_inner_injection() -> anyhow::Result<()> {
+        test((
+            indoc! {r#"\
+            <p>Comment toggle on this line should use the HTML comment token(s).</p>
+            <script type="text/javascript">
+              // Comment toggle #[|on this line s]#hould use the javascript comment token(s).
+              foo();
+            </script>
+        "#},
+            ":lang html<ret> C",
+            indoc! {r#"\
+            <p>Comment toggle on this line should use the HTML comment token(s).</p>
+            <script type="text/javascript">
+              // Comment toggle #[|/* on this line s */]#hould use the javascript comment token(s).
+              foo();
+            </script>
+        "#},
+        ))
+        .await?;
+
+        Ok(())
+    }
+    #[tokio::test(flavor = "multi_thread")]
+    async fn block_uncomment_inner_injection() -> anyhow::Result<()> {
+        test((
+            indoc! {r#"\
+            <p>Comment toggle on this line should use the HTML comment token(s).</p>
+            <script type="text/javascript">
+              // Comment toggle #[|/* on this line s */]#hould use the javascript comment token(s).
+              foo();
+            </script>
+        "#},
+            ":lang html<ret> C",
+            indoc! {r#"\
+            <p>Comment toggle on this line should use the HTML comment token(s).</p>
+            <script type="text/javascript">
+              // Comment toggle #[|on this line s]#hould use the javascript comment token(s).
+              foo();
+            </script>
+        "#},
+        ))
+        .await?;
+
+        Ok(())
+    }
+}
+
+mod injected_comment_tokens_continue_comment {
+    use super::*;
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn adds_new_comment_on_newline() -> anyhow::Result<()> {
+        test((
+            indoc! {r#"
+            <p>Some text 1234</p>
+            <script type="text/javascript">
+              // This line should #[|c]#ontinue comments
+              foo();
+            </script>
+        "#},
+            ":lang html<ret>i<ret>",
+            indoc! {r#"
+            <p>Some text 1234</p>
+            <script type="text/javascript">
+              // This line should
+              // #[|c]#ontinue comments
+              foo();
+            </script>
+        "#},
+        ))
+        .await?;
+        Ok(())
+    }
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn continues_comment() -> anyhow::Result<()> {
+        test((
+            indoc! {r#"\
+            <p>Some text 1234</p>
+            <script type="text/javascript">
+              // This line should
+              // #[|c]#ontinue comments
+              foo();
+            </script>
+        "#},
+            ":lang html<ret>i<ret>",
+            indoc! {r#"\
+            <p>Some text 1234</p>
+            <script type="text/javascript">
+              // This line should
+              //
+              // #[|c]#ontinue comments
+              foo();
+            </script>
+        "#},
+        ))
+        .await?;
+        Ok(())
+    }
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn test_injected_comment_tokens_continue_comment_d() -> anyhow::Result<()> {
+    test((
+        indoc! {r#"\
+            <p>Some text 1234</p>
+            <script type="text/javascript">
+              // This line should #[|c]#ontinue comments
+              foo();
+            </script>
+        "#},
+        ":lang html<ret>O",
+        indoc! {r#"\
+            <p>Some text 1234</p>
+            <script type="text/javascript">
+              // #[
+            |]#  // This line should continue comments
+              foo();
+            </script>
+        "#},
+    ))
+    .await?;
+
+    Ok(())
+}
+
+mod multiple_selections_different_injection_layers {
+    use super::*;
+
+    #[tokio::test(flavor = "multi_thread")]
+    async fn comments_two_different_injection_layers_with_different_comments() -> anyhow::Result<()>
+    {
+        test((
+            indoc! {r#"\
+            <p>Comment toggle #[|on this line ]#should use the HTML comment token(s).</p>
+            <script type="text/javascript">
+              Comment toggle #(|on this line )#should use the javascript comment token(s).
+              foo();
+            </script>
+        "#},
+            ":lang html<ret> c",
+            indoc! {r#"\
+            <!-- <p>Comment toggle #[|on this line ]#should use the HTML comment token(s).</p> -->
+            <script type="text/javascript">
+              // Comment toggle #(|on this line )#should use the javascript comment token(s).
+              foo();
+            </script>
+        "#},
+        ))
+        .await?;
+        Ok(())
+    }
+    #[tokio::test(flavor = "multi_thread")]
+    async fn uncomments_two_different_injection_layers_with_different_comments(
+    ) -> anyhow::Result<()> {
+        test((
+            indoc! {r#"\
+            <!-- <p>Comment toggle #[|on this line ]#should use the HTML comment token(s).</p> -->
+            <script type="text/javascript">
+              // Comment toggle #(|on this line )#should use the javascript comment token(s).
+              foo();
+            </script>
+        "#},
+            ":lang html<ret> c",
+            indoc! {r#"\
+            <p>Comment toggle #[|on this line ]#should use the HTML comment token(s).</p>
+            <script type="text/javascript">
+              Comment toggle #(|on this line )#should use the javascript comment token(s).
+              foo();
+            </script>
+        "#},
+        ))
+        .await?;
+        Ok(())
+    }
+    #[tokio::test(flavor = "multi_thread")]
+    async fn works_with_multiple_selections() -> anyhow::Result<()> {
+        test((
+            indoc! {r#"\
+            <p>Comment toggle #(|on this line )#should use the HTML comment token(s).</p>
+            <script type="text/javascript">
+              // Comment toggle #[|on this line ]#should use the javascript comment token(s).
+              foo();
+            </script>
+        "#},
+            ":lang html<ret> c",
+            indoc! {r#"\
+            <!-- <p>Comment toggle #(|on this line )#should use the HTML comment token(s).</p> -->
+            <script type="text/javascript">
+              Comment toggle #[|on this line ]#should use the javascript comment token(s).
+              foo();
+            </script>
+        "#},
+        ))
+        .await?;
+        Ok(())
+    }
+    #[tokio::test(flavor = "multi_thread")]
+    async fn works_with_nested_injection_layers_html_js_then_css() -> anyhow::Result<()> {
+        test((
+            indoc! {r#"\
+            <!-- <p>Comment toggle #(|on this line)# should use the HTML comment token(s).</p> -->
+            <script type="text/javascript">
+              // Comment toggle #(|on this line)# should use the javascript comment token(s).
+              foo();
+              css`
+                h#[tml {
+                  background-color: |]#red;
+                }
+              `
+            </script>
+        "#},
+            ":lang html<ret> c",
+            indoc! {r#"\
+            <p>Comment toggle #(|on this line)# should use the HTML comment token(s).</p>
+            <script type="text/javascript">
+              Comment toggle #(|on this line)# should use the javascript comment token(s).
+              foo();
+              css`
+                /* h#[tml { */
+                  /* background-color: |]#red; */
+                }
+              `
+            </script>
+        "#},
+        ))
+        .await?;
+        Ok(())
+    }
+    #[tokio::test(flavor = "multi_thread")]
+    async fn full_line_selection_commenting() -> anyhow::Result<()> {
+        test((
+            indoc! {r#"\
+            <p>Comment toggle on this line should use the HTML comment token(s).</p>
+            <script type="text/javascript">
+              // Comment toggle on this line should use the javascript comment token(s).
+            #[  foo();
+              css`
+            |]#    html {
+                  background-color: red;
+                }
+              `;
+            </script>
+        "#},
+            ":lang html<ret> c",
+            indoc! {r#"\
+            <p>Comment toggle on this line should use the HTML comment token(s).</p>
+            <script type="text/javascript">
+              // Comment toggle on this line should use the javascript comment token(s).
+            #[  // foo();
+              // css`
+            |]#    html {
+                  background-color: red;
+                }
+              `;
+            </script>
+        "#},
+        ))
+        .await?;
+        test((
+            indoc! {r#"\
+            <p>Comment toggle on this line should use the HTML comment token(s).</p>
+            <script type="text/javascript">
+              // Comment toggle on this line should use the javascript comment token(s).
+            #[  // foo();
+              // css`
+            |]#    html {
+                  background-color: red;
+                }
+              `;
+            </script>
+        "#},
+            ":lang html<ret> c",
+            indoc! {r#"\
+            <p>Comment toggle on this line should use the HTML comment token(s).</p>
+            <script type="text/javascript">
+              // Comment toggle on this line should use the javascript comment token(s).
+            #[  foo();
+              css`
+            |]#    html {
+                  background-color: red;
+                }
+              `;
+            </script>
+        "#},
+        ))
+        .await?;
+        Ok(())
+    }
+    #[tokio::test(flavor = "multi_thread")]
+    async fn block_comment_toggle_across_different_layers() -> anyhow::Result<()> {
+        test((
+            indoc! {r#"\
+            <p>Comment toggle #(|on this line)# should use the HTML comment token(s).</p>
+            <script type="text/javascript">
+              // Comment toggle #[|on this line]# should use the javascript comment token(s).
+              foo();
+            </script>
+        "#},
+            ":lang html<ret> C",
+            indoc! {r#"\
+            <p>Comment toggle #(|<!-- on this line -->)# should use the HTML comment token(s).</p>
+            <script type="text/javascript">
+              // Comment toggle #[|/* on this line */]# should use the javascript comment token(s).
+              foo();
+            </script>
+        "#},
+        ))
+        .await?;
+        Ok(())
+    }
+    #[tokio::test(flavor = "multi_thread")]
+    async fn multiple_selections_same_line() -> anyhow::Result<()> {
+        test((
+            indoc! {r#"\
+                <p>C#[|o]#mment t#(|o)#ggle #(|o)#n this line sh#(|o)#uld use the HTML c#(|o)#mment t#(|o)#ken(s).</p>
+                <script type="text/javascript">
+                  // Comment toggle on this line should use the javascript comment token(s).
+                  foo();
+                  css`
+                    html {
+                      background-color: red;
+                    }
+                  `;
+                </script>
+            "#},
+            ":lang html<ret> C",
+            indoc! {r#"\
+                <p>C#[|<!-- o -->]#mment t#(|<!-- o -->)#ggle #(|<!-- o -->)#n this line sh#(|<!-- o -->)#uld use the HTML c#(|<!-- o -->)#mment t#(|<!-- o -->)#ken(s).</p>
+                <script type="text/javascript">
+                  // Comment toggle on this line should use the javascript comment token(s).
+                  foo();
+                  css`
+                    html {
+                      background-color: red;
+                    }
+                  `;
+                </script>
+            "#},
+        ))
+        .await?;
+        test((
+            indoc! {r#"\
+                <p>C#[|<!-- o -->]#mment t#(|<!-- o -->)#ggle #(|<!-- o -->)#n this line sh#(|<!-- o -->)#uld use the HTML c#(|<!-- o -->)#mment t#(|<!-- o -->)#ken(s).</p>
+                <script type="text/javascript">
+                  // Comment toggle on this line should use the javascript comment token(s).
+                  foo();
+                  css`
+                    html {
+                      background-color: red;
+                    }
+                  `;
+                </script>
+            "#},
+            ":lang html<ret> C",
+            indoc! {r#"\
+                <p>C#[|o]#mment t#(|o)#ggle #(|o)#n this line sh#(|o)#uld use the HTML c#(|o)#mment t#(|o)#ken(s).</p>
+                <script type="text/javascript">
+                  // Comment toggle on this line should use the javascript comment token(s).
+                  foo();
+                  css`
+                    html {
+                      background-color: red;
+                    }
+                  `;
+                </script>
+            "#},
+        ))
+        .await?;
+        Ok(())
+    }
+    #[tokio::test(flavor = "multi_thread")]
+    async fn many_single_line_selections() -> anyhow::Result<()> {
+        test((
+            indoc! {r#"\
+                <p>C#[|o]#mment t#(|o)#ggle #(|o)#n this line sh#(|o)#uld use the HTML c#(|o)#mment t#(|o)#ken(s).</p>
+                <script type="text/javascript">
+                  // C#(|o)#mment t#(|o)#ggle #(|o)#n this line sh#(|o)#uld use the javascript c#(|o)#mment t#(|o)#ken(s).
+                  f#(|o)##(|o)#();
+                  css`
+                    html {
+                      backgr#(|o)#und-c#(|o)#l#(|o)#r: red;
+                    }
+                  `;
+                </script>
+            "#},
+            ":lang html<ret> C",
+            indoc! {r#"\
+                <p>C#[|<!-- o -->]#mment t#(|<!-- o -->)#ggle #(|<!-- o -->)#n this line sh#(|<!-- o -->)#uld use the HTML c#(|<!-- o -->)#mment t#(|<!-- o -->)#ken(s).</p>
+                <script type="text/javascript">
+                  // C#(|/* o */)#mment t#(|/* o */)#ggle #(|/* o */)#n this line sh#(|/* o */)#uld use the javascript c#(|/* o */)#mment t#(|/* o */)#ken(s).
+                  f#(|/* o */)##(|/* o */)#();
+                  css`
+                    html {
+                      backgr#(|/* o */)#und-c#(|/* o */)#l#(|/* o */)#r: red;
+                    }
+                  `;
+                </script>
+            "#},
+        ))
+        .await?;
+        test((
+            indoc! {r#"\
+                <p>C#[|<!-- o -->]#mment t#(|<!-- o -->)#ggle #(|<!-- o -->)#n this line sh#(|<!-- o -->)#uld use the HTML c#(|<!-- o -->)#mment t#(|<!-- o -->)#ken(s).</p>
+                <script type="text/javascript">
+                  // C#(|/* o */)#mment t#(|/* o */)#ggle #(|/* o */)#n this line sh#(|/* o */)#uld use the javascript c#(|/* o */)#mment t#(|/* o */)#ken(s).
+                  f#(|/* o */)##(|/* o */)#();
+                  css`
+                    html {
+                      backgr#(|/* o */)#und-c#(|/* o */)#l#(|/* o */)#r: red;
+                    }
+                  `;
+                </script>
+            "#},
+            ":lang html<ret> C",
+            indoc! {r#"\
+                <p>C#[|o]#mment t#(|o)#ggle #(|o)#n this line sh#(|o)#uld use the HTML c#(|o)#mment t#(|o)#ken(s).</p>
+                <script type="text/javascript">
+                  // C#(|o)#mment t#(|o)#ggle #(|o)#n this line sh#(|o)#uld use the javascript c#(|o)#mment t#(|o)#ken(s).
+                  f#(|o)##(|o)#();
+                  css`
+                    html {
+                      backgr#(|o)#und-c#(|o)#l#(|o)#r: red;
+                    }
+                  `;
+                </script>
+            "#},
+        ))
+        .await?;
+        Ok(())
+    }
+}
+
+/// A selection that spans across several injections takes comment tokens
+/// from the injection with the bigger scope
+#[tokio::test(flavor = "multi_thread")]
+async fn test_injected_comment_tokens_selection_across_different_layers() -> anyhow::Result<()> {
+    test((
+        indoc! {r#"\
+            <p>Comment tog#[|gle on this line should use the HTML comment token(s).</p>
+            <script type="text/javascript">
+              // Comment togg]#le on this line should use the javascript comment token(s).
+              foo();
+            </script>
+        "#},
+        ":lang html<ret> c",
+        indoc! {r#"\
+            <!-- <p>Comment tog#[|gle on this line should use the HTML comment token(s).</p> -->
+            <!-- <script type="text/javascript"> -->
+              <!-- // Comment togg]#le on this line should use the javascript comment token(s). -->
+              foo();
+            </script>
+        "#},
+    ))
+    .await?;
+    test((
+        indoc! {r#"\
+            <p>Comment tog#[|gle on this line should use the HTML comment token(s).</p>
+            <script type="text/javascript">
+              // Comment togg]#le on this line should use the javascript comment token(s).
+              foo();
+            </script>
+        "#},
+        ":lang html<ret> C",
+        indoc! {r#"\
+            <p>Comment tog#[|<!-- gle on this line should use the HTML comment token(s).</p>
+            <script type="text/javascript">
+              // Comment togg -->]#le on this line should use the javascript comment token(s).
+              foo();
+            </script>
+        "#},
+    ))
+    .await?;
+
+    Ok(())
+}
+
+#[tokio::test(flavor = "multi_thread")]
+async fn test_join_selections_comment() -> anyhow::Result<()> {
+    test((
+        indoc! {"\
+            /// #[a|]#bc
+            /// def
+        "},
+        ":lang rust<ret>J",
+        indoc! {"\
+            /// #[a|]#bc def
+        "},
+    ))
+    .await?;
+
+    // Only join if the comment token matches the previous line.
+    test((
+        indoc! {"\
+            #[| // a
+            // b
+            /// c
+            /// d
+            e
+            /// f
+            // g]#
+        "},
+        ":lang rust<ret>J",
+        indoc! {"\
+            #[| // a b /// c d e f // g]#
+        "},
+    ))
+    .await?;
+
+    test((
+        "#[|\t// Join comments
+\t// with indent]#",
+        ":lang go<ret>J",
+        "#[|\t// Join comments with indent]#",
+    ))
+    .await?;
+
+    Ok(())
+}

languages.toml

@@ -1426,8 +1426,7 @@ scope = "source.svelte"
 injection-regex = "svelte"
 file-types = ["svelte"]
 indent = { tab-width = 2, unit = "  " }
-comment-token = "//"
+block-comment-tokens = { start = "<!--", end = "-->" }
-block-comment-tokens = { start = "/*", end = "*/" }
 language-servers = [ "svelteserver" ]
 
 [[grammar]]